Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/bs4/tests/test_tree.py @ 5:9b1c78e6ba9c draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author shellac
date Mon, 01 Jun 2020 08:59:25 -0400
parents 79f47841a781
children
comparison
equal deleted inserted replaced
4:79f47841a781 5:9b1c78e6ba9c
1 # -*- coding: utf-8 -*-
2 """Tests for Beautiful Soup's tree traversal methods.
3
4 The tree traversal methods are the main advantage of using Beautiful
5 Soup over just using a parser.
6
7 Different parsers will build different Beautiful Soup trees given the
8 same markup, but all Beautiful Soup trees can be traversed with the
9 methods tested here.
10 """
11
12 from pdb import set_trace
13 import copy
14 import pickle
15 import re
16 import warnings
17 from bs4 import BeautifulSoup
18 from bs4.builder import (
19 builder_registry,
20 HTMLParserTreeBuilder,
21 )
22 from bs4.element import (
23 PY3K,
24 CData,
25 Comment,
26 Declaration,
27 Doctype,
28 Formatter,
29 NavigableString,
30 Script,
31 SoupStrainer,
32 Stylesheet,
33 Tag,
34 TemplateString,
35 )
36 from bs4.testing import (
37 SoupTest,
38 skipIf,
39 )
40
41 XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None)
42 LXML_PRESENT = (builder_registry.lookup("lxml") is not None)
43
44 class TreeTest(SoupTest):
45
46 def assertSelects(self, tags, should_match):
47 """Make sure that the given tags have the correct text.
48
49 This is used in tests that define a bunch of tags, each
50 containing a single string, and then select certain strings by
51 some mechanism.
52 """
53 self.assertEqual([tag.string for tag in tags], should_match)
54
55 def assertSelectsIDs(self, tags, should_match):
56 """Make sure that the given tags have the correct IDs.
57
58 This is used in tests that define a bunch of tags, each
59 containing a single string, and then select certain strings by
60 some mechanism.
61 """
62 self.assertEqual([tag['id'] for tag in tags], should_match)
63
64
65 class TestFind(TreeTest):
66 """Basic tests of the find() method.
67
68 find() just calls find_all() with limit=1, so it's not tested all
69 that thouroughly here.
70 """
71
72 def test_find_tag(self):
73 soup = self.soup("<a>1</a><b>2</b><a>3</a><b>4</b>")
74 self.assertEqual(soup.find("b").string, "2")
75
76 def test_unicode_text_find(self):
77 soup = self.soup('<h1>Räksmörgås</h1>')
78 self.assertEqual(soup.find(string='Räksmörgås'), 'Räksmörgås')
79
80 def test_unicode_attribute_find(self):
81 soup = self.soup('<h1 id="Räksmörgås">here it is</h1>')
82 str(soup)
83 self.assertEqual("here it is", soup.find(id='Räksmörgås').text)
84
85
86 def test_find_everything(self):
87 """Test an optimization that finds all tags."""
88 soup = self.soup("<a>foo</a><b>bar</b>")
89 self.assertEqual(2, len(soup.find_all()))
90
91 def test_find_everything_with_name(self):
92 """Test an optimization that finds all tags with a given name."""
93 soup = self.soup("<a>foo</a><b>bar</b><a>baz</a>")
94 self.assertEqual(2, len(soup.find_all('a')))
95
96 class TestFindAll(TreeTest):
97 """Basic tests of the find_all() method."""
98
99 def test_find_all_text_nodes(self):
100 """You can search the tree for text nodes."""
101 soup = self.soup("<html>Foo<b>bar</b>\xbb</html>")
102 # Exact match.
103 self.assertEqual(soup.find_all(string="bar"), ["bar"])
104 self.assertEqual(soup.find_all(text="bar"), ["bar"])
105 # Match any of a number of strings.
106 self.assertEqual(
107 soup.find_all(text=["Foo", "bar"]), ["Foo", "bar"])
108 # Match a regular expression.
109 self.assertEqual(soup.find_all(text=re.compile('.*')),
110 ["Foo", "bar", '\xbb'])
111 # Match anything.
112 self.assertEqual(soup.find_all(text=True),
113 ["Foo", "bar", '\xbb'])
114
115 def test_find_all_limit(self):
116 """You can limit the number of items returned by find_all."""
117 soup = self.soup("<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>")
118 self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"])
119 self.assertSelects(soup.find_all('a', limit=1), ["1"])
120 self.assertSelects(
121 soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"])
122
123 # A limit of 0 means no limit.
124 self.assertSelects(
125 soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"])
126
127 def test_calling_a_tag_is_calling_findall(self):
128 soup = self.soup("<a>1</a><b>2<a id='foo'>3</a></b>")
129 self.assertSelects(soup('a', limit=1), ["1"])
130 self.assertSelects(soup.b(id="foo"), ["3"])
131
132 def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self):
133 soup = self.soup("<a></a>")
134 # Create a self-referential list.
135 l = []
136 l.append(l)
137
138 # Without special code in _normalize_search_value, this would cause infinite
139 # recursion.
140 self.assertEqual([], soup.find_all(l))
141
142 def test_find_all_resultset(self):
143 """All find_all calls return a ResultSet"""
144 soup = self.soup("<a></a>")
145 result = soup.find_all("a")
146 self.assertTrue(hasattr(result, "source"))
147
148 result = soup.find_all(True)
149 self.assertTrue(hasattr(result, "source"))
150
151 result = soup.find_all(text="foo")
152 self.assertTrue(hasattr(result, "source"))
153
154
155 class TestFindAllBasicNamespaces(TreeTest):
156
157 def test_find_by_namespaced_name(self):
158 soup = self.soup('<mathml:msqrt>4</mathml:msqrt><a svg:fill="red">')
159 self.assertEqual("4", soup.find("mathml:msqrt").string)
160 self.assertEqual("a", soup.find(attrs= { "svg:fill" : "red" }).name)
161
162
163 class TestFindAllByName(TreeTest):
164 """Test ways of finding tags by tag name."""
165
166 def setUp(self):
167 super(TreeTest, self).setUp()
168 self.tree = self.soup("""<a>First tag.</a>
169 <b>Second tag.</b>
170 <c>Third <a>Nested tag.</a> tag.</c>""")
171
172 def test_find_all_by_tag_name(self):
173 # Find all the <a> tags.
174 self.assertSelects(
175 self.tree.find_all('a'), ['First tag.', 'Nested tag.'])
176
177 def test_find_all_by_name_and_text(self):
178 self.assertSelects(
179 self.tree.find_all('a', text='First tag.'), ['First tag.'])
180
181 self.assertSelects(
182 self.tree.find_all('a', text=True), ['First tag.', 'Nested tag.'])
183
184 self.assertSelects(
185 self.tree.find_all('a', text=re.compile("tag")),
186 ['First tag.', 'Nested tag.'])
187
188
189 def test_find_all_on_non_root_element(self):
190 # You can call find_all on any node, not just the root.
191 self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.'])
192
193 def test_calling_element_invokes_find_all(self):
194 self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.'])
195
196 def test_find_all_by_tag_strainer(self):
197 self.assertSelects(
198 self.tree.find_all(SoupStrainer('a')),
199 ['First tag.', 'Nested tag.'])
200
201 def test_find_all_by_tag_names(self):
202 self.assertSelects(
203 self.tree.find_all(['a', 'b']),
204 ['First tag.', 'Second tag.', 'Nested tag.'])
205
206 def test_find_all_by_tag_dict(self):
207 self.assertSelects(
208 self.tree.find_all({'a' : True, 'b' : True}),
209 ['First tag.', 'Second tag.', 'Nested tag.'])
210
211 def test_find_all_by_tag_re(self):
212 self.assertSelects(
213 self.tree.find_all(re.compile('^[ab]$')),
214 ['First tag.', 'Second tag.', 'Nested tag.'])
215
216 def test_find_all_with_tags_matching_method(self):
217 # You can define an oracle method that determines whether
218 # a tag matches the search.
219 def id_matches_name(tag):
220 return tag.name == tag.get('id')
221
222 tree = self.soup("""<a id="a">Match 1.</a>
223 <a id="1">Does not match.</a>
224 <b id="b">Match 2.</a>""")
225
226 self.assertSelects(
227 tree.find_all(id_matches_name), ["Match 1.", "Match 2."])
228
229 def test_find_with_multi_valued_attribute(self):
230 soup = self.soup(
231 "<div class='a b'>1</div><div class='a c'>2</div><div class='a d'>3</div>"
232 )
233 r1 = soup.find('div', 'a d');
234 r2 = soup.find('div', re.compile(r'a d'));
235 r3, r4 = soup.find_all('div', ['a b', 'a d']);
236 self.assertEqual('3', r1.string)
237 self.assertEqual('3', r2.string)
238 self.assertEqual('1', r3.string)
239 self.assertEqual('3', r4.string)
240
241
242 class TestFindAllByAttribute(TreeTest):
243
244 def test_find_all_by_attribute_name(self):
245 # You can pass in keyword arguments to find_all to search by
246 # attribute.
247 tree = self.soup("""
248 <a id="first">Matching a.</a>
249 <a id="second">
250 Non-matching <b id="first">Matching b.</b>a.
251 </a>""")
252 self.assertSelects(tree.find_all(id='first'),
253 ["Matching a.", "Matching b."])
254
255 def test_find_all_by_utf8_attribute_value(self):
256 peace = "םולש".encode("utf8")
257 data = '<a title="םולש"></a>'.encode("utf8")
258 soup = self.soup(data)
259 self.assertEqual([soup.a], soup.find_all(title=peace))
260 self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8")))
261 self.assertEqual([soup.a], soup.find_all(title=[peace, "something else"]))
262
263 def test_find_all_by_attribute_dict(self):
264 # You can pass in a dictionary as the argument 'attrs'. This
265 # lets you search for attributes like 'name' (a fixed argument
266 # to find_all) and 'class' (a reserved word in Python.)
267 tree = self.soup("""
268 <a name="name1" class="class1">Name match.</a>
269 <a name="name2" class="class2">Class match.</a>
270 <a name="name3" class="class3">Non-match.</a>
271 <name1>A tag called 'name1'.</name1>
272 """)
273
274 # This doesn't do what you want.
275 self.assertSelects(tree.find_all(name='name1'),
276 ["A tag called 'name1'."])
277 # This does what you want.
278 self.assertSelects(tree.find_all(attrs={'name' : 'name1'}),
279 ["Name match."])
280
281 self.assertSelects(tree.find_all(attrs={'class' : 'class2'}),
282 ["Class match."])
283
284 def test_find_all_by_class(self):
285 tree = self.soup("""
286 <a class="1">Class 1.</a>
287 <a class="2">Class 2.</a>
288 <b class="1">Class 1.</b>
289 <c class="3 4">Class 3 and 4.</c>
290 """)
291
292 # Passing in the class_ keyword argument will search against
293 # the 'class' attribute.
294 self.assertSelects(tree.find_all('a', class_='1'), ['Class 1.'])
295 self.assertSelects(tree.find_all('c', class_='3'), ['Class 3 and 4.'])
296 self.assertSelects(tree.find_all('c', class_='4'), ['Class 3 and 4.'])
297
298 # Passing in a string to 'attrs' will also search the CSS class.
299 self.assertSelects(tree.find_all('a', '1'), ['Class 1.'])
300 self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.'])
301 self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.'])
302 self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.'])
303
304 def test_find_by_class_when_multiple_classes_present(self):
305 tree = self.soup("<gar class='foo bar'>Found it</gar>")
306
307 f = tree.find_all("gar", class_=re.compile("o"))
308 self.assertSelects(f, ["Found it"])
309
310 f = tree.find_all("gar", class_=re.compile("a"))
311 self.assertSelects(f, ["Found it"])
312
313 # If the search fails to match the individual strings "foo" and "bar",
314 # it will be tried against the combined string "foo bar".
315 f = tree.find_all("gar", class_=re.compile("o b"))
316 self.assertSelects(f, ["Found it"])
317
318 def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self):
319 soup = self.soup("<a class='bar'>Found it</a>")
320
321 self.assertSelects(soup.find_all("a", re.compile("ba")), ["Found it"])
322
323 def big_attribute_value(value):
324 return len(value) > 3
325
326 self.assertSelects(soup.find_all("a", big_attribute_value), [])
327
328 def small_attribute_value(value):
329 return len(value) <= 3
330
331 self.assertSelects(
332 soup.find_all("a", small_attribute_value), ["Found it"])
333
334 def test_find_all_with_string_for_attrs_finds_multiple_classes(self):
335 soup = self.soup('<a class="foo bar"></a><a class="foo"></a>')
336 a, a2 = soup.find_all("a")
337 self.assertEqual([a, a2], soup.find_all("a", "foo"))
338 self.assertEqual([a], soup.find_all("a", "bar"))
339
340 # If you specify the class as a string that contains a
341 # space, only that specific value will be found.
342 self.assertEqual([a], soup.find_all("a", class_="foo bar"))
343 self.assertEqual([a], soup.find_all("a", "foo bar"))
344 self.assertEqual([], soup.find_all("a", "bar foo"))
345
346 def test_find_all_by_attribute_soupstrainer(self):
347 tree = self.soup("""
348 <a id="first">Match.</a>
349 <a id="second">Non-match.</a>""")
350
351 strainer = SoupStrainer(attrs={'id' : 'first'})
352 self.assertSelects(tree.find_all(strainer), ['Match.'])
353
354 def test_find_all_with_missing_attribute(self):
355 # You can pass in None as the value of an attribute to find_all.
356 # This will match tags that do not have that attribute set.
357 tree = self.soup("""<a id="1">ID present.</a>
358 <a>No ID present.</a>
359 <a id="">ID is empty.</a>""")
360 self.assertSelects(tree.find_all('a', id=None), ["No ID present."])
361
362 def test_find_all_with_defined_attribute(self):
363 # You can pass in None as the value of an attribute to find_all.
364 # This will match tags that have that attribute set to any value.
365 tree = self.soup("""<a id="1">ID present.</a>
366 <a>No ID present.</a>
367 <a id="">ID is empty.</a>""")
368 self.assertSelects(
369 tree.find_all(id=True), ["ID present.", "ID is empty."])
370
371 def test_find_all_with_numeric_attribute(self):
372 # If you search for a number, it's treated as a string.
373 tree = self.soup("""<a id=1>Unquoted attribute.</a>
374 <a id="1">Quoted attribute.</a>""")
375
376 expected = ["Unquoted attribute.", "Quoted attribute."]
377 self.assertSelects(tree.find_all(id=1), expected)
378 self.assertSelects(tree.find_all(id="1"), expected)
379
380 def test_find_all_with_list_attribute_values(self):
381 # You can pass a list of attribute values instead of just one,
382 # and you'll get tags that match any of the values.
383 tree = self.soup("""<a id="1">1</a>
384 <a id="2">2</a>
385 <a id="3">3</a>
386 <a>No ID.</a>""")
387 self.assertSelects(tree.find_all(id=["1", "3", "4"]),
388 ["1", "3"])
389
390 def test_find_all_with_regular_expression_attribute_value(self):
391 # You can pass a regular expression as an attribute value, and
392 # you'll get tags whose values for that attribute match the
393 # regular expression.
394 tree = self.soup("""<a id="a">One a.</a>
395 <a id="aa">Two as.</a>
396 <a id="ab">Mixed as and bs.</a>
397 <a id="b">One b.</a>
398 <a>No ID.</a>""")
399
400 self.assertSelects(tree.find_all(id=re.compile("^a+$")),
401 ["One a.", "Two as."])
402
403 def test_find_by_name_and_containing_string(self):
404 soup = self.soup("<b>foo</b><b>bar</b><a>foo</a>")
405 a = soup.a
406
407 self.assertEqual([a], soup.find_all("a", text="foo"))
408 self.assertEqual([], soup.find_all("a", text="bar"))
409 self.assertEqual([], soup.find_all("a", text="bar"))
410
411 def test_find_by_name_and_containing_string_when_string_is_buried(self):
412 soup = self.soup("<a>foo</a><a><b><c>foo</c></b></a>")
413 self.assertEqual(soup.find_all("a"), soup.find_all("a", text="foo"))
414
415 def test_find_by_attribute_and_containing_string(self):
416 soup = self.soup('<b id="1">foo</b><a id="2">foo</a>')
417 a = soup.a
418
419 self.assertEqual([a], soup.find_all(id=2, text="foo"))
420 self.assertEqual([], soup.find_all(id=1, text="bar"))
421
422
423 class TestSmooth(TreeTest):
424 """Test Tag.smooth."""
425
426 def test_smooth(self):
427 soup = self.soup("<div>a</div>")
428 div = soup.div
429 div.append("b")
430 div.append("c")
431 div.append(Comment("Comment 1"))
432 div.append(Comment("Comment 2"))
433 div.append("d")
434 builder = self.default_builder()
435 span = Tag(soup, builder, 'span')
436 span.append('1')
437 span.append('2')
438 div.append(span)
439
440 # At this point the tree has a bunch of adjacent
441 # NavigableStrings. This is normal, but it has no meaning in
442 # terms of HTML, so we may want to smooth things out for
443 # output.
444
445 # Since the <span> tag has two children, its .string is None.
446 self.assertEqual(None, div.span.string)
447
448 self.assertEqual(7, len(div.contents))
449 div.smooth()
450 self.assertEqual(5, len(div.contents))
451
452 # The three strings at the beginning of div.contents have been
453 # merged into on string.
454 #
455 self.assertEqual('abc', div.contents[0])
456
457 # The call is recursive -- the <span> tag was also smoothed.
458 self.assertEqual('12', div.span.string)
459
460 # The two comments have _not_ been merged, even though
461 # comments are strings. Merging comments would change the
462 # meaning of the HTML.
463 self.assertEqual('Comment 1', div.contents[1])
464 self.assertEqual('Comment 2', div.contents[2])
465
466
467 class TestIndex(TreeTest):
468 """Test Tag.index"""
469 def test_index(self):
470 tree = self.soup("""<div>
471 <a>Identical</a>
472 <b>Not identical</b>
473 <a>Identical</a>
474
475 <c><d>Identical with child</d></c>
476 <b>Also not identical</b>
477 <c><d>Identical with child</d></c>
478 </div>""")
479 div = tree.div
480 for i, element in enumerate(div.contents):
481 self.assertEqual(i, div.index(element))
482 self.assertRaises(ValueError, tree.index, 1)
483
484
485 class TestParentOperations(TreeTest):
486 """Test navigation and searching through an element's parents."""
487
488 def setUp(self):
489 super(TestParentOperations, self).setUp()
490 self.tree = self.soup('''<ul id="empty"></ul>
491 <ul id="top">
492 <ul id="middle">
493 <ul id="bottom">
494 <b>Start here</b>
495 </ul>
496 </ul>''')
497 self.start = self.tree.b
498
499
500 def test_parent(self):
501 self.assertEqual(self.start.parent['id'], 'bottom')
502 self.assertEqual(self.start.parent.parent['id'], 'middle')
503 self.assertEqual(self.start.parent.parent.parent['id'], 'top')
504
505 def test_parent_of_top_tag_is_soup_object(self):
506 top_tag = self.tree.contents[0]
507 self.assertEqual(top_tag.parent, self.tree)
508
509 def test_soup_object_has_no_parent(self):
510 self.assertEqual(None, self.tree.parent)
511
512 def test_find_parents(self):
513 self.assertSelectsIDs(
514 self.start.find_parents('ul'), ['bottom', 'middle', 'top'])
515 self.assertSelectsIDs(
516 self.start.find_parents('ul', id="middle"), ['middle'])
517
518 def test_find_parent(self):
519 self.assertEqual(self.start.find_parent('ul')['id'], 'bottom')
520 self.assertEqual(self.start.find_parent('ul', id='top')['id'], 'top')
521
522 def test_parent_of_text_element(self):
523 text = self.tree.find(text="Start here")
524 self.assertEqual(text.parent.name, 'b')
525
526 def test_text_element_find_parent(self):
527 text = self.tree.find(text="Start here")
528 self.assertEqual(text.find_parent('ul')['id'], 'bottom')
529
530 def test_parent_generator(self):
531 parents = [parent['id'] for parent in self.start.parents
532 if parent is not None and 'id' in parent.attrs]
533 self.assertEqual(parents, ['bottom', 'middle', 'top'])
534
535
536 class ProximityTest(TreeTest):
537
538 def setUp(self):
539 super(TreeTest, self).setUp()
540 self.tree = self.soup(
541 '<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>')
542
543
544 class TestNextOperations(ProximityTest):
545
546 def setUp(self):
547 super(TestNextOperations, self).setUp()
548 self.start = self.tree.b
549
550 def test_next(self):
551 self.assertEqual(self.start.next_element, "One")
552 self.assertEqual(self.start.next_element.next_element['id'], "2")
553
554 def test_next_of_last_item_is_none(self):
555 last = self.tree.find(text="Three")
556 self.assertEqual(last.next_element, None)
557
558 def test_next_of_root_is_none(self):
559 # The document root is outside the next/previous chain.
560 self.assertEqual(self.tree.next_element, None)
561
562 def test_find_all_next(self):
563 self.assertSelects(self.start.find_all_next('b'), ["Two", "Three"])
564 self.start.find_all_next(id=3)
565 self.assertSelects(self.start.find_all_next(id=3), ["Three"])
566
567 def test_find_next(self):
568 self.assertEqual(self.start.find_next('b')['id'], '2')
569 self.assertEqual(self.start.find_next(text="Three"), "Three")
570
571 def test_find_next_for_text_element(self):
572 text = self.tree.find(text="One")
573 self.assertEqual(text.find_next("b").string, "Two")
574 self.assertSelects(text.find_all_next("b"), ["Two", "Three"])
575
576 def test_next_generator(self):
577 start = self.tree.find(text="Two")
578 successors = [node for node in start.next_elements]
579 # There are two successors: the final <b> tag and its text contents.
580 tag, contents = successors
581 self.assertEqual(tag['id'], '3')
582 self.assertEqual(contents, "Three")
583
584 class TestPreviousOperations(ProximityTest):
585
586 def setUp(self):
587 super(TestPreviousOperations, self).setUp()
588 self.end = self.tree.find(text="Three")
589
590 def test_previous(self):
591 self.assertEqual(self.end.previous_element['id'], "3")
592 self.assertEqual(self.end.previous_element.previous_element, "Two")
593
594 def test_previous_of_first_item_is_none(self):
595 first = self.tree.find('html')
596 self.assertEqual(first.previous_element, None)
597
598 def test_previous_of_root_is_none(self):
599 # The document root is outside the next/previous chain.
600 # XXX This is broken!
601 #self.assertEqual(self.tree.previous_element, None)
602 pass
603
604 def test_find_all_previous(self):
605 # The <b> tag containing the "Three" node is the predecessor
606 # of the "Three" node itself, which is why "Three" shows up
607 # here.
608 self.assertSelects(
609 self.end.find_all_previous('b'), ["Three", "Two", "One"])
610 self.assertSelects(self.end.find_all_previous(id=1), ["One"])
611
612 def test_find_previous(self):
613 self.assertEqual(self.end.find_previous('b')['id'], '3')
614 self.assertEqual(self.end.find_previous(text="One"), "One")
615
616 def test_find_previous_for_text_element(self):
617 text = self.tree.find(text="Three")
618 self.assertEqual(text.find_previous("b").string, "Three")
619 self.assertSelects(
620 text.find_all_previous("b"), ["Three", "Two", "One"])
621
622 def test_previous_generator(self):
623 start = self.tree.find(text="One")
624 predecessors = [node for node in start.previous_elements]
625
626 # There are four predecessors: the <b> tag containing "One"
627 # the <body> tag, the <head> tag, and the <html> tag.
628 b, body, head, html = predecessors
629 self.assertEqual(b['id'], '1')
630 self.assertEqual(body.name, "body")
631 self.assertEqual(head.name, "head")
632 self.assertEqual(html.name, "html")
633
634
635 class SiblingTest(TreeTest):
636
637 def setUp(self):
638 super(SiblingTest, self).setUp()
639 markup = '''<html>
640 <span id="1">
641 <span id="1.1"></span>
642 </span>
643 <span id="2">
644 <span id="2.1"></span>
645 </span>
646 <span id="3">
647 <span id="3.1"></span>
648 </span>
649 <span id="4"></span>
650 </html>'''
651 # All that whitespace looks good but makes the tests more
652 # difficult. Get rid of it.
653 markup = re.compile(r"\n\s*").sub("", markup)
654 self.tree = self.soup(markup)
655
656
657 class TestNextSibling(SiblingTest):
658
659 def setUp(self):
660 super(TestNextSibling, self).setUp()
661 self.start = self.tree.find(id="1")
662
663 def test_next_sibling_of_root_is_none(self):
664 self.assertEqual(self.tree.next_sibling, None)
665
666 def test_next_sibling(self):
667 self.assertEqual(self.start.next_sibling['id'], '2')
668 self.assertEqual(self.start.next_sibling.next_sibling['id'], '3')
669
670 # Note the difference between next_sibling and next_element.
671 self.assertEqual(self.start.next_element['id'], '1.1')
672
673 def test_next_sibling_may_not_exist(self):
674 self.assertEqual(self.tree.html.next_sibling, None)
675
676 nested_span = self.tree.find(id="1.1")
677 self.assertEqual(nested_span.next_sibling, None)
678
679 last_span = self.tree.find(id="4")
680 self.assertEqual(last_span.next_sibling, None)
681
682 def test_find_next_sibling(self):
683 self.assertEqual(self.start.find_next_sibling('span')['id'], '2')
684
685 def test_next_siblings(self):
686 self.assertSelectsIDs(self.start.find_next_siblings("span"),
687 ['2', '3', '4'])
688
689 self.assertSelectsIDs(self.start.find_next_siblings(id='3'), ['3'])
690
691 def test_next_sibling_for_text_element(self):
692 soup = self.soup("Foo<b>bar</b>baz")
693 start = soup.find(text="Foo")
694 self.assertEqual(start.next_sibling.name, 'b')
695 self.assertEqual(start.next_sibling.next_sibling, 'baz')
696
697 self.assertSelects(start.find_next_siblings('b'), ['bar'])
698 self.assertEqual(start.find_next_sibling(text="baz"), "baz")
699 self.assertEqual(start.find_next_sibling(text="nonesuch"), None)
700
701
702 class TestPreviousSibling(SiblingTest):
703
704 def setUp(self):
705 super(TestPreviousSibling, self).setUp()
706 self.end = self.tree.find(id="4")
707
708 def test_previous_sibling_of_root_is_none(self):
709 self.assertEqual(self.tree.previous_sibling, None)
710
711 def test_previous_sibling(self):
712 self.assertEqual(self.end.previous_sibling['id'], '3')
713 self.assertEqual(self.end.previous_sibling.previous_sibling['id'], '2')
714
715 # Note the difference between previous_sibling and previous_element.
716 self.assertEqual(self.end.previous_element['id'], '3.1')
717
718 def test_previous_sibling_may_not_exist(self):
719 self.assertEqual(self.tree.html.previous_sibling, None)
720
721 nested_span = self.tree.find(id="1.1")
722 self.assertEqual(nested_span.previous_sibling, None)
723
724 first_span = self.tree.find(id="1")
725 self.assertEqual(first_span.previous_sibling, None)
726
727 def test_find_previous_sibling(self):
728 self.assertEqual(self.end.find_previous_sibling('span')['id'], '3')
729
730 def test_previous_siblings(self):
731 self.assertSelectsIDs(self.end.find_previous_siblings("span"),
732 ['3', '2', '1'])
733
734 self.assertSelectsIDs(self.end.find_previous_siblings(id='1'), ['1'])
735
736 def test_previous_sibling_for_text_element(self):
737 soup = self.soup("Foo<b>bar</b>baz")
738 start = soup.find(text="baz")
739 self.assertEqual(start.previous_sibling.name, 'b')
740 self.assertEqual(start.previous_sibling.previous_sibling, 'Foo')
741
742 self.assertSelects(start.find_previous_siblings('b'), ['bar'])
743 self.assertEqual(start.find_previous_sibling(text="Foo"), "Foo")
744 self.assertEqual(start.find_previous_sibling(text="nonesuch"), None)
745
746
747 class TestTag(SoupTest):
748
749 # Test various methods of Tag.
750
751 def test__should_pretty_print(self):
752 # Test the rules about when a tag should be pretty-printed.
753 tag = self.soup("").new_tag("a_tag")
754
755 # No list of whitespace-preserving tags -> pretty-print
756 tag._preserve_whitespace_tags = None
757 self.assertEqual(True, tag._should_pretty_print(0))
758
759 # List exists but tag is not on the list -> pretty-print
760 tag.preserve_whitespace_tags = ["some_other_tag"]
761 self.assertEqual(True, tag._should_pretty_print(1))
762
763 # Indent level is None -> don't pretty-print
764 self.assertEqual(False, tag._should_pretty_print(None))
765
766 # Tag is on the whitespace-preserving list -> don't pretty-print
767 tag.preserve_whitespace_tags = ["some_other_tag", "a_tag"]
768 self.assertEqual(False, tag._should_pretty_print(1))
769
770
771 class TestTagCreation(SoupTest):
772 """Test the ability to create new tags."""
773 def test_new_tag(self):
774 soup = self.soup("")
775 new_tag = soup.new_tag("foo", bar="baz", attrs={"name": "a name"})
776 self.assertTrue(isinstance(new_tag, Tag))
777 self.assertEqual("foo", new_tag.name)
778 self.assertEqual(dict(bar="baz", name="a name"), new_tag.attrs)
779 self.assertEqual(None, new_tag.parent)
780
781 def test_tag_inherits_self_closing_rules_from_builder(self):
782 if XML_BUILDER_PRESENT:
783 xml_soup = BeautifulSoup("", "lxml-xml")
784 xml_br = xml_soup.new_tag("br")
785 xml_p = xml_soup.new_tag("p")
786
787 # Both the <br> and <p> tag are empty-element, just because
788 # they have no contents.
789 self.assertEqual(b"<br/>", xml_br.encode())
790 self.assertEqual(b"<p/>", xml_p.encode())
791
792 html_soup = BeautifulSoup("", "html.parser")
793 html_br = html_soup.new_tag("br")
794 html_p = html_soup.new_tag("p")
795
796 # The HTML builder users HTML's rules about which tags are
797 # empty-element tags, and the new tags reflect these rules.
798 self.assertEqual(b"<br/>", html_br.encode())
799 self.assertEqual(b"<p></p>", html_p.encode())
800
801 def test_new_string_creates_navigablestring(self):
802 soup = self.soup("")
803 s = soup.new_string("foo")
804 self.assertEqual("foo", s)
805 self.assertTrue(isinstance(s, NavigableString))
806
807 def test_new_string_can_create_navigablestring_subclass(self):
808 soup = self.soup("")
809 s = soup.new_string("foo", Comment)
810 self.assertEqual("foo", s)
811 self.assertTrue(isinstance(s, Comment))
812
813 class TestTreeModification(SoupTest):
814
815 def test_attribute_modification(self):
816 soup = self.soup('<a id="1"></a>')
817 soup.a['id'] = 2
818 self.assertEqual(soup.decode(), self.document_for('<a id="2"></a>'))
819 del(soup.a['id'])
820 self.assertEqual(soup.decode(), self.document_for('<a></a>'))
821 soup.a['id2'] = 'foo'
822 self.assertEqual(soup.decode(), self.document_for('<a id2="foo"></a>'))
823
824 def test_new_tag_creation(self):
825 builder = builder_registry.lookup('html')()
826 soup = self.soup("<body></body>", builder=builder)
827 a = Tag(soup, builder, 'a')
828 ol = Tag(soup, builder, 'ol')
829 a['href'] = 'http://foo.com/'
830 soup.body.insert(0, a)
831 soup.body.insert(1, ol)
832 self.assertEqual(
833 soup.body.encode(),
834 b'<body><a href="http://foo.com/"></a><ol></ol></body>')
835
836 def test_append_to_contents_moves_tag(self):
837 doc = """<p id="1">Don't leave me <b>here</b>.</p>
838 <p id="2">Don\'t leave!</p>"""
839 soup = self.soup(doc)
840 second_para = soup.find(id='2')
841 bold = soup.b
842
843 # Move the <b> tag to the end of the second paragraph.
844 soup.find(id='2').append(soup.b)
845
846 # The <b> tag is now a child of the second paragraph.
847 self.assertEqual(bold.parent, second_para)
848
849 self.assertEqual(
850 soup.decode(), self.document_for(
851 '<p id="1">Don\'t leave me .</p>\n'
852 '<p id="2">Don\'t leave!<b>here</b></p>'))
853
854 def test_replace_with_returns_thing_that_was_replaced(self):
855 text = "<a></a><b><c></c></b>"
856 soup = self.soup(text)
857 a = soup.a
858 new_a = a.replace_with(soup.c)
859 self.assertEqual(a, new_a)
860
861 def test_unwrap_returns_thing_that_was_replaced(self):
862 text = "<a><b></b><c></c></a>"
863 soup = self.soup(text)
864 a = soup.a
865 new_a = a.unwrap()
866 self.assertEqual(a, new_a)
867
868 def test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent(self):
869 soup = self.soup("<a><b>Foo</b></a><c>Bar</c>")
870 a = soup.a
871 a.extract()
872 self.assertEqual(None, a.parent)
873 self.assertRaises(ValueError, a.unwrap)
874 self.assertRaises(ValueError, a.replace_with, soup.c)
875
876 def test_replace_tag_with_itself(self):
877 text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>"
878 soup = self.soup(text)
879 c = soup.c
880 soup.c.replace_with(c)
881 self.assertEqual(soup.decode(), self.document_for(text))
882
883 def test_replace_tag_with_its_parent_raises_exception(self):
884 text = "<a><b></b></a>"
885 soup = self.soup(text)
886 self.assertRaises(ValueError, soup.b.replace_with, soup.a)
887
888 def test_insert_tag_into_itself_raises_exception(self):
889 text = "<a><b></b></a>"
890 soup = self.soup(text)
891 self.assertRaises(ValueError, soup.a.insert, 0, soup.a)
892
893 def test_insert_beautifulsoup_object_inserts_children(self):
894 """Inserting one BeautifulSoup object into another actually inserts all
895 of its children -- you'll never combine BeautifulSoup objects.
896 """
897 soup = self.soup("<p>And now, a word:</p><p>And we're back.</p>")
898
899 text = "<p>p2</p><p>p3</p>"
900 to_insert = self.soup(text)
901 soup.insert(1, to_insert)
902
903 for i in soup.descendants:
904 assert not isinstance(i, BeautifulSoup)
905
906 p1, p2, p3, p4 = list(soup.children)
907 self.assertEqual("And now, a word:", p1.string)
908 self.assertEqual("p2", p2.string)
909 self.assertEqual("p3", p3.string)
910 self.assertEqual("And we're back.", p4.string)
911
912
913 def test_replace_with_maintains_next_element_throughout(self):
914 soup = self.soup('<p><a>one</a><b>three</b></p>')
915 a = soup.a
916 b = a.contents[0]
917 # Make it so the <a> tag has two text children.
918 a.insert(1, "two")
919
920 # Now replace each one with the empty string.
921 left, right = a.contents
922 left.replaceWith('')
923 right.replaceWith('')
924
925 # The <b> tag is still connected to the tree.
926 self.assertEqual("three", soup.b.string)
927
928 def test_replace_final_node(self):
929 soup = self.soup("<b>Argh!</b>")
930 soup.find(text="Argh!").replace_with("Hooray!")
931 new_text = soup.find(text="Hooray!")
932 b = soup.b
933 self.assertEqual(new_text.previous_element, b)
934 self.assertEqual(new_text.parent, b)
935 self.assertEqual(new_text.previous_element.next_element, new_text)
936 self.assertEqual(new_text.next_element, None)
937
938 def test_consecutive_text_nodes(self):
939 # A builder should never create two consecutive text nodes,
940 # but if you insert one next to another, Beautiful Soup will
941 # handle it correctly.
942 soup = self.soup("<a><b>Argh!</b><c></c></a>")
943 soup.b.insert(1, "Hooray!")
944
945 self.assertEqual(
946 soup.decode(), self.document_for(
947 "<a><b>Argh!Hooray!</b><c></c></a>"))
948
949 new_text = soup.find(text="Hooray!")
950 self.assertEqual(new_text.previous_element, "Argh!")
951 self.assertEqual(new_text.previous_element.next_element, new_text)
952
953 self.assertEqual(new_text.previous_sibling, "Argh!")
954 self.assertEqual(new_text.previous_sibling.next_sibling, new_text)
955
956 self.assertEqual(new_text.next_sibling, None)
957 self.assertEqual(new_text.next_element, soup.c)
958
959 def test_insert_string(self):
960 soup = self.soup("<a></a>")
961 soup.a.insert(0, "bar")
962 soup.a.insert(0, "foo")
963 # The string were added to the tag.
964 self.assertEqual(["foo", "bar"], soup.a.contents)
965 # And they were converted to NavigableStrings.
966 self.assertEqual(soup.a.contents[0].next_element, "bar")
967
968 def test_insert_tag(self):
969 builder = self.default_builder()
970 soup = self.soup(
971 "<a><b>Find</b><c>lady!</c><d></d></a>", builder=builder)
972 magic_tag = Tag(soup, builder, 'magictag')
973 magic_tag.insert(0, "the")
974 soup.a.insert(1, magic_tag)
975
976 self.assertEqual(
977 soup.decode(), self.document_for(
978 "<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>"))
979
980 # Make sure all the relationships are hooked up correctly.
981 b_tag = soup.b
982 self.assertEqual(b_tag.next_sibling, magic_tag)
983 self.assertEqual(magic_tag.previous_sibling, b_tag)
984
985 find = b_tag.find(text="Find")
986 self.assertEqual(find.next_element, magic_tag)
987 self.assertEqual(magic_tag.previous_element, find)
988
989 c_tag = soup.c
990 self.assertEqual(magic_tag.next_sibling, c_tag)
991 self.assertEqual(c_tag.previous_sibling, magic_tag)
992
993 the = magic_tag.find(text="the")
994 self.assertEqual(the.parent, magic_tag)
995 self.assertEqual(the.next_element, c_tag)
996 self.assertEqual(c_tag.previous_element, the)
997
998 def test_append_child_thats_already_at_the_end(self):
999 data = "<a><b></b></a>"
1000 soup = self.soup(data)
1001 soup.a.append(soup.b)
1002 self.assertEqual(data, soup.decode())
1003
1004 def test_extend(self):
1005 data = "<a><b><c><d><e><f><g></g></f></e></d></c></b></a>"
1006 soup = self.soup(data)
1007 l = [soup.g, soup.f, soup.e, soup.d, soup.c, soup.b]
1008 soup.a.extend(l)
1009 self.assertEqual("<a><g></g><f></f><e></e><d></d><c></c><b></b></a>", soup.decode())
1010
1011 def test_move_tag_to_beginning_of_parent(self):
1012 data = "<a><b></b><c></c><d></d></a>"
1013 soup = self.soup(data)
1014 soup.a.insert(0, soup.d)
1015 self.assertEqual("<a><d></d><b></b><c></c></a>", soup.decode())
1016
1017 def test_insert_works_on_empty_element_tag(self):
1018 # This is a little strange, since most HTML parsers don't allow
1019 # markup like this to come through. But in general, we don't
1020 # know what the parser would or wouldn't have allowed, so
1021 # I'm letting this succeed for now.
1022 soup = self.soup("<br/>")
1023 soup.br.insert(1, "Contents")
1024 self.assertEqual(str(soup.br), "<br>Contents</br>")
1025
1026 def test_insert_before(self):
1027 soup = self.soup("<a>foo</a><b>bar</b>")
1028 soup.b.insert_before("BAZ")
1029 soup.a.insert_before("QUUX")
1030 self.assertEqual(
1031 soup.decode(), self.document_for("QUUX<a>foo</a>BAZ<b>bar</b>"))
1032
1033 soup.a.insert_before(soup.b)
1034 self.assertEqual(
1035 soup.decode(), self.document_for("QUUX<b>bar</b><a>foo</a>BAZ"))
1036
1037 # Can't insert an element before itself.
1038 b = soup.b
1039 self.assertRaises(ValueError, b.insert_before, b)
1040
1041 # Can't insert before if an element has no parent.
1042 b.extract()
1043 self.assertRaises(ValueError, b.insert_before, "nope")
1044
1045 # Can insert an identical element
1046 soup = self.soup("<a>")
1047 soup.a.insert_before(soup.new_tag("a"))
1048
1049 def test_insert_multiple_before(self):
1050 soup = self.soup("<a>foo</a><b>bar</b>")
1051 soup.b.insert_before("BAZ", " ", "QUUX")
1052 soup.a.insert_before("QUUX", " ", "BAZ")
1053 self.assertEqual(
1054 soup.decode(), self.document_for("QUUX BAZ<a>foo</a>BAZ QUUX<b>bar</b>"))
1055
1056 soup.a.insert_before(soup.b, "FOO")
1057 self.assertEqual(
1058 soup.decode(), self.document_for("QUUX BAZ<b>bar</b>FOO<a>foo</a>BAZ QUUX"))
1059
1060 def test_insert_after(self):
1061 soup = self.soup("<a>foo</a><b>bar</b>")
1062 soup.b.insert_after("BAZ")
1063 soup.a.insert_after("QUUX")
1064 self.assertEqual(
1065 soup.decode(), self.document_for("<a>foo</a>QUUX<b>bar</b>BAZ"))
1066 soup.b.insert_after(soup.a)
1067 self.assertEqual(
1068 soup.decode(), self.document_for("QUUX<b>bar</b><a>foo</a>BAZ"))
1069
1070 # Can't insert an element after itself.
1071 b = soup.b
1072 self.assertRaises(ValueError, b.insert_after, b)
1073
1074 # Can't insert after if an element has no parent.
1075 b.extract()
1076 self.assertRaises(ValueError, b.insert_after, "nope")
1077
1078 # Can insert an identical element
1079 soup = self.soup("<a>")
1080 soup.a.insert_before(soup.new_tag("a"))
1081
1082 def test_insert_multiple_after(self):
1083 soup = self.soup("<a>foo</a><b>bar</b>")
1084 soup.b.insert_after("BAZ", " ", "QUUX")
1085 soup.a.insert_after("QUUX", " ", "BAZ")
1086 self.assertEqual(
1087 soup.decode(), self.document_for("<a>foo</a>QUUX BAZ<b>bar</b>BAZ QUUX"))
1088 soup.b.insert_after(soup.a, "FOO ")
1089 self.assertEqual(
1090 soup.decode(), self.document_for("QUUX BAZ<b>bar</b><a>foo</a>FOO BAZ QUUX"))
1091
1092 def test_insert_after_raises_exception_if_after_has_no_meaning(self):
1093 soup = self.soup("")
1094 tag = soup.new_tag("a")
1095 string = soup.new_string("")
1096 self.assertRaises(ValueError, string.insert_after, tag)
1097 self.assertRaises(NotImplementedError, soup.insert_after, tag)
1098 self.assertRaises(ValueError, tag.insert_after, tag)
1099
1100 def test_insert_before_raises_notimplementederror_if_before_has_no_meaning(self):
1101 soup = self.soup("")
1102 tag = soup.new_tag("a")
1103 string = soup.new_string("")
1104 self.assertRaises(ValueError, string.insert_before, tag)
1105 self.assertRaises(NotImplementedError, soup.insert_before, tag)
1106 self.assertRaises(ValueError, tag.insert_before, tag)
1107
1108 def test_replace_with(self):
1109 soup = self.soup(
1110 "<p>There's <b>no</b> business like <b>show</b> business</p>")
1111 no, show = soup.find_all('b')
1112 show.replace_with(no)
1113 self.assertEqual(
1114 soup.decode(),
1115 self.document_for(
1116 "<p>There's business like <b>no</b> business</p>"))
1117
1118 self.assertEqual(show.parent, None)
1119 self.assertEqual(no.parent, soup.p)
1120 self.assertEqual(no.next_element, "no")
1121 self.assertEqual(no.next_sibling, " business")
1122
1123 def test_replace_first_child(self):
1124 data = "<a><b></b><c></c></a>"
1125 soup = self.soup(data)
1126 soup.b.replace_with(soup.c)
1127 self.assertEqual("<a><c></c></a>", soup.decode())
1128
1129 def test_replace_last_child(self):
1130 data = "<a><b></b><c></c></a>"
1131 soup = self.soup(data)
1132 soup.c.replace_with(soup.b)
1133 self.assertEqual("<a><b></b></a>", soup.decode())
1134
1135 def test_nested_tag_replace_with(self):
1136 soup = self.soup(
1137 """<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>""")
1138
1139 # Replace the entire <b> tag and its contents ("reserve the
1140 # right") with the <f> tag ("refuse").
1141 remove_tag = soup.b
1142 move_tag = soup.f
1143 remove_tag.replace_with(move_tag)
1144
1145 self.assertEqual(
1146 soup.decode(), self.document_for(
1147 "<a>We<f>refuse</f></a><e>to<g>service</g></e>"))
1148
1149 # The <b> tag is now an orphan.
1150 self.assertEqual(remove_tag.parent, None)
1151 self.assertEqual(remove_tag.find(text="right").next_element, None)
1152 self.assertEqual(remove_tag.previous_element, None)
1153 self.assertEqual(remove_tag.next_sibling, None)
1154 self.assertEqual(remove_tag.previous_sibling, None)
1155
1156 # The <f> tag is now connected to the <a> tag.
1157 self.assertEqual(move_tag.parent, soup.a)
1158 self.assertEqual(move_tag.previous_element, "We")
1159 self.assertEqual(move_tag.next_element.next_element, soup.e)
1160 self.assertEqual(move_tag.next_sibling, None)
1161
1162 # The gap where the <f> tag used to be has been mended, and
1163 # the word "to" is now connected to the <g> tag.
1164 to_text = soup.find(text="to")
1165 g_tag = soup.g
1166 self.assertEqual(to_text.next_element, g_tag)
1167 self.assertEqual(to_text.next_sibling, g_tag)
1168 self.assertEqual(g_tag.previous_element, to_text)
1169 self.assertEqual(g_tag.previous_sibling, to_text)
1170
1171 def test_unwrap(self):
1172 tree = self.soup("""
1173 <p>Unneeded <em>formatting</em> is unneeded</p>
1174 """)
1175 tree.em.unwrap()
1176 self.assertEqual(tree.em, None)
1177 self.assertEqual(tree.p.text, "Unneeded formatting is unneeded")
1178
1179 def test_wrap(self):
1180 soup = self.soup("I wish I was bold.")
1181 value = soup.string.wrap(soup.new_tag("b"))
1182 self.assertEqual(value.decode(), "<b>I wish I was bold.</b>")
1183 self.assertEqual(
1184 soup.decode(), self.document_for("<b>I wish I was bold.</b>"))
1185
1186 def test_wrap_extracts_tag_from_elsewhere(self):
1187 soup = self.soup("<b></b>I wish I was bold.")
1188 soup.b.next_sibling.wrap(soup.b)
1189 self.assertEqual(
1190 soup.decode(), self.document_for("<b>I wish I was bold.</b>"))
1191
1192 def test_wrap_puts_new_contents_at_the_end(self):
1193 soup = self.soup("<b>I like being bold.</b>I wish I was bold.")
1194 soup.b.next_sibling.wrap(soup.b)
1195 self.assertEqual(2, len(soup.b.contents))
1196 self.assertEqual(
1197 soup.decode(), self.document_for(
1198 "<b>I like being bold.I wish I was bold.</b>"))
1199
1200 def test_extract(self):
1201 soup = self.soup(
1202 '<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>')
1203
1204 self.assertEqual(len(soup.body.contents), 3)
1205 extracted = soup.find(id="nav").extract()
1206
1207 self.assertEqual(
1208 soup.decode(), "<html><body>Some content. More content.</body></html>")
1209 self.assertEqual(extracted.decode(), '<div id="nav">Nav crap</div>')
1210
1211 # The extracted tag is now an orphan.
1212 self.assertEqual(len(soup.body.contents), 2)
1213 self.assertEqual(extracted.parent, None)
1214 self.assertEqual(extracted.previous_element, None)
1215 self.assertEqual(extracted.next_element.next_element, None)
1216
1217 # The gap where the extracted tag used to be has been mended.
1218 content_1 = soup.find(text="Some content. ")
1219 content_2 = soup.find(text=" More content.")
1220 self.assertEqual(content_1.next_element, content_2)
1221 self.assertEqual(content_1.next_sibling, content_2)
1222 self.assertEqual(content_2.previous_element, content_1)
1223 self.assertEqual(content_2.previous_sibling, content_1)
1224
1225 def test_extract_distinguishes_between_identical_strings(self):
1226 soup = self.soup("<a>foo</a><b>bar</b>")
1227 foo_1 = soup.a.string
1228 bar_1 = soup.b.string
1229 foo_2 = soup.new_string("foo")
1230 bar_2 = soup.new_string("bar")
1231 soup.a.append(foo_2)
1232 soup.b.append(bar_2)
1233
1234 # Now there are two identical strings in the <a> tag, and two
1235 # in the <b> tag. Let's remove the first "foo" and the second
1236 # "bar".
1237 foo_1.extract()
1238 bar_2.extract()
1239 self.assertEqual(foo_2, soup.a.string)
1240 self.assertEqual(bar_2, soup.b.string)
1241
1242 def test_extract_multiples_of_same_tag(self):
1243 soup = self.soup("""
1244 <html>
1245 <head>
1246 <script>foo</script>
1247 </head>
1248 <body>
1249 <script>bar</script>
1250 <a></a>
1251 </body>
1252 <script>baz</script>
1253 </html>""")
1254 [soup.script.extract() for i in soup.find_all("script")]
1255 self.assertEqual("<body>\n\n<a></a>\n</body>", str(soup.body))
1256
1257
1258 def test_extract_works_when_element_is_surrounded_by_identical_strings(self):
1259 soup = self.soup(
1260 '<html>\n'
1261 '<body>hi</body>\n'
1262 '</html>')
1263 soup.find('body').extract()
1264 self.assertEqual(None, soup.find('body'))
1265
1266
1267 def test_clear(self):
1268 """Tag.clear()"""
1269 soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>")
1270 # clear using extract()
1271 a = soup.a
1272 soup.p.clear()
1273 self.assertEqual(len(soup.p.contents), 0)
1274 self.assertTrue(hasattr(a, "contents"))
1275
1276 # clear using decompose()
1277 em = a.em
1278 a.clear(decompose=True)
1279 self.assertEqual(0, len(em.contents))
1280
1281
1282 def test_decompose(self):
1283 # Test PageElement.decompose() and PageElement.decomposed
1284 soup = self.soup("<p><a>String <em>Italicized</em></a></p><p>Another para</p>")
1285 p1, p2 = soup.find_all('p')
1286 a = p1.a
1287 text = p1.em.string
1288 for i in [p1, p2, a, text]:
1289 self.assertEqual(False, i.decomposed)
1290
1291 # This sets p1 and everything beneath it to decomposed.
1292 p1.decompose()
1293 for i in [p1, a, text]:
1294 self.assertEqual(True, i.decomposed)
1295 # p2 is unaffected.
1296 self.assertEqual(False, p2.decomposed)
1297
1298 def test_string_set(self):
1299 """Tag.string = 'string'"""
1300 soup = self.soup("<a></a> <b><c></c></b>")
1301 soup.a.string = "foo"
1302 self.assertEqual(soup.a.contents, ["foo"])
1303 soup.b.string = "bar"
1304 self.assertEqual(soup.b.contents, ["bar"])
1305
1306 def test_string_set_does_not_affect_original_string(self):
1307 soup = self.soup("<a><b>foo</b><c>bar</c>")
1308 soup.b.string = soup.c.string
1309 self.assertEqual(soup.a.encode(), b"<a><b>bar</b><c>bar</c></a>")
1310
1311 def test_set_string_preserves_class_of_string(self):
1312 soup = self.soup("<a></a>")
1313 cdata = CData("foo")
1314 soup.a.string = cdata
1315 self.assertTrue(isinstance(soup.a.string, CData))
1316
1317 class TestElementObjects(SoupTest):
1318 """Test various features of element objects."""
1319
1320 def test_len(self):
1321 """The length of an element is its number of children."""
1322 soup = self.soup("<top>1<b>2</b>3</top>")
1323
1324 # The BeautifulSoup object itself contains one element: the
1325 # <top> tag.
1326 self.assertEqual(len(soup.contents), 1)
1327 self.assertEqual(len(soup), 1)
1328
1329 # The <top> tag contains three elements: the text node "1", the
1330 # <b> tag, and the text node "3".
1331 self.assertEqual(len(soup.top), 3)
1332 self.assertEqual(len(soup.top.contents), 3)
1333
1334 def test_member_access_invokes_find(self):
1335 """Accessing a Python member .foo invokes find('foo')"""
1336 soup = self.soup('<b><i></i></b>')
1337 self.assertEqual(soup.b, soup.find('b'))
1338 self.assertEqual(soup.b.i, soup.find('b').find('i'))
1339 self.assertEqual(soup.a, None)
1340
1341 def test_deprecated_member_access(self):
1342 soup = self.soup('<b><i></i></b>')
1343 with warnings.catch_warnings(record=True) as w:
1344 tag = soup.bTag
1345 self.assertEqual(soup.b, tag)
1346 self.assertEqual(
1347 '.bTag is deprecated, use .find("b") instead. If you really were looking for a tag called bTag, use .find("bTag")',
1348 str(w[0].message))
1349
1350 def test_has_attr(self):
1351 """has_attr() checks for the presence of an attribute.
1352
1353 Please note note: has_attr() is different from
1354 __in__. has_attr() checks the tag's attributes and __in__
1355 checks the tag's chidlren.
1356 """
1357 soup = self.soup("<foo attr='bar'>")
1358 self.assertTrue(soup.foo.has_attr('attr'))
1359 self.assertFalse(soup.foo.has_attr('attr2'))
1360
1361
1362 def test_attributes_come_out_in_alphabetical_order(self):
1363 markup = '<b a="1" z="5" m="3" f="2" y="4"></b>'
1364 self.assertSoupEquals(markup, '<b a="1" f="2" m="3" y="4" z="5"></b>')
1365
1366 def test_string(self):
1367 # A tag that contains only a text node makes that node
1368 # available as .string.
1369 soup = self.soup("<b>foo</b>")
1370 self.assertEqual(soup.b.string, 'foo')
1371
1372 def test_empty_tag_has_no_string(self):
1373 # A tag with no children has no .stirng.
1374 soup = self.soup("<b></b>")
1375 self.assertEqual(soup.b.string, None)
1376
1377 def test_tag_with_multiple_children_has_no_string(self):
1378 # A tag with no children has no .string.
1379 soup = self.soup("<a>foo<b></b><b></b></b>")
1380 self.assertEqual(soup.b.string, None)
1381
1382 soup = self.soup("<a>foo<b></b>bar</b>")
1383 self.assertEqual(soup.b.string, None)
1384
1385 # Even if all the children are strings, due to trickery,
1386 # it won't work--but this would be a good optimization.
1387 soup = self.soup("<a>foo</b>")
1388 soup.a.insert(1, "bar")
1389 self.assertEqual(soup.a.string, None)
1390
1391 def test_tag_with_recursive_string_has_string(self):
1392 # A tag with a single child which has a .string inherits that
1393 # .string.
1394 soup = self.soup("<a><b>foo</b></a>")
1395 self.assertEqual(soup.a.string, "foo")
1396 self.assertEqual(soup.string, "foo")
1397
1398 def test_lack_of_string(self):
1399 """Only a tag containing a single text node has a .string."""
1400 soup = self.soup("<b>f<i>e</i>o</b>")
1401 self.assertFalse(soup.b.string)
1402
1403 soup = self.soup("<b></b>")
1404 self.assertFalse(soup.b.string)
1405
1406 def test_all_text(self):
1407 """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated"""
1408 soup = self.soup("<a>a<b>r</b> <r> t </r></a>")
1409 self.assertEqual(soup.a.text, "ar t ")
1410 self.assertEqual(soup.a.get_text(strip=True), "art")
1411 self.assertEqual(soup.a.get_text(","), "a,r, , t ")
1412 self.assertEqual(soup.a.get_text(",", strip=True), "a,r,t")
1413
1414 def test_get_text_ignores_special_string_containers(self):
1415 soup = self.soup("foo<!--IGNORE-->bar")
1416 self.assertEqual(soup.get_text(), "foobar")
1417
1418 self.assertEqual(
1419 soup.get_text(types=(NavigableString, Comment)), "fooIGNOREbar")
1420 self.assertEqual(
1421 soup.get_text(types=None), "fooIGNOREbar")
1422
1423 soup = self.soup("foo<style>CSS</style><script>Javascript</script>bar")
1424 self.assertEqual(soup.get_text(), "foobar")
1425
1426 def test_all_strings_ignores_special_string_containers(self):
1427 soup = self.soup("foo<!--IGNORE-->bar")
1428 self.assertEqual(['foo', 'bar'], list(soup.strings))
1429
1430 soup = self.soup("foo<style>CSS</style><script>Javascript</script>bar")
1431 self.assertEqual(['foo', 'bar'], list(soup.strings))
1432
1433
1434 class TestCDAtaListAttributes(SoupTest):
1435
1436 """Testing cdata-list attributes like 'class'.
1437 """
1438 def test_single_value_becomes_list(self):
1439 soup = self.soup("<a class='foo'>")
1440 self.assertEqual(["foo"],soup.a['class'])
1441
1442 def test_multiple_values_becomes_list(self):
1443 soup = self.soup("<a class='foo bar'>")
1444 self.assertEqual(["foo", "bar"], soup.a['class'])
1445
1446 def test_multiple_values_separated_by_weird_whitespace(self):
1447 soup = self.soup("<a class='foo\tbar\nbaz'>")
1448 self.assertEqual(["foo", "bar", "baz"],soup.a['class'])
1449
1450 def test_attributes_joined_into_string_on_output(self):
1451 soup = self.soup("<a class='foo\tbar'>")
1452 self.assertEqual(b'<a class="foo bar"></a>', soup.a.encode())
1453
1454 def test_get_attribute_list(self):
1455 soup = self.soup("<a id='abc def'>")
1456 self.assertEqual(['abc def'], soup.a.get_attribute_list('id'))
1457
1458 def test_accept_charset(self):
1459 soup = self.soup('<form accept-charset="ISO-8859-1 UTF-8">')
1460 self.assertEqual(['ISO-8859-1', 'UTF-8'], soup.form['accept-charset'])
1461
1462 def test_cdata_attribute_applying_only_to_one_tag(self):
1463 data = '<a accept-charset="ISO-8859-1 UTF-8"></a>'
1464 soup = self.soup(data)
1465 # We saw in another test that accept-charset is a cdata-list
1466 # attribute for the <form> tag. But it's not a cdata-list
1467 # attribute for any other tag.
1468 self.assertEqual('ISO-8859-1 UTF-8', soup.a['accept-charset'])
1469
1470 def test_string_has_immutable_name_property(self):
1471 string = self.soup("s").string
1472 self.assertEqual(None, string.name)
1473 def t():
1474 string.name = 'foo'
1475 self.assertRaises(AttributeError, t)
1476
1477 class TestPersistence(SoupTest):
1478 "Testing features like pickle and deepcopy."
1479
1480 def setUp(self):
1481 super(TestPersistence, self).setUp()
1482 self.page = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"
1483 "http://www.w3.org/TR/REC-html40/transitional.dtd">
1484 <html>
1485 <head>
1486 <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
1487 <title>Beautiful Soup: We called him Tortoise because he taught us.</title>
1488 <link rev="made" href="mailto:leonardr@segfault.org">
1489 <meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping.">
1490 <meta name="generator" content="Markov Approximation 1.4 (module: leonardr)">
1491 <meta name="author" content="Leonard Richardson">
1492 </head>
1493 <body>
1494 <a href="foo">foo</a>
1495 <a href="foo"><b>bar</b></a>
1496 </body>
1497 </html>"""
1498 self.tree = self.soup(self.page)
1499
1500 def test_pickle_and_unpickle_identity(self):
1501 # Pickling a tree, then unpickling it, yields a tree identical
1502 # to the original.
1503 dumped = pickle.dumps(self.tree, 2)
1504 loaded = pickle.loads(dumped)
1505 self.assertEqual(loaded.__class__, BeautifulSoup)
1506 self.assertEqual(loaded.decode(), self.tree.decode())
1507
1508 def test_deepcopy_identity(self):
1509 # Making a deepcopy of a tree yields an identical tree.
1510 copied = copy.deepcopy(self.tree)
1511 self.assertEqual(copied.decode(), self.tree.decode())
1512
1513 def test_copy_preserves_encoding(self):
1514 soup = BeautifulSoup(b'<p>&nbsp;</p>', 'html.parser')
1515 encoding = soup.original_encoding
1516 copy = soup.__copy__()
1517 self.assertEqual("<p> </p>", str(copy))
1518 self.assertEqual(encoding, copy.original_encoding)
1519
1520 def test_copy_preserves_builder_information(self):
1521
1522 tag = self.soup('<p></p>').p
1523
1524 # Simulate a tag obtained from a source file.
1525 tag.sourceline = 10
1526 tag.sourcepos = 33
1527
1528 copied = tag.__copy__()
1529
1530 # The TreeBuilder object is no longer availble, but information
1531 # obtained from it gets copied over to the new Tag object.
1532 self.assertEqual(tag.sourceline, copied.sourceline)
1533 self.assertEqual(tag.sourcepos, copied.sourcepos)
1534 self.assertEqual(
1535 tag.can_be_empty_element, copied.can_be_empty_element
1536 )
1537 self.assertEqual(
1538 tag.cdata_list_attributes, copied.cdata_list_attributes
1539 )
1540 self.assertEqual(
1541 tag.preserve_whitespace_tags, copied.preserve_whitespace_tags
1542 )
1543
1544
1545 def test_unicode_pickle(self):
1546 # A tree containing Unicode characters can be pickled.
1547 html = "<b>\N{SNOWMAN}</b>"
1548 soup = self.soup(html)
1549 dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL)
1550 loaded = pickle.loads(dumped)
1551 self.assertEqual(loaded.decode(), soup.decode())
1552
1553 def test_copy_navigablestring_is_not_attached_to_tree(self):
1554 html = "<b>Foo<a></a></b><b>Bar</b>"
1555 soup = self.soup(html)
1556 s1 = soup.find(string="Foo")
1557 s2 = copy.copy(s1)
1558 self.assertEqual(s1, s2)
1559 self.assertEqual(None, s2.parent)
1560 self.assertEqual(None, s2.next_element)
1561 self.assertNotEqual(None, s1.next_sibling)
1562 self.assertEqual(None, s2.next_sibling)
1563 self.assertEqual(None, s2.previous_element)
1564
1565 def test_copy_navigablestring_subclass_has_same_type(self):
1566 html = "<b><!--Foo--></b>"
1567 soup = self.soup(html)
1568 s1 = soup.string
1569 s2 = copy.copy(s1)
1570 self.assertEqual(s1, s2)
1571 self.assertTrue(isinstance(s2, Comment))
1572
1573 def test_copy_entire_soup(self):
1574 html = "<div><b>Foo<a></a></b><b>Bar</b></div>end"
1575 soup = self.soup(html)
1576 soup_copy = copy.copy(soup)
1577 self.assertEqual(soup, soup_copy)
1578
1579 def test_copy_tag_copies_contents(self):
1580 html = "<div><b>Foo<a></a></b><b>Bar</b></div>end"
1581 soup = self.soup(html)
1582 div = soup.div
1583 div_copy = copy.copy(div)
1584
1585 # The two tags look the same, and evaluate to equal.
1586 self.assertEqual(str(div), str(div_copy))
1587 self.assertEqual(div, div_copy)
1588
1589 # But they're not the same object.
1590 self.assertFalse(div is div_copy)
1591
1592 # And they don't have the same relation to the parse tree. The
1593 # copy is not associated with a parse tree at all.
1594 self.assertEqual(None, div_copy.parent)
1595 self.assertEqual(None, div_copy.previous_element)
1596 self.assertEqual(None, div_copy.find(string='Bar').next_element)
1597 self.assertNotEqual(None, div.find(string='Bar').next_element)
1598
1599 class TestSubstitutions(SoupTest):
1600
1601 def test_default_formatter_is_minimal(self):
1602 markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
1603 soup = self.soup(markup)
1604 decoded = soup.decode(formatter="minimal")
1605 # The < is converted back into &lt; but the e-with-acute is left alone.
1606 self.assertEqual(
1607 decoded,
1608 self.document_for(
1609 "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"))
1610
1611 def test_formatter_html(self):
1612 markup = "<br><b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
1613 soup = self.soup(markup)
1614 decoded = soup.decode(formatter="html")
1615 self.assertEqual(
1616 decoded,
1617 self.document_for("<br/><b>&lt;&lt;Sacr&eacute; bleu!&gt;&gt;</b>"))
1618
1619 def test_formatter_html5(self):
1620 markup = "<br><b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
1621 soup = self.soup(markup)
1622 decoded = soup.decode(formatter="html5")
1623 self.assertEqual(
1624 decoded,
1625 self.document_for("<br><b>&lt;&lt;Sacr&eacute; bleu!&gt;&gt;</b>"))
1626
1627 def test_formatter_minimal(self):
1628 markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
1629 soup = self.soup(markup)
1630 decoded = soup.decode(formatter="minimal")
1631 # The < is converted back into &lt; but the e-with-acute is left alone.
1632 self.assertEqual(
1633 decoded,
1634 self.document_for(
1635 "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"))
1636
1637 def test_formatter_null(self):
1638 markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
1639 soup = self.soup(markup)
1640 decoded = soup.decode(formatter=None)
1641 # Neither the angle brackets nor the e-with-acute are converted.
1642 # This is not valid HTML, but it's what the user wanted.
1643 self.assertEqual(decoded,
1644 self.document_for("<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>"))
1645
1646 def test_formatter_custom(self):
1647 markup = "<b>&lt;foo&gt;</b><b>bar</b><br/>"
1648 soup = self.soup(markup)
1649 decoded = soup.decode(formatter = lambda x: x.upper())
1650 # Instead of normal entity conversion code, the custom
1651 # callable is called on every string.
1652 self.assertEqual(
1653 decoded,
1654 self.document_for("<b><FOO></b><b>BAR</b><br/>"))
1655
1656 def test_formatter_is_run_on_attribute_values(self):
1657 markup = '<a href="http://a.com?a=b&c=é">e</a>'
1658 soup = self.soup(markup)
1659 a = soup.a
1660
1661 expect_minimal = '<a href="http://a.com?a=b&amp;c=é">e</a>'
1662
1663 self.assertEqual(expect_minimal, a.decode())
1664 self.assertEqual(expect_minimal, a.decode(formatter="minimal"))
1665
1666 expect_html = '<a href="http://a.com?a=b&amp;c=&eacute;">e</a>'
1667 self.assertEqual(expect_html, a.decode(formatter="html"))
1668
1669 self.assertEqual(markup, a.decode(formatter=None))
1670 expect_upper = '<a href="HTTP://A.COM?A=B&C=É">E</a>'
1671 self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper()))
1672
1673 def test_formatter_skips_script_tag_for_html_documents(self):
1674 doc = """
1675 <script type="text/javascript">
1676 console.log("< < hey > > ");
1677 </script>
1678 """
1679 encoded = BeautifulSoup(doc, 'html.parser').encode()
1680 self.assertTrue(b"< < hey > >" in encoded)
1681
1682 def test_formatter_skips_style_tag_for_html_documents(self):
1683 doc = """
1684 <style type="text/css">
1685 console.log("< < hey > > ");
1686 </style>
1687 """
1688 encoded = BeautifulSoup(doc, 'html.parser').encode()
1689 self.assertTrue(b"< < hey > >" in encoded)
1690
1691 def test_prettify_leaves_preformatted_text_alone(self):
1692 soup = self.soup("<div> foo <pre> \tbar\n \n </pre> baz <textarea> eee\nfff\t</textarea></div>")
1693 # Everything outside the <pre> tag is reformatted, but everything
1694 # inside is left alone.
1695 self.assertEqual(
1696 '<div>\n foo\n <pre> \tbar\n \n </pre>\n baz\n <textarea> eee\nfff\t</textarea>\n</div>',
1697 soup.div.prettify())
1698
1699 def test_prettify_accepts_formatter_function(self):
1700 soup = BeautifulSoup("<html><body>foo</body></html>", 'html.parser')
1701 pretty = soup.prettify(formatter = lambda x: x.upper())
1702 self.assertTrue("FOO" in pretty)
1703
1704 def test_prettify_outputs_unicode_by_default(self):
1705 soup = self.soup("<a></a>")
1706 self.assertEqual(str, type(soup.prettify()))
1707
1708 def test_prettify_can_encode_data(self):
1709 soup = self.soup("<a></a>")
1710 self.assertEqual(bytes, type(soup.prettify("utf-8")))
1711
1712 def test_html_entity_substitution_off_by_default(self):
1713 markup = "<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>"
1714 soup = self.soup(markup)
1715 encoded = soup.b.encode("utf-8")
1716 self.assertEqual(encoded, markup.encode('utf-8'))
1717
1718 def test_encoding_substitution(self):
1719 # Here's the <meta> tag saying that a document is
1720 # encoded in Shift-JIS.
1721 meta_tag = ('<meta content="text/html; charset=x-sjis" '
1722 'http-equiv="Content-type"/>')
1723 soup = self.soup(meta_tag)
1724
1725 # Parse the document, and the charset apprears unchanged.
1726 self.assertEqual(soup.meta['content'], 'text/html; charset=x-sjis')
1727
1728 # Encode the document into some encoding, and the encoding is
1729 # substituted into the meta tag.
1730 utf_8 = soup.encode("utf-8")
1731 self.assertTrue(b"charset=utf-8" in utf_8)
1732
1733 euc_jp = soup.encode("euc_jp")
1734 self.assertTrue(b"charset=euc_jp" in euc_jp)
1735
1736 shift_jis = soup.encode("shift-jis")
1737 self.assertTrue(b"charset=shift-jis" in shift_jis)
1738
1739 utf_16_u = soup.encode("utf-16").decode("utf-16")
1740 self.assertTrue("charset=utf-16" in utf_16_u)
1741
1742 def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self):
1743 markup = ('<head><meta content="text/html; charset=x-sjis" '
1744 'http-equiv="Content-type"/></head><pre>foo</pre>')
1745
1746 # Beautiful Soup used to try to rewrite the meta tag even if the
1747 # meta tag got filtered out by the strainer. This test makes
1748 # sure that doesn't happen.
1749 strainer = SoupStrainer('pre')
1750 soup = self.soup(markup, parse_only=strainer)
1751 self.assertEqual(soup.contents[0].name, 'pre')
1752
1753 class TestEncoding(SoupTest):
1754 """Test the ability to encode objects into strings."""
1755
1756 def test_unicode_string_can_be_encoded(self):
1757 html = "<b>\N{SNOWMAN}</b>"
1758 soup = self.soup(html)
1759 self.assertEqual(soup.b.string.encode("utf-8"),
1760 "\N{SNOWMAN}".encode("utf-8"))
1761
1762 def test_tag_containing_unicode_string_can_be_encoded(self):
1763 html = "<b>\N{SNOWMAN}</b>"
1764 soup = self.soup(html)
1765 self.assertEqual(
1766 soup.b.encode("utf-8"), html.encode("utf-8"))
1767
1768 def test_encoding_substitutes_unrecognized_characters_by_default(self):
1769 html = "<b>\N{SNOWMAN}</b>"
1770 soup = self.soup(html)
1771 self.assertEqual(soup.b.encode("ascii"), b"<b>&#9731;</b>")
1772
1773 def test_encoding_can_be_made_strict(self):
1774 html = "<b>\N{SNOWMAN}</b>"
1775 soup = self.soup(html)
1776 self.assertRaises(
1777 UnicodeEncodeError, soup.encode, "ascii", errors="strict")
1778
1779 def test_decode_contents(self):
1780 html = "<b>\N{SNOWMAN}</b>"
1781 soup = self.soup(html)
1782 self.assertEqual("\N{SNOWMAN}", soup.b.decode_contents())
1783
1784 def test_encode_contents(self):
1785 html = "<b>\N{SNOWMAN}</b>"
1786 soup = self.soup(html)
1787 self.assertEqual(
1788 "\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents(
1789 encoding="utf8"))
1790
1791 def test_deprecated_renderContents(self):
1792 html = "<b>\N{SNOWMAN}</b>"
1793 soup = self.soup(html)
1794 self.assertEqual(
1795 "\N{SNOWMAN}".encode("utf8"), soup.b.renderContents())
1796
1797 def test_repr(self):
1798 html = "<b>\N{SNOWMAN}</b>"
1799 soup = self.soup(html)
1800 if PY3K:
1801 self.assertEqual(html, repr(soup))
1802 else:
1803 self.assertEqual(b'<b>\\u2603</b>', repr(soup))
1804
1805 class TestFormatter(SoupTest):
1806
1807 def test_default_attributes(self):
1808 # Test the default behavior of Formatter.attributes().
1809 formatter = Formatter()
1810 tag = Tag(name="tag")
1811 tag['b'] = 1
1812 tag['a'] = 2
1813
1814 # Attributes come out sorted by name. In Python 3, attributes
1815 # normally come out of a dictionary in the order they were
1816 # added.
1817 self.assertEqual([('a', 2), ('b', 1)], formatter.attributes(tag))
1818
1819 # This works even if Tag.attrs is None, though this shouldn't
1820 # normally happen.
1821 tag.attrs = None
1822 self.assertEqual([], formatter.attributes(tag))
1823
1824 def test_sort_attributes(self):
1825 # Test the ability to override Formatter.attributes() to,
1826 # e.g., disable the normal sorting of attributes.
1827 class UnsortedFormatter(Formatter):
1828 def attributes(self, tag):
1829 self.called_with = tag
1830 for k, v in sorted(tag.attrs.items()):
1831 if k == 'ignore':
1832 continue
1833 yield k,v
1834
1835 soup = self.soup('<p cval="1" aval="2" ignore="ignored"></p>')
1836 formatter = UnsortedFormatter()
1837 decoded = soup.decode(formatter=formatter)
1838
1839 # attributes() was called on the <p> tag. It filtered out one
1840 # attribute and sorted the other two.
1841 self.assertEqual(formatter.called_with, soup.p)
1842 self.assertEqual('<p aval="2" cval="1"></p>', decoded)
1843
1844
1845 class TestNavigableStringSubclasses(SoupTest):
1846
1847 def test_cdata(self):
1848 # None of the current builders turn CDATA sections into CData
1849 # objects, but you can create them manually.
1850 soup = self.soup("")
1851 cdata = CData("foo")
1852 soup.insert(1, cdata)
1853 self.assertEqual(str(soup), "<![CDATA[foo]]>")
1854 self.assertEqual(soup.find(text="foo"), "foo")
1855 self.assertEqual(soup.contents[0], "foo")
1856
1857 def test_cdata_is_never_formatted(self):
1858 """Text inside a CData object is passed into the formatter.
1859
1860 But the return value is ignored.
1861 """
1862
1863 self.count = 0
1864 def increment(*args):
1865 self.count += 1
1866 return "BITTER FAILURE"
1867
1868 soup = self.soup("")
1869 cdata = CData("<><><>")
1870 soup.insert(1, cdata)
1871 self.assertEqual(
1872 b"<![CDATA[<><><>]]>", soup.encode(formatter=increment))
1873 self.assertEqual(1, self.count)
1874
1875 def test_doctype_ends_in_newline(self):
1876 # Unlike other NavigableString subclasses, a DOCTYPE always ends
1877 # in a newline.
1878 doctype = Doctype("foo")
1879 soup = self.soup("")
1880 soup.insert(1, doctype)
1881 self.assertEqual(soup.encode(), b"<!DOCTYPE foo>\n")
1882
1883 def test_declaration(self):
1884 d = Declaration("foo")
1885 self.assertEqual("<?foo?>", d.output_ready())
1886
1887 def test_default_string_containers(self):
1888 # In some cases, we use different NavigableString subclasses for
1889 # the same text in different tags.
1890 soup = self.soup(
1891 "<div>text</div><script>text</script><style>text</style>"
1892 )
1893 self.assertEqual(
1894 [NavigableString, Script, Stylesheet],
1895 [x.__class__ for x in soup.find_all(text=True)]
1896 )
1897
1898 # The TemplateString is a little unusual because it's generally found
1899 # _inside_ children of a <template> element, not a direct child of the
1900 # <template> element.
1901 soup = self.soup(
1902 "<template>Some text<p>In a tag</p></template>Some text outside"
1903 )
1904 assert all(isinstance(x, TemplateString) for x in soup.template.strings)
1905
1906 # Once the <template> tag closed, we went back to using
1907 # NavigableString.
1908 outside = soup.template.next_sibling
1909 assert isinstance(outside, NavigableString)
1910 assert not isinstance(outside, TemplateString)
1911
1912 class TestSoupSelector(TreeTest):
1913
1914 HTML = """
1915 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
1916 "http://www.w3.org/TR/html4/strict.dtd">
1917 <html>
1918 <head>
1919 <title>The title</title>
1920 <link rel="stylesheet" href="blah.css" type="text/css" id="l1">
1921 </head>
1922 <body>
1923 <custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag>
1924 <div id="main" class="fancy">
1925 <div id="inner">
1926 <h1 id="header1">An H1</h1>
1927 <p>Some text</p>
1928 <p class="onep" id="p1">Some more text</p>
1929 <h2 id="header2">An H2</h2>
1930 <p class="class1 class2 class3" id="pmulti">Another</p>
1931 <a href="http://bob.example.org/" rel="friend met" id="bob">Bob</a>
1932 <h2 id="header3">Another H2</h2>
1933 <a id="me" href="http://simonwillison.net/" rel="me">me</a>
1934 <span class="s1">
1935 <a href="#" id="s1a1">span1a1</a>
1936 <a href="#" id="s1a2">span1a2 <span id="s1a2s1">test</span></a>
1937 <span class="span2">
1938 <a href="#" id="s2a1">span2a1</a>
1939 </span>
1940 <span class="span3"></span>
1941 <custom-dashed-tag class="dashed" id="dash2"/>
1942 <div data-tag="dashedvalue" id="data1"/>
1943 </span>
1944 </div>
1945 <x id="xid">
1946 <z id="zida"/>
1947 <z id="zidab"/>
1948 <z id="zidac"/>
1949 </x>
1950 <y id="yid">
1951 <z id="zidb"/>
1952 </y>
1953 <p lang="en" id="lang-en">English</p>
1954 <p lang="en-gb" id="lang-en-gb">English UK</p>
1955 <p lang="en-us" id="lang-en-us">English US</p>
1956 <p lang="fr" id="lang-fr">French</p>
1957 </div>
1958
1959 <div id="footer">
1960 </div>
1961 """
1962
1963 def setUp(self):
1964 self.soup = BeautifulSoup(self.HTML, 'html.parser')
1965
1966 def assertSelects(self, selector, expected_ids, **kwargs):
1967 el_ids = [el['id'] for el in self.soup.select(selector, **kwargs)]
1968 el_ids.sort()
1969 expected_ids.sort()
1970 self.assertEqual(expected_ids, el_ids,
1971 "Selector %s, expected [%s], got [%s]" % (
1972 selector, ', '.join(expected_ids), ', '.join(el_ids)
1973 )
1974 )
1975
1976 assertSelect = assertSelects
1977
1978 def assertSelectMultiple(self, *tests):
1979 for selector, expected_ids in tests:
1980 self.assertSelect(selector, expected_ids)
1981
1982 def test_one_tag_one(self):
1983 els = self.soup.select('title')
1984 self.assertEqual(len(els), 1)
1985 self.assertEqual(els[0].name, 'title')
1986 self.assertEqual(els[0].contents, ['The title'])
1987
1988 def test_one_tag_many(self):
1989 els = self.soup.select('div')
1990 self.assertEqual(len(els), 4)
1991 for div in els:
1992 self.assertEqual(div.name, 'div')
1993
1994 el = self.soup.select_one('div')
1995 self.assertEqual('main', el['id'])
1996
1997 def test_select_one_returns_none_if_no_match(self):
1998 match = self.soup.select_one('nonexistenttag')
1999 self.assertEqual(None, match)
2000
2001
2002 def test_tag_in_tag_one(self):
2003 els = self.soup.select('div div')
2004 self.assertSelects('div div', ['inner', 'data1'])
2005
2006 def test_tag_in_tag_many(self):
2007 for selector in ('html div', 'html body div', 'body div'):
2008 self.assertSelects(selector, ['data1', 'main', 'inner', 'footer'])
2009
2010
2011 def test_limit(self):
2012 self.assertSelects('html div', ['main'], limit=1)
2013 self.assertSelects('html body div', ['inner', 'main'], limit=2)
2014 self.assertSelects('body div', ['data1', 'main', 'inner', 'footer'],
2015 limit=10)
2016
2017 def test_tag_no_match(self):
2018 self.assertEqual(len(self.soup.select('del')), 0)
2019
2020 def test_invalid_tag(self):
2021 self.assertRaises(SyntaxError, self.soup.select, 'tag%t')
2022
2023 def test_select_dashed_tag_ids(self):
2024 self.assertSelects('custom-dashed-tag', ['dash1', 'dash2'])
2025
2026 def test_select_dashed_by_id(self):
2027 dashed = self.soup.select('custom-dashed-tag[id=\"dash2\"]')
2028 self.assertEqual(dashed[0].name, 'custom-dashed-tag')
2029 self.assertEqual(dashed[0]['id'], 'dash2')
2030
2031 def test_dashed_tag_text(self):
2032 self.assertEqual(self.soup.select('body > custom-dashed-tag')[0].text, 'Hello there.')
2033
2034 def test_select_dashed_matches_find_all(self):
2035 self.assertEqual(self.soup.select('custom-dashed-tag'), self.soup.find_all('custom-dashed-tag'))
2036
2037 def test_header_tags(self):
2038 self.assertSelectMultiple(
2039 ('h1', ['header1']),
2040 ('h2', ['header2', 'header3']),
2041 )
2042
2043 def test_class_one(self):
2044 for selector in ('.onep', 'p.onep', 'html p.onep'):
2045 els = self.soup.select(selector)
2046 self.assertEqual(len(els), 1)
2047 self.assertEqual(els[0].name, 'p')
2048 self.assertEqual(els[0]['class'], ['onep'])
2049
2050 def test_class_mismatched_tag(self):
2051 els = self.soup.select('div.onep')
2052 self.assertEqual(len(els), 0)
2053
2054 def test_one_id(self):
2055 for selector in ('div#inner', '#inner', 'div div#inner'):
2056 self.assertSelects(selector, ['inner'])
2057
2058 def test_bad_id(self):
2059 els = self.soup.select('#doesnotexist')
2060 self.assertEqual(len(els), 0)
2061
2062 def test_items_in_id(self):
2063 els = self.soup.select('div#inner p')
2064 self.assertEqual(len(els), 3)
2065 for el in els:
2066 self.assertEqual(el.name, 'p')
2067 self.assertEqual(els[1]['class'], ['onep'])
2068 self.assertFalse(els[0].has_attr('class'))
2069
2070 def test_a_bunch_of_emptys(self):
2071 for selector in ('div#main del', 'div#main div.oops', 'div div#main'):
2072 self.assertEqual(len(self.soup.select(selector)), 0)
2073
2074 def test_multi_class_support(self):
2075 for selector in ('.class1', 'p.class1', '.class2', 'p.class2',
2076 '.class3', 'p.class3', 'html p.class2', 'div#inner .class2'):
2077 self.assertSelects(selector, ['pmulti'])
2078
2079 def test_multi_class_selection(self):
2080 for selector in ('.class1.class3', '.class3.class2',
2081 '.class1.class2.class3'):
2082 self.assertSelects(selector, ['pmulti'])
2083
2084 def test_child_selector(self):
2085 self.assertSelects('.s1 > a', ['s1a1', 's1a2'])
2086 self.assertSelects('.s1 > a span', ['s1a2s1'])
2087
2088 def test_child_selector_id(self):
2089 self.assertSelects('.s1 > a#s1a2 span', ['s1a2s1'])
2090
2091 def test_attribute_equals(self):
2092 self.assertSelectMultiple(
2093 ('p[class="onep"]', ['p1']),
2094 ('p[id="p1"]', ['p1']),
2095 ('[class="onep"]', ['p1']),
2096 ('[id="p1"]', ['p1']),
2097 ('link[rel="stylesheet"]', ['l1']),
2098 ('link[type="text/css"]', ['l1']),
2099 ('link[href="blah.css"]', ['l1']),
2100 ('link[href="no-blah.css"]', []),
2101 ('[rel="stylesheet"]', ['l1']),
2102 ('[type="text/css"]', ['l1']),
2103 ('[href="blah.css"]', ['l1']),
2104 ('[href="no-blah.css"]', []),
2105 ('p[href="no-blah.css"]', []),
2106 ('[href="no-blah.css"]', []),
2107 )
2108
2109 def test_attribute_tilde(self):
2110 self.assertSelectMultiple(
2111 ('p[class~="class1"]', ['pmulti']),
2112 ('p[class~="class2"]', ['pmulti']),
2113 ('p[class~="class3"]', ['pmulti']),
2114 ('[class~="class1"]', ['pmulti']),
2115 ('[class~="class2"]', ['pmulti']),
2116 ('[class~="class3"]', ['pmulti']),
2117 ('a[rel~="friend"]', ['bob']),
2118 ('a[rel~="met"]', ['bob']),
2119 ('[rel~="friend"]', ['bob']),
2120 ('[rel~="met"]', ['bob']),
2121 )
2122
2123 def test_attribute_startswith(self):
2124 self.assertSelectMultiple(
2125 ('[rel^="style"]', ['l1']),
2126 ('link[rel^="style"]', ['l1']),
2127 ('notlink[rel^="notstyle"]', []),
2128 ('[rel^="notstyle"]', []),
2129 ('link[rel^="notstyle"]', []),
2130 ('link[href^="bla"]', ['l1']),
2131 ('a[href^="http://"]', ['bob', 'me']),
2132 ('[href^="http://"]', ['bob', 'me']),
2133 ('[id^="p"]', ['pmulti', 'p1']),
2134 ('[id^="m"]', ['me', 'main']),
2135 ('div[id^="m"]', ['main']),
2136 ('a[id^="m"]', ['me']),
2137 ('div[data-tag^="dashed"]', ['data1'])
2138 )
2139
2140 def test_attribute_endswith(self):
2141 self.assertSelectMultiple(
2142 ('[href$=".css"]', ['l1']),
2143 ('link[href$=".css"]', ['l1']),
2144 ('link[id$="1"]', ['l1']),
2145 ('[id$="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1', 'dash1']),
2146 ('div[id$="1"]', ['data1']),
2147 ('[id$="noending"]', []),
2148 )
2149
2150 def test_attribute_contains(self):
2151 self.assertSelectMultiple(
2152 # From test_attribute_startswith
2153 ('[rel*="style"]', ['l1']),
2154 ('link[rel*="style"]', ['l1']),
2155 ('notlink[rel*="notstyle"]', []),
2156 ('[rel*="notstyle"]', []),
2157 ('link[rel*="notstyle"]', []),
2158 ('link[href*="bla"]', ['l1']),
2159 ('[href*="http://"]', ['bob', 'me']),
2160 ('[id*="p"]', ['pmulti', 'p1']),
2161 ('div[id*="m"]', ['main']),
2162 ('a[id*="m"]', ['me']),
2163 # From test_attribute_endswith
2164 ('[href*=".css"]', ['l1']),
2165 ('link[href*=".css"]', ['l1']),
2166 ('link[id*="1"]', ['l1']),
2167 ('[id*="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1', 'dash1']),
2168 ('div[id*="1"]', ['data1']),
2169 ('[id*="noending"]', []),
2170 # New for this test
2171 ('[href*="."]', ['bob', 'me', 'l1']),
2172 ('a[href*="."]', ['bob', 'me']),
2173 ('link[href*="."]', ['l1']),
2174 ('div[id*="n"]', ['main', 'inner']),
2175 ('div[id*="nn"]', ['inner']),
2176 ('div[data-tag*="edval"]', ['data1'])
2177 )
2178
2179 def test_attribute_exact_or_hypen(self):
2180 self.assertSelectMultiple(
2181 ('p[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
2182 ('[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
2183 ('p[lang|="fr"]', ['lang-fr']),
2184 ('p[lang|="gb"]', []),
2185 )
2186
2187 def test_attribute_exists(self):
2188 self.assertSelectMultiple(
2189 ('[rel]', ['l1', 'bob', 'me']),
2190 ('link[rel]', ['l1']),
2191 ('a[rel]', ['bob', 'me']),
2192 ('[lang]', ['lang-en', 'lang-en-gb', 'lang-en-us', 'lang-fr']),
2193 ('p[class]', ['p1', 'pmulti']),
2194 ('[blah]', []),
2195 ('p[blah]', []),
2196 ('div[data-tag]', ['data1'])
2197 )
2198
2199 def test_quoted_space_in_selector_name(self):
2200 html = """<div style="display: wrong">nope</div>
2201 <div style="display: right">yes</div>
2202 """
2203 soup = BeautifulSoup(html, 'html.parser')
2204 [chosen] = soup.select('div[style="display: right"]')
2205 self.assertEqual("yes", chosen.string)
2206
2207 def test_unsupported_pseudoclass(self):
2208 self.assertRaises(
2209 NotImplementedError, self.soup.select, "a:no-such-pseudoclass")
2210
2211 self.assertRaises(
2212 SyntaxError, self.soup.select, "a:nth-of-type(a)")
2213
2214 def test_nth_of_type(self):
2215 # Try to select first paragraph
2216 els = self.soup.select('div#inner p:nth-of-type(1)')
2217 self.assertEqual(len(els), 1)
2218 self.assertEqual(els[0].string, 'Some text')
2219
2220 # Try to select third paragraph
2221 els = self.soup.select('div#inner p:nth-of-type(3)')
2222 self.assertEqual(len(els), 1)
2223 self.assertEqual(els[0].string, 'Another')
2224
2225 # Try to select (non-existent!) fourth paragraph
2226 els = self.soup.select('div#inner p:nth-of-type(4)')
2227 self.assertEqual(len(els), 0)
2228
2229 # Zero will select no tags.
2230 els = self.soup.select('div p:nth-of-type(0)')
2231 self.assertEqual(len(els), 0)
2232
2233 def test_nth_of_type_direct_descendant(self):
2234 els = self.soup.select('div#inner > p:nth-of-type(1)')
2235 self.assertEqual(len(els), 1)
2236 self.assertEqual(els[0].string, 'Some text')
2237
2238 def test_id_child_selector_nth_of_type(self):
2239 self.assertSelects('#inner > p:nth-of-type(2)', ['p1'])
2240
2241 def test_select_on_element(self):
2242 # Other tests operate on the tree; this operates on an element
2243 # within the tree.
2244 inner = self.soup.find("div", id="main")
2245 selected = inner.select("div")
2246 # The <div id="inner"> tag was selected. The <div id="footer">
2247 # tag was not.
2248 self.assertSelectsIDs(selected, ['inner', 'data1'])
2249
2250 def test_overspecified_child_id(self):
2251 self.assertSelects(".fancy #inner", ['inner'])
2252 self.assertSelects(".normal #inner", [])
2253
2254 def test_adjacent_sibling_selector(self):
2255 self.assertSelects('#p1 + h2', ['header2'])
2256 self.assertSelects('#p1 + h2 + p', ['pmulti'])
2257 self.assertSelects('#p1 + #header2 + .class1', ['pmulti'])
2258 self.assertEqual([], self.soup.select('#p1 + p'))
2259
2260 def test_general_sibling_selector(self):
2261 self.assertSelects('#p1 ~ h2', ['header2', 'header3'])
2262 self.assertSelects('#p1 ~ #header2', ['header2'])
2263 self.assertSelects('#p1 ~ h2 + a', ['me'])
2264 self.assertSelects('#p1 ~ h2 + [rel="me"]', ['me'])
2265 self.assertEqual([], self.soup.select('#inner ~ h2'))
2266
2267 def test_dangling_combinator(self):
2268 self.assertRaises(SyntaxError, self.soup.select, 'h1 >')
2269
2270 def test_sibling_combinator_wont_select_same_tag_twice(self):
2271 self.assertSelects('p[lang] ~ p', ['lang-en-gb', 'lang-en-us', 'lang-fr'])
2272
2273 # Test the selector grouping operator (the comma)
2274 def test_multiple_select(self):
2275 self.assertSelects('x, y', ['xid', 'yid'])
2276
2277 def test_multiple_select_with_no_space(self):
2278 self.assertSelects('x,y', ['xid', 'yid'])
2279
2280 def test_multiple_select_with_more_space(self):
2281 self.assertSelects('x, y', ['xid', 'yid'])
2282
2283 def test_multiple_select_duplicated(self):
2284 self.assertSelects('x, x', ['xid'])
2285
2286 def test_multiple_select_sibling(self):
2287 self.assertSelects('x, y ~ p[lang=fr]', ['xid', 'lang-fr'])
2288
2289 def test_multiple_select_tag_and_direct_descendant(self):
2290 self.assertSelects('x, y > z', ['xid', 'zidb'])
2291
2292 def test_multiple_select_direct_descendant_and_tags(self):
2293 self.assertSelects('div > x, y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac'])
2294
2295 def test_multiple_select_indirect_descendant(self):
2296 self.assertSelects('div x,y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac'])
2297
2298 def test_invalid_multiple_select(self):
2299 self.assertRaises(SyntaxError, self.soup.select, ',x, y')
2300 self.assertRaises(SyntaxError, self.soup.select, 'x,,y')
2301
2302 def test_multiple_select_attrs(self):
2303 self.assertSelects('p[lang=en], p[lang=en-gb]', ['lang-en', 'lang-en-gb'])
2304
2305 def test_multiple_select_ids(self):
2306 self.assertSelects('x, y > z[id=zida], z[id=zidab], z[id=zidb]', ['xid', 'zidb', 'zidab'])
2307
2308 def test_multiple_select_nested(self):
2309 self.assertSelects('body > div > x, y > z', ['xid', 'zidb'])
2310
2311 def test_select_duplicate_elements(self):
2312 # When markup contains duplicate elements, a multiple select
2313 # will find all of them.
2314 markup = '<div class="c1"/><div class="c2"/><div class="c1"/>'
2315 soup = BeautifulSoup(markup, 'html.parser')
2316 selected = soup.select(".c1, .c2")
2317 self.assertEqual(3, len(selected))
2318
2319 # Verify that find_all finds the same elements, though because
2320 # of an implementation detail it finds them in a different
2321 # order.
2322 for element in soup.find_all(class_=['c1', 'c2']):
2323 assert element in selected