././@PaxHeader0000000000000000000000000000003400000000000011452 xustar000000000000000028 mtime=1631060004.6335707 beautifulsoup4-4.10.0/0000775000175000017500000000000000000000000016210 5ustar00leonardrleonardr00000000000000././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1483369082.0 beautifulsoup4-4.10.0/COPYING.txt0000664000175000017500000000244300000000000020064 0ustar00leonardrleonardr00000000000000Beautiful Soup is made available under the MIT license: Copyright (c) 2004-2017 Leonard Richardson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Beautiful Soup incorporates code from the html5lib library, which is also made available under the MIT license. Copyright (c) 2006-2013 James Graham and other contributors ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1545665720.0 beautifulsoup4-4.10.0/LICENSE0000644000175000017500000000264700000000000017224 0ustar00leonardrleonardr00000000000000Beautiful Soup is made available under the MIT license: Copyright (c) 2004-2019 Leonard Richardson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Beautiful Soup incorporates code from the html5lib library, which is also made available under the MIT license. Copyright (c) 2006-2013 James Graham and other contributors Beautiful Soup depends on the soupsieve library, which is also made available under the MIT license. Copyright (c) 2018 Isaac Muse ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1532186221.0 beautifulsoup4-4.10.0/MANIFEST.in0000644000175000017500000000033300000000000017743 0ustar00leonardrleonardr00000000000000include test-all-versions include convert-py3k include LICENSE include *.txt include doc*/Makefile include doc*/source/*.py include doc*/source/*.rst include doc*/source/*.jpg include scripts/*.py include scripts/*.txt ././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1631059740.0 beautifulsoup4-4.10.0/NEWS.txt0000644000175000017500000017363600000000000017543 0ustar00leonardrleonardr00000000000000Beautiful Soup's official support for Python 2 ended on December 31st, 2020. The final release to support Python 2 was Beautiful Soup 4.9.3. In the Launchpad Bazaar repository, the final revision to support Python 2 was revision 605. = 4.10.0 (20210907) * This is the first release of Beautiful Soup to only support Python 3. I dropped Python 2 support to maintain support for newer versions (58 and up) of setuptools. See: https://github.com/pypa/setuptools/issues/2769 [bug=1942919] * The behavior of methods like .get_text() and .strings now differs depending on the type of tag. The change is visible with HTML tags like
This numeric entity is missing the final semicolon:
, that attribute value was closed by the subsequent tag
a
This document contains (do you see it?)
This document ends with That attribute value was bogus
The doctype is invalid because it contains extra whitespace
That boolean attribute had no value
Here's a nonexistent entity: &#foo; (do you see it?)
This document ends before the entity finishes: >

Paragraphs shouldn't contain block display elements, but this one does:

you see?

Multiple values for the same attribute.
Here's a table
This tag contains nothing but whitespace:

This p tag is cut off by

the end of the blockquote tag
Here's a nested table:
foo
This table contains bare markup
\n
This document contains a surprise doctype
Mixed case tags are folded to lowercase
Tag name contains Unicode characters
Attribute name contains Unicode characters
""" class SoupTest(unittest.TestCase): @property def default_builder(self): return default_builder def soup(self, markup, **kwargs): """Build a Beautiful Soup object from markup.""" builder = kwargs.pop('builder', self.default_builder) return BeautifulSoup(markup, builder=builder, **kwargs) def document_for(self, markup, **kwargs): """Turn an HTML fragment into a document. The details depend on the builder. """ return self.default_builder(**kwargs).test_fragment_to_document(markup) def assertSoupEquals(self, to_parse, compare_parsed_to=None): builder = self.default_builder obj = BeautifulSoup(to_parse, builder=builder) if compare_parsed_to is None: compare_parsed_to = to_parse # Verify that the documents come out the same. self.assertEqual(obj.decode(), self.document_for(compare_parsed_to)) # Also run some checks on the BeautifulSoup object itself: # Verify that every tag that was opened was eventually closed. # There are no tags in the open tag counter. assert all(v==0 for v in list(obj.open_tag_counter.values())) # The only tag in the tag stack is the one for the root # document. self.assertEqual( [obj.ROOT_TAG_NAME], [x.name for x in obj.tagStack] ) def assertConnectedness(self, element): """Ensure that next_element and previous_element are properly set for all descendants of the given element. """ earlier = None for e in element.descendants: if earlier: self.assertEqual(e, earlier.next_element) self.assertEqual(earlier, e.previous_element) earlier = e def linkage_validator(self, el, _recursive_call=False): """Ensure proper linkage throughout the document.""" descendant = None # Document element should have no previous element or previous sibling. # It also shouldn't have a next sibling. if el.parent is None: assert el.previous_element is None,\ "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format( el, el.previous_element, None ) assert el.previous_sibling is None,\ "Bad previous_sibling\nNODE: {}\nPREV: {}\nEXPECTED: {}".format( el, el.previous_sibling, None ) assert el.next_sibling is None,\ "Bad next_sibling\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format( el, el.next_sibling, None ) idx = 0 child = None last_child = None last_idx = len(el.contents) - 1 for child in el.contents: descendant = None # Parent should link next element to their first child # That child should have no previous sibling if idx == 0: if el.parent is not None: assert el.next_element is child,\ "Bad next_element\nNODE: {}\nNEXT: {}\nEXPECTED: {}".format( el, el.next_element, child ) assert child.previous_element is el,\ "Bad previous_element\nNODE: {}\nPREV: {}\nEXPECTED: {}".format( child, child.previous_element, el ) assert child.previous_sibling is None,\ "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED: {}".format( child, child.previous_sibling, None ) # If not the first child, previous index should link as sibling to this index # Previous element should match the last index or the last bubbled up descendant else: assert child.previous_sibling is el.contents[idx - 1],\ "Bad previous_sibling\nNODE: {}\nPREV {}\nEXPECTED {}".format( child, child.previous_sibling, el.contents[idx - 1] ) assert el.contents[idx - 1].next_sibling is child,\ "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format( el.contents[idx - 1], el.contents[idx - 1].next_sibling, child ) if last_child is not None: assert child.previous_element is last_child,\ "Bad previous_element\nNODE: {}\nPREV {}\nEXPECTED {}\nCONTENTS {}".format( child, child.previous_element, last_child, child.parent.contents ) assert last_child.next_element is child,\ "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format( last_child, last_child.next_element, child ) if isinstance(child, Tag) and child.contents: descendant = self.linkage_validator(child, True) # A bubbled up descendant should have no next siblings assert descendant.next_sibling is None,\ "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format( descendant, descendant.next_sibling, None ) # Mark last child as either the bubbled up descendant or the current child if descendant is not None: last_child = descendant else: last_child = child # If last child, there are non next siblings if idx == last_idx: assert child.next_sibling is None,\ "Bad next_sibling\nNODE: {}\nNEXT {}\nEXPECTED {}".format( child, child.next_sibling, None ) idx += 1 child = descendant if descendant is not None else child if child is None: child = el if not _recursive_call and child is not None: target = el while True: if target is None: assert child.next_element is None, \ "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format( child, child.next_element, None ) break elif target.next_sibling is not None: assert child.next_element is target.next_sibling, \ "Bad next_element\nNODE: {}\nNEXT {}\nEXPECTED {}".format( child, child.next_element, target.next_sibling ) break target = target.parent # We are done, so nothing to return return None else: # Return the child to the recursive caller return child class TreeBuilderSmokeTest(object): # Tests that are common to HTML and XML tree builders. def test_fuzzed_input(self): # This test centralizes in one place the various fuzz tests # for Beautiful Soup created by the oss-fuzz project. # These strings superficially resemble markup, but they # generally can't be parsed into anything. The best we can # hope for is that parsing these strings won't crash the # parser. # # n.b. This markup is commented out because these fuzz tests # _do_ crash the parser. However the crashes are due to bugs # in html.parser, not Beautiful Soup -- otherwise I'd fix the # bugs! bad_markup = [ # https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=28873 # https://github.com/guidovranken/python-library-fuzzers/blob/master/corp-html/519e5b4269a01185a0d5e76295251921da2f0700 # https://bugs.python.org/issue37747 # #b'\nSome CSS" ) assert isinstance(soup.style.string, Stylesheet) assert isinstance(soup.script.string, Script) soup = self.soup( "" ) assert isinstance(soup.style.string, Stylesheet) # The contents of the style tag resemble an HTML comment, but # it's not treated as a comment. self.assertEqual("", soup.style.string) assert isinstance(soup.style.string, Stylesheet) def test_pickle_and_unpickle_identity(self): # Pickling a tree, then unpickling it, yields a tree identical # to the original. tree = self.soup("foo") dumped = pickle.dumps(tree, 2) loaded = pickle.loads(dumped) self.assertEqual(loaded.__class__, BeautifulSoup) self.assertEqual(loaded.decode(), tree.decode()) def assertDoctypeHandled(self, doctype_fragment): """Assert that a given doctype string is handled correctly.""" doctype_str, soup = self._document_with_doctype(doctype_fragment) # Make sure a Doctype object was created. doctype = soup.contents[0] self.assertEqual(doctype.__class__, Doctype) self.assertEqual(doctype, doctype_fragment) self.assertEqual( soup.encode("utf8")[:len(doctype_str)], doctype_str ) # Make sure that the doctype was correctly associated with the # parse tree and that the rest of the document parsed. self.assertEqual(soup.p.contents[0], 'foo') def _document_with_doctype(self, doctype_fragment, doctype_string="DOCTYPE"): """Generate and parse a document with the given doctype.""" doctype = '' % (doctype_string, doctype_fragment) markup = doctype + '\n

foo

' soup = self.soup(markup) return doctype.encode("utf8"), soup def test_normal_doctypes(self): """Make sure normal, everyday HTML doctypes are handled correctly.""" self.assertDoctypeHandled("html") self.assertDoctypeHandled( 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"') def test_empty_doctype(self): soup = self.soup("") doctype = soup.contents[0] self.assertEqual("", doctype.strip()) def test_mixed_case_doctype(self): # A lowercase or mixed-case doctype becomes a Doctype. for doctype_fragment in ("doctype", "DocType"): doctype_str, soup = self._document_with_doctype( "html", doctype_fragment ) # Make sure a Doctype object was created and that the DOCTYPE # is uppercase. doctype = soup.contents[0] self.assertEqual(doctype.__class__, Doctype) self.assertEqual(doctype, "html") self.assertEqual( soup.encode("utf8")[:len(doctype_str)], b"" ) # Make sure that the doctype was correctly associated with the # parse tree and that the rest of the document parsed. self.assertEqual(soup.p.contents[0], 'foo') def test_public_doctype_with_url(self): doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"' self.assertDoctypeHandled(doctype) def test_system_doctype(self): self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"') def test_namespaced_system_doctype(self): # We can handle a namespaced doctype with a system ID. self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"') def test_namespaced_public_doctype(self): # Test a namespaced doctype with a public id. self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"') def test_real_xhtml_document(self): """A real XHTML document should come out more or less the same as it went in.""" markup = b""" Hello. Goodbye. """ soup = self.soup(markup) self.assertEqual( soup.encode("utf-8").replace(b"\n", b""), markup.replace(b"\n", b"")) def test_namespaced_html(self): """When a namespaced XML document is parsed as HTML it should be treated as HTML with weird tag names. """ markup = b"""content""" soup = self.soup(markup) self.assertEqual(2, len(soup.find_all("ns1:foo"))) def test_processing_instruction(self): # We test both Unicode and bytestring to verify that # process_markup correctly sets processing_instruction_class # even when the markup is already Unicode and there is no # need to process anything. markup = """""" soup = self.soup(markup) self.assertEqual(markup, soup.decode()) markup = b"""""" soup = self.soup(markup) self.assertEqual(markup, soup.encode("utf8")) def test_deepcopy(self): """Make sure you can copy the tree builder. This is important because the builder is part of a BeautifulSoup object, and we want to be able to copy that. """ copy.deepcopy(self.default_builder) def test_p_tag_is_never_empty_element(self): """A

tag is never designated as an empty-element tag. Even if the markup shows it as an empty-element tag, it shouldn't be presented that way. """ soup = self.soup("

") self.assertFalse(soup.p.is_empty_element) self.assertEqual(str(soup.p), "

") def test_unclosed_tags_get_closed(self): """A tag that's not closed by the end of the document should be closed. This applies to all tags except empty-element tags. """ self.assertSoupEquals("

", "

") self.assertSoupEquals("", "") self.assertSoupEquals("
", "
") def test_br_is_always_empty_element_tag(self): """A
tag is designated as an empty-element tag. Some parsers treat

as one
tag, some parsers as two tags, but it should always be an empty-element tag. """ soup = self.soup("

") self.assertTrue(soup.br.is_empty_element) self.assertEqual(str(soup.br), "
") def test_nested_formatting_elements(self): self.assertSoupEquals("") def test_double_head(self): html = ''' Ordinary HEAD element test Hello, world! ''' soup = self.soup(html) self.assertEqual("text/javascript", soup.find('script')['type']) def test_comment(self): # Comments are represented as Comment objects. markup = "

foobaz

" self.assertSoupEquals(markup) soup = self.soup(markup) comment = soup.find(text="foobar") self.assertEqual(comment.__class__, Comment) # The comment is properly integrated into the tree. foo = soup.find(text="foo") self.assertEqual(comment, foo.next_element) baz = soup.find(text="baz") self.assertEqual(comment, baz.previous_element) def test_preserved_whitespace_in_pre_and_textarea(self): """Whitespace must be preserved in 
 and "
        self.assertSoupEquals(pre_markup)
        self.assertSoupEquals(textarea_markup)

        soup = self.soup(pre_markup)
        self.assertEqual(soup.pre.prettify(), pre_markup)

        soup = self.soup(textarea_markup)
        self.assertEqual(soup.textarea.prettify(), textarea_markup)

        soup = self.soup("")
        self.assertEqual(soup.textarea.prettify(), "")

    def test_nested_inline_elements(self):
        """Inline elements can be nested indefinitely."""
        b_tag = "Inside a B tag"
        self.assertSoupEquals(b_tag)

        nested_b_tag = "
A nested tag
"
        self.assertSoupEquals(nested_b_tag)

        double_nested_b_tag = "
A doubly nested tag
"
        self.assertSoupEquals(nested_b_tag)

    def test_nested_block_level_elements(self):
        """Block elements can be nested."""
        soup = self.soup('
Foo
')
        blockquote = soup.blockquote
        self.assertEqual(blockquote.p.b.string, 'Foo')
        self.assertEqual(blockquote.b.string, 'Foo')

    def test_correctly_nested_tables(self):
        """One table can go inside another one."""
        markup = (''
                  ''
                  "')

        self.assertSoupEquals(
            markup,
            '
Here's another table:"
                  ''
                  ''
                  '
foo
Here\'s another table:'
            '
foo
'
            '
')

        self.assertSoupEquals(
            ""
            ""
            "
Foo
Bar
Baz
")

    def test_multivalued_attribute_with_whitespace(self):
        # Whitespace separating the values of a multi-valued attribute
        # should be ignored.

        markup = '
'
        soup = self.soup(markup)
        self.assertEqual(['foo', 'bar'], soup.div['class'])

        # If you search by the literal name of the class it's like the whitespace
        # wasn't there.
        self.assertEqual(soup.div, soup.find('div', class_="foo bar"))
        
    def test_deeply_nested_multivalued_attribute(self):
        # html5lib can set the attributes of the same tag many times
        # as it rearranges the tree. This has caused problems with
        # multivalued attributes.
        markup = '
'
        soup = self.soup(markup)
        self.assertEqual(["css"], soup.div.div['class'])

    def test_multivalued_attribute_on_html(self):
        # html5lib uses a different API to set the attributes ot the
        #  tag. This has caused problems with multivalued
        # attributes.
        markup = ''
        soup = self.soup(markup)
        self.assertEqual(["a", "b"], soup.html['class'])

    def test_angle_brackets_in_attribute_values_are_escaped(self):
        self.assertSoupEquals('', '')

    def test_strings_resembling_character_entity_references(self):
        # "&T" and "&p" look like incomplete character entities, but they are
        # not.
        self.assertSoupEquals(
            "
• AT&T is in the s&p 500
",
            "
\u2022 AT&T is in the s&p 500
"
        )

    def test_apos_entity(self):
        self.assertSoupEquals(
            "
Bob's Bar
",
            "
Bob's Bar
",
        )
        
    def test_entities_in_foreign_document_encoding(self):
        # “ and ” are invalid numeric entities referencing
        # Windows-1252 characters. - references a character common
        # to Windows-1252 and Unicode, and ☃ references a
        # character only found in Unicode.
        #
        # All of these entities should be converted to Unicode
        # characters.
        markup = "
“Hello” -☃
"
        soup = self.soup(markup)
        self.assertEqual("“Hello” -☃", soup.p.string)
        
    def test_entities_in_attributes_converted_to_unicode(self):
        expect = '
'
        self.assertSoupEquals('
', expect)
        self.assertSoupEquals('
', expect)
        self.assertSoupEquals('
', expect)
        self.assertSoupEquals('
', expect)

    def test_entities_in_text_converted_to_unicode(self):
        expect = '
pi\N{LATIN SMALL LETTER N WITH TILDE}ata
'
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)
        self.assertSoupEquals("
piñata
", expect)

    def test_quot_entity_converted_to_quotation_mark(self):
        self.assertSoupEquals("
I said "good day!"
",
                              '
I said "good day!"
')

    def test_out_of_range_entity(self):
        expect = "\N{REPLACEMENT CHARACTER}"
        self.assertSoupEquals("�", expect)
        self.assertSoupEquals("�", expect)
        self.assertSoupEquals("�", expect)
       
    def test_multipart_strings(self):
        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
        soup = self.soup("
\nfoo
")
        self.assertEqual("p", soup.h2.string.next_element.name)
        self.assertEqual("p", soup.p.name)
        self.assertConnectedness(soup)

    def test_empty_element_tags(self):
        """Verify consistent handling of empty-element tags,
        no matter how they come in through the markup.
        """
        self.assertSoupEquals('


', "


")
        self.assertSoupEquals('


', "


")
        
    def test_head_tag_between_head_and_body(self):
        "Prevent recurrence of a bug in the html5lib treebuilder."
        content = """
  
  foo

"""
        soup = self.soup(content)
        self.assertNotEqual(None, soup.html.body)
        self.assertConnectedness(soup)

    def test_multiple_copies_of_a_tag(self):
        "Prevent recurrence of a bug in the html5lib treebuilder."
        content = """

 
   
  

"""
        soup = self.soup(content)
        self.assertConnectedness(soup.article)

    def test_basic_namespaces(self):
        """Parsers don't need to *understand* namespaces, but at the
        very least they should not choke on namespaces or lose
        data."""

        markup = b'4'
        soup = self.soup(markup)
        self.assertEqual(markup, soup.encode())
        html = soup.html
        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
        self.assertEqual(
            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
        self.assertEqual(
            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])

    def test_multivalued_attribute_value_becomes_list(self):
        markup = b''
        soup = self.soup(markup)
        self.assertEqual(['foo', 'bar'], soup.a['class'])
        
    #
    # Generally speaking, tests below this point are more tests of
    # Beautiful Soup than tests of the tree builders. But parsers are
    # weird, so we run these tests separately for every tree builder
    # to detect any differences between them.
    #

    def test_can_parse_unicode_document(self):
        # A seemingly innocuous document... but it's in Unicode! And
        # it contains characters that can't be represented in the
        # encoding found in the  declaration! The horror!
        markup = 'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
        soup = self.soup(markup)
        self.assertEqual('Sacr\xe9 bleu!', soup.body.string)

    def test_soupstrainer(self):
        """Parsers should be able to work with SoupStrainers."""
        strainer = SoupStrainer("b")
        soup = self.soup("A bold  statement",
                         parse_only=strainer)
        self.assertEqual(soup.decode(), "bold")

    def test_single_quote_attribute_values_become_double_quotes(self):
        self.assertSoupEquals("",
                              '')

    def test_attribute_values_with_nested_quotes_are_left_alone(self):
        text = """a"""
        self.assertSoupEquals(text)

    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
        text = """a"""
        soup = self.soup(text)
        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
        self.assertSoupEquals(
            soup.foo.decode(),
            """a""")

    def test_ampersand_in_attribute_value_gets_escaped(self):
        self.assertSoupEquals('',
                              '')

        self.assertSoupEquals(
            'foo',
            'foo')

    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
        self.assertSoupEquals('')

    def test_entities_in_strings_converted_during_parsing(self):
        # Both XML and HTML entities are converted to Unicode characters
        # during parsing.
        text = "
<<sacré bleu!>>
"
        expected = "
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
"
        self.assertSoupEquals(text, expected)

    def test_smart_quotes_converted_on_the_way_in(self):
        # Microsoft smart quotes are converted to Unicode characters during
        # parsing.
        quote = b"
\x91Foo\x92
"
        soup = self.soup(quote)
        self.assertEqual(
            soup.p.string,
            "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")

    def test_non_breaking_spaces_converted_on_the_way_in(self):
        soup = self.soup("  ")
        self.assertEqual(soup.a.string, "\N{NO-BREAK SPACE}" * 2)

    def test_entities_converted_on_the_way_out(self):
        text = "
<<sacré bleu!>>
"
        expected = "
<<sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>
".encode("utf-8")
        soup = self.soup(text)
        self.assertEqual(soup.p.encode("utf-8"), expected)

    def test_real_iso_latin_document(self):
        # Smoke test of interrelated functionality, using an
        # easy-to-understand document.

        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
        unicode_html = '
Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!
'

        # That's because we're going to encode it into ISO-Latin-1, and use
        # that to test.
        iso_latin_html = unicode_html.encode("iso-8859-1")

        # Parse the ISO-Latin-1 HTML.
        soup = self.soup(iso_latin_html)
        # Encode it to UTF-8.
        result = soup.encode("utf-8")

        # What do we expect the result to look like? Well, it would
        # look like unicode_html, except that the META tag would say
        # UTF-8 instead of ISO-Latin-1.
        expected = unicode_html.replace("ISO-Latin-1", "utf-8")

        # And, of course, it would be in UTF-8, not Unicode.
        expected = expected.encode("utf-8")

        # Ta-da!
        self.assertEqual(result, expected)

    def test_real_shift_jis_document(self):
        # Smoke test to make sure the parser can handle a document in
        # Shift-JIS encoding, without choking.
        shift_jis_html = (
            b'
'
            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
            b'
')
        unicode_html = shift_jis_html.decode("shift-jis")
        soup = self.soup(unicode_html)

        # Make sure the parse tree is correctly encoded to various
        # encodings.
        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))

    def test_real_hebrew_document(self):
        # A real-world test to make sure we can convert ISO-8859-9 (a
        # Hebrew encoding) to UTF-8.
        hebrew_document = b'Hebrew (ISO 8859-8) in Visual Directionality
Hebrew (ISO 8859-8) in Visual Directionality
\xed\xe5\xec\xf9'
        soup = self.soup(
            hebrew_document, from_encoding="iso8859-8")
        # Some tree builders call it iso8859-8, others call it iso-8859-9.
        # That's not a difference we really care about.
        assert soup.original_encoding in ('iso8859-8', 'iso-8859-8')
        self.assertEqual(
            soup.encode('utf-8'),
            hebrew_document.decode("iso8859-8").encode("utf-8"))

    def test_meta_tag_reflects_current_encoding(self):
        # Here's the  tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('')

        # Here's a document incorporating that meta tag.
        shift_jis_html = (
            '\n%s\n'
            ''
            'Shift-JIS markup goes here.') % meta_tag
        soup = self.soup(shift_jis_html)

        # Parse the document, and the charset is seemingly unaffected.
        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
        content = parsed_meta['content']
        self.assertEqual('text/html; charset=x-sjis', content)

        # But that value is actually a ContentMetaAttributeValue object.
        self.assertTrue(isinstance(content, ContentMetaAttributeValue))

        # And it will take on a value that reflects its current
        # encoding.
        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))

        # For the rest of the story, see TestSubstitutions in
        # test_tree.py.

    def test_html5_style_meta_tag_reflects_current_encoding(self):
        # Here's the  tag saying that a document is
        # encoded in Shift-JIS.
        meta_tag = ('')

        # Here's a document incorporating that meta tag.
        shift_jis_html = (
            '\n%s\n'
            ''
            'Shift-JIS markup goes here.') % meta_tag
        soup = self.soup(shift_jis_html)

        # Parse the document, and the charset is seemingly unaffected.
        parsed_meta = soup.find('meta', id="encoding")
        charset = parsed_meta['charset']
        self.assertEqual('x-sjis', charset)

        # But that value is actually a CharsetMetaAttributeValue object.
        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))

        # And it will take on a value that reflects its current
        # encoding.
        self.assertEqual('utf8', charset.encode("utf8"))

    def test_python_specific_encodings_not_used_in_charset(self):
        # You can encode an HTML document using a Python-specific
        # encoding, but that encoding won't be mentioned _inside_ the
        # resulting document. Instead, the document will appear to
        # have no encoding.
        for markup in [
            b''
            b''
        ]:
            soup = self.soup(markup)
            for encoding in PYTHON_SPECIFIC_ENCODINGS:
                if encoding in (
                    'idna', 'mbcs', 'oem', 'undefined',
                    'string_escape', 'string-escape'
                ):
                    # For one reason or another, these will raise an
                    # exception if we actually try to use them, so don't
                    # bother.
                    continue
                encoded = soup.encode(encoding)
                assert b'meta charset=""' in encoded
                assert encoding.encode("ascii") not in encoded
        
    def test_tag_with_no_attributes_can_have_attributes_added(self):
        data = self.soup("text")
        data.a['foo'] = 'bar'
        self.assertEqual('text', data.a.decode())

    def test_closing_tag_with_no_opening_tag(self):
        # Without BeautifulSoup.open_tag_counter, the  tag will
        # cause _popToTag to be called over and over again as we look
        # for a  tag that wasn't there. The result is that 'text2'
        # will show up outside the body of the document.
        soup = self.soup("
text1
text2
")
        self.assertEqual(
            "
text1
text2
", soup.body.decode()
        )
        
    def test_worst_case(self):
        """Test the worst case (currently) for linking issues."""

        soup = self.soup(BAD_DOCUMENT)
        self.linkage_validator(soup)


class XMLTreeBuilderSmokeTest(TreeBuilderSmokeTest):

    def test_pickle_and_unpickle_identity(self):
        # Pickling a tree, then unpickling it, yields a tree identical
        # to the original.
        tree = self.soup("foo")
        dumped = pickle.dumps(tree, 2)
        loaded = pickle.loads(dumped)
        self.assertEqual(loaded.__class__, BeautifulSoup)
        self.assertEqual(loaded.decode(), tree.decode())

    def test_docstring_generated(self):
        soup = self.soup("")
        self.assertEqual(
            soup.encode(), b'\n')

    def test_xml_declaration(self):
        markup = b"""\n"""
        soup = self.soup(markup)
        self.assertEqual(markup, soup.encode("utf8"))

    def test_python_specific_encodings_not_used_in_xml_declaration(self):
        # You can encode an XML document using a Python-specific
        # encoding, but that encoding won't be mentioned _inside_ the
        # resulting document.
        markup = b"""\n"""
        soup = self.soup(markup)
        for encoding in PYTHON_SPECIFIC_ENCODINGS:
            if encoding in (
                'idna', 'mbcs', 'oem', 'undefined',
                'string_escape', 'string-escape'
            ):
                # For one reason or another, these will raise an
                # exception if we actually try to use them, so don't
                # bother.
                continue
            encoded = soup.encode(encoding)
            assert b'' in encoded
            assert encoding.encode("ascii") not in encoded

    def test_processing_instruction(self):
        markup = b"""\n"""
        soup = self.soup(markup)
        self.assertEqual(markup, soup.encode("utf8"))

    def test_real_xhtml_document(self):
        """A real XHTML document should come out *exactly* the same as it went in."""
        markup = b"""


Hello.
Goodbye.
"""
        soup = self.soup(markup)
        self.assertEqual(
            soup.encode("utf-8"), markup)
       
    def test_nested_namespaces(self):
        doc = b"""





"""
        soup = self.soup(doc)
        self.assertEqual(doc, soup.encode())
        
    def test_formatter_processes_script_tag_for_xml_documents(self):
        doc = """
  
"""
        soup = BeautifulSoup(doc, "lxml-xml")
        # lxml would have stripped this while parsing, but we can add
        # it later.
        soup.script.string = 'console.log("< < hey > > ");'
        encoded = soup.encode()
        self.assertTrue(b"< < hey > >" in encoded)

    def test_can_parse_unicode_document(self):
        markup = 'Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!'
        soup = self.soup(markup)
        self.assertEqual('Sacr\xe9 bleu!', soup.root.string)

    def test_popping_namespaced_tag(self):
        markup = 'b2012-07-02T20:33:42Zcd'
        soup = self.soup(markup)
        self.assertEqual(
            str(soup.rss), markup)

    def test_docstring_includes_correct_encoding(self):
        soup = self.soup("")
        self.assertEqual(
            soup.encode("latin1"),
            b'\n')

    def test_large_xml_document(self):
        """A large XML document should come out the same as it went in."""
        markup = (b'\n'
                  + b'0' * (2**12)
                  + b'')
        soup = self.soup(markup)
        self.assertEqual(soup.encode("utf-8"), markup)


    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
        self.assertSoupEquals("
", "
")
        self.assertSoupEquals("
foo
")

    def test_namespaces_are_preserved(self):
        markup = 'This tag is in the a namespaceThis tag is in the b namespace'
        soup = self.soup(markup)
        root = soup.root
        self.assertEqual("http://example.com/", root['xmlns:a'])
        self.assertEqual("http://example.net/", root['xmlns:b'])

    def test_closing_namespaced_tag(self):
        markup = '
20010504
'
        soup = self.soup(markup)
        self.assertEqual(str(soup.p), markup)

    def test_namespaced_attributes(self):
        markup = ''
        soup = self.soup(markup)
        self.assertEqual(str(soup.foo), markup)

    def test_namespaced_attributes_xml_namespace(self):
        markup = 'bar'
        soup = self.soup(markup)
        self.assertEqual(str(soup.foo), markup)

    def test_find_by_prefixed_name(self):
        doc = """
foo
    bar
    baz

"""
        soup = self.soup(doc)

        # There are three  tags.
        self.assertEqual(3, len(soup.find_all('tag')))

        # But two of them are ns1:tag and one of them is ns2:tag.
        self.assertEqual(2, len(soup.find_all('ns1:tag')))
        self.assertEqual(1, len(soup.find_all('ns2:tag')))
        
        self.assertEqual(1, len(soup.find_all('ns2:tag', key='value')))
        self.assertEqual(3, len(soup.find_all(['ns1:tag', 'ns2:tag'])))
        
    def test_copy_tag_preserves_namespace(self):
        xml = """
"""
    
        soup = self.soup(xml)
        tag = soup.document
        duplicate = copy.copy(tag)

        # The two tags have the same namespace prefix.
        self.assertEqual(tag.prefix, duplicate.prefix)

    def test_worst_case(self):
        """Test the worst case (currently) for linking issues."""

        soup = self.soup(BAD_DOCUMENT)
        self.linkage_validator(soup)


class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
    """Smoke test for a tree builder that supports HTML5."""

    def test_real_xhtml_document(self):
        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
        # XHTML documents in any particular way.
        pass

    def test_html_tags_have_namespace(self):
        markup = ""
        soup = self.soup(markup)
        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)

    def test_svg_tags_have_namespace(self):
        markup = ''
        soup = self.soup(markup)
        namespace = "http://www.w3.org/2000/svg"
        self.assertEqual(namespace, soup.svg.namespace)
        self.assertEqual(namespace, soup.circle.namespace)


    def test_mathml_tags_have_namespace(self):
        markup = '5'
        soup = self.soup(markup)
        namespace = 'http://www.w3.org/1998/Math/MathML'
        self.assertEqual(namespace, soup.math.namespace)
        self.assertEqual(namespace, soup.msqrt.namespace)

    def test_xml_declaration_becomes_comment(self):
        markup = ''
        soup = self.soup(markup)
        self.assertTrue(isinstance(soup.contents[0], Comment))
        self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
        self.assertEqual("html", soup.contents[0].next_element.name)

def skipIf(condition, reason):
   def nothing(test, *args, **kwargs):
       return None

   def decorator(test_item):
       if condition:
           return nothing
       else:
           return test_item

   return decorator
././@PaxHeader0000000000000000000000000000003400000000000011452 xustar000000000000000028 mtime=1631060004.6255705
beautifulsoup4-4.10.0/bs4/tests/0000775000175000017500000000000000000000000020042 5ustar00leonardrleonardr00000000000000././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1586116452.0
beautifulsoup4-4.10.0/bs4/tests/__init__.py0000644000175000017500000000003300000000000022145 0ustar00leonardrleonardr00000000000000"The beautifulsoup tests."
././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1586116452.0
beautifulsoup4-4.10.0/bs4/tests/test_builder_registry.py0000644000175000017500000001271600000000000025036 0ustar00leonardrleonardr00000000000000"""Tests of the builder registry."""

import unittest
import warnings

from bs4 import BeautifulSoup
from bs4.builder import (
    builder_registry as registry,
    HTMLParserTreeBuilder,
    TreeBuilderRegistry,
)

try:
    from bs4.builder import HTML5TreeBuilder
    HTML5LIB_PRESENT = True
except ImportError:
    HTML5LIB_PRESENT = False

try:
    from bs4.builder import (
        LXMLTreeBuilderForXML,
        LXMLTreeBuilder,
        )
    LXML_PRESENT = True
except ImportError:
    LXML_PRESENT = False


class BuiltInRegistryTest(unittest.TestCase):
    """Test the built-in registry with the default builders registered."""

    def test_combination(self):
        if LXML_PRESENT:
            self.assertEqual(registry.lookup('fast', 'html'),
                             LXMLTreeBuilder)

        if LXML_PRESENT:
            self.assertEqual(registry.lookup('permissive', 'xml'),
                             LXMLTreeBuilderForXML)
        self.assertEqual(registry.lookup('strict', 'html'),
                          HTMLParserTreeBuilder)
        if HTML5LIB_PRESENT:
            self.assertEqual(registry.lookup('html5lib', 'html'),
                              HTML5TreeBuilder)

    def test_lookup_by_markup_type(self):
        if LXML_PRESENT:
            self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
        else:
            self.assertEqual(registry.lookup('xml'), None)
            if HTML5LIB_PRESENT:
                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
            else:
                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)

    def test_named_library(self):
        if LXML_PRESENT:
            self.assertEqual(registry.lookup('lxml', 'xml'),
                             LXMLTreeBuilderForXML)
            self.assertEqual(registry.lookup('lxml', 'html'),
                             LXMLTreeBuilder)
        if HTML5LIB_PRESENT:
            self.assertEqual(registry.lookup('html5lib'),
                              HTML5TreeBuilder)

        self.assertEqual(registry.lookup('html.parser'),
                          HTMLParserTreeBuilder)

    def test_beautifulsoup_constructor_does_lookup(self):

        with warnings.catch_warnings(record=True) as w:
            # This will create a warning about not explicitly
            # specifying a parser, but we'll ignore it.

            # You can pass in a string.
            BeautifulSoup("", features="html")
            # Or a list of strings.
            BeautifulSoup("", features=["html", "fast"])

        # You'll get an exception if BS can't find an appropriate
        # builder.
        self.assertRaises(ValueError, BeautifulSoup,
                          "", features="no-such-feature")

class RegistryTest(unittest.TestCase):
    """Test the TreeBuilderRegistry class in general."""

    def setUp(self):
        self.registry = TreeBuilderRegistry()

    def builder_for_features(self, *feature_list):
        cls = type('Builder_' + '_'.join(feature_list),
                   (object,), {'features' : feature_list})

        self.registry.register(cls)
        return cls

    def test_register_with_no_features(self):
        builder = self.builder_for_features()

        # Since the builder advertises no features, you can't find it
        # by looking up features.
        self.assertEqual(self.registry.lookup('foo'), None)

        # But you can find it by doing a lookup with no features, if
        # this happens to be the only registered builder.
        self.assertEqual(self.registry.lookup(), builder)

    def test_register_with_features_makes_lookup_succeed(self):
        builder = self.builder_for_features('foo', 'bar')
        self.assertEqual(self.registry.lookup('foo'), builder)
        self.assertEqual(self.registry.lookup('bar'), builder)

    def test_lookup_fails_when_no_builder_implements_feature(self):
        builder = self.builder_for_features('foo', 'bar')
        self.assertEqual(self.registry.lookup('baz'), None)

    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
        builder1 = self.builder_for_features('foo')
        builder2 = self.builder_for_features('bar')
        self.assertEqual(self.registry.lookup(), builder2)

    def test_lookup_fails_when_no_tree_builders_registered(self):
        self.assertEqual(self.registry.lookup(), None)

    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
        has_one = self.builder_for_features('foo')
        has_the_other = self.builder_for_features('bar')
        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
        lacks_one = self.builder_for_features('bar')
        has_the_other = self.builder_for_features('foo')

        # There are two builders featuring 'foo' and 'bar', but
        # the one that also features 'quux' was registered later.
        self.assertEqual(self.registry.lookup('foo', 'bar'),
                          has_both_late)

        # There is only one builder featuring 'foo', 'bar', and 'baz'.
        self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
                          has_both_early)

    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
        builder1 = self.builder_for_features('foo', 'bar')
        builder2 = self.builder_for_features('foo', 'baz')
        self.assertEqual(self.registry.lookup('bar', 'baz'), None)
././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1586116452.0
beautifulsoup4-4.10.0/bs4/tests/test_docs.py0000644000175000017500000000205300000000000022401 0ustar00leonardrleonardr00000000000000"Test harness for doctests."

# pylint: disable-msg=E0611,W0142

__metaclass__ = type
__all__ = [
    'additional_tests',
    ]

import atexit
import doctest
import os
#from pkg_resources import (
#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
import unittest

DOCTEST_FLAGS = (
    doctest.ELLIPSIS |
    doctest.NORMALIZE_WHITESPACE |
    doctest.REPORT_NDIFF)


# def additional_tests():
#     "Run the doc tests (README.txt and docs/*, if any exist)"
#     doctest_files = [
#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
#     if resource_exists('bs4', 'docs'):
#         for name in resource_listdir('bs4', 'docs'):
#             if name.endswith('.txt'):
#                 doctest_files.append(
#                     os.path.abspath(
#                         resource_filename('bs4', 'docs/%s' % name)))
#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
#     atexit.register(cleanup_resources)
#     return unittest.TestSuite((
#         doctest.DocFileSuite(*doctest_files, **kwargs)))
././@PaxHeader0000000000000000000000000000002600000000000011453 xustar000000000000000022 mtime=1631057807.0
beautifulsoup4-4.10.0/bs4/tests/test_html5lib.py0000644000175000017500000002046300000000000023176 0ustar00leonardrleonardr00000000000000"""Tests to ensure that the html5lib tree builder generates good trees."""

import warnings

try:
    from bs4.builder import HTML5TreeBuilder
    HTML5LIB_PRESENT = True
except ImportError as e:
    HTML5LIB_PRESENT = False
from bs4.element import SoupStrainer
from bs4.testing import (
    HTML5TreeBuilderSmokeTest,
    SoupTest,
    skipIf,
)

@skipIf(
    not HTML5LIB_PRESENT,
    "html5lib seems not to be present, not testing its tree builder.")
class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
    """See ``HTML5TreeBuilderSmokeTest``."""

    @property
    def default_builder(self):
        return HTML5TreeBuilder

    def test_soupstrainer(self):
        # The html5lib tree builder does not support SoupStrainers.
        strainer = SoupStrainer("b")
        markup = "
A bold statement.
"
        with warnings.catch_warnings(record=True) as w:
            soup = self.soup(markup, parse_only=strainer)
        self.assertEqual(
            soup.decode(), self.document_for(markup))

        self.assertTrue(
            "the html5lib tree builder doesn't support parse_only" in
            str(w[0].message))

    def test_correctly_nested_tables(self):
        """html5lib inserts  tags where other parsers don't."""
        markup = (''
                  ''
                  "')

        self.assertSoupEquals(
            markup,
            '
Here's another table:"
                  ''
                  ''
                  '
foo
Here\'s another table:'
            '
foo
'
            '
')

        self.assertSoupEquals(
            ""
            ""
            "
Foo
Bar
Baz
")

    def test_xml_declaration_followed_by_doctype(self):
        markup = '''


  
  
  
   
foo

  
'''
        soup = self.soup(markup)
        # Verify that we can reach the 
 tag; this means the tree is connected.
        self.assertEqual(b"
foo
", soup.p.encode())

    def test_reparented_markup(self):
        markup = '
foo
\n
bar
'
        soup = self.soup(markup)
        self.assertEqual("
foo
\n
bar
", soup.body.decode())
        self.assertEqual(2, len(soup.find_all('p')))


    def test_reparented_markup_ends_with_whitespace(self):
        markup = '
foo
\n
bar
\n'
        soup = self.soup(markup)
        self.assertEqual("
foo
\n
bar
\n", soup.body.decode())
        self.assertEqual(2, len(soup.find_all('p')))

    def test_reparented_markup_containing_identical_whitespace_nodes(self):
        """Verify that we keep the two whitespace nodes in this
        document distinct when reparenting the adjacent  tags.
        """
        markup = '
  
'
        soup = self.soup(markup)
        space1, space2 = soup.find_all(string=' ')
        tbody1, tbody2 = soup.find_all('tbody')
        assert space1.next_element is tbody1
        assert tbody2.next_element is space2

    def test_reparented_markup_containing_children(self):
        markup = ''
        soup = self.soup(markup)
        noscript = soup.noscript
        self.assertEqual("target", noscript.next_element)
        target = soup.find(string='target')

        # The 'aftermath' string was duplicated; we want the second one.
        final_aftermath = soup.find_all(string='aftermath')[-1]

        # The