diff --git a/src/crawlee/_utils/sitemap.py b/src/crawlee/_utils/sitemap.py index d7844e9168..190381c1f2 100644 --- a/src/crawlee/_utils/sitemap.py +++ b/src/crawlee/_utils/sitemap.py @@ -127,8 +127,9 @@ def endElement(self, name: str) -> None: self._current_tag = None - if name == 'url' and 'loc' in self._current_url: - self.items.append({'type': 'url', **self._current_url}) + if name == 'url': + if 'loc' in self._current_url: + self.items.append({'type': 'url', **self._current_url}) self._current_url = {} diff --git a/tests/unit/_utils/test_sitemap.py b/tests/unit/_utils/test_sitemap.py index 8471331d7d..e746c314c7 100644 --- a/tests/unit/_utils/test_sitemap.py +++ b/tests/unit/_utils/test_sitemap.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, cast from unittest.mock import AsyncMock, MagicMock +from xml.sax.expatreader import ExpatParser import pytest from yarl import URL @@ -412,6 +413,22 @@ def test_xml_handler_resets_current_tag_on_end_element() -> None: assert handler._buffer == 'https://example.com/' +def test_xml_handler_discards_metadata_from_url_without_loc() -> None: + """A block with metadata but no must not leak its metadata into the next .""" + handler = _XMLSaxSitemapHandler() + parser = ExpatParser() + parser.setContentHandler(handler) + parser.feed( + '' + '' + '2005-01-010.1daily' + 'https://example.com/real-page' + '' + ) + + assert handler.items == [{'type': 'url', 'loc': 'https://example.com/real-page'}] + + async def test_txt_parser_flush_clears_buffer() -> None: """Feeding more data after flush() must not concatenate the previously flushed URL.""" parser = _TxtSitemapParser()