From 79b8599808ed7dc62b270594c42228d077799d36 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:46:05 +0000 Subject: [PATCH 1/6] Fix sitemap generation bugs and add a command line interface Fixes: - resolve exclusions against the site root so a site below a folder that matches an exclusion (e.g. 'media') is no longer wiped out - percent-encode URLs (spaces, '#', '&', non-ASCII) so sitemaps are valid - match extensions case-insensitively and never treat 'x.html.bak' as HTML - honour documented priority keys such as 'blog/' and '.PHP' - validate root_path/base_url/max_urls, clamp priorities to 0.00-1.00 - sort and de-duplicate entries for reproducible output - drop stale split parts and write output atomically - replace the minidom round-trip with a fast streaming XML writer New: - localsitemap CLI (also 'python -m localsitemap') - glob exclusions, exclude_hidden, include_homepage, sort_entries options - generate_sitemap() now returns the list of written files Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- localsitemap/__init__.py | 802 +++++++++++++++++++++++++++++---------- localsitemap/__main__.py | 8 + localsitemap/_version.py | 7 + localsitemap/cli.py | 135 +++++++ test.py | 26 +- test_localsitemap.py | 568 +++++++++++++++++++++++---- 6 files changed, 1256 insertions(+), 290 deletions(-) create mode 100644 localsitemap/__main__.py create mode 100644 localsitemap/_version.py create mode 100644 localsitemap/cli.py diff --git a/localsitemap/__init__.py b/localsitemap/__init__.py index 5456ff4..7918e98 100644 --- a/localsitemap/__init__.py +++ b/localsitemap/__init__.py @@ -1,125 +1,522 @@ -import os -import datetime -from xml.etree.ElementTree import Element, SubElement, tostring -from xml.dom import minidom +"""LocalSiteMap - generate standard-compliant sitemaps from local files. -def is_excluded(path, root_path, excluded_paths): - """ - Checks if a path is excluded based on a list of excluded paths/directories. - Only direct matches or matching path components are excluded, preventing partial substring matches. - """ - if not excluded_paths: - return False +The package crawls a directory tree that mirrors a website and writes a +``sitemap.xml`` file (or a set of numbered sitemaps plus a ``sitemapindex`` +when the URL limit is exceeded). - # Get absolute path and split into segments - abs_path = os.path.abspath(path) - path_segments = abs_path.split(os.sep) +Only the Python standard library is used, so the package has no runtime +dependencies. +""" - for excl in excluded_paths: - if not excl: - continue - # Standardize separators in excl - excl_norm = excl.replace('/', os.sep).replace('\\', os.sep) +import datetime +import fnmatch +import math +import os +import re +import tempfile +import warnings +from urllib.parse import quote + +from ._version import __version__ + +__all__ = [ + "__version__", + "generate_sitemap", + "get_priority", + "get_utc_lastmod", + "is_excluded", + "write_sitemap_file", + "write_sitemap_index_file", +] + +SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9" +XSI_NS = "http://www.w3.org/2001/XMLSchema-instance" +SITEMAP_SCHEMA = SITEMAP_NS + "/sitemap.xsd" + +#: Hard limit enforced by the sitemap protocol (and Google) per sitemap file. +MAX_URLS_PER_SITEMAP = 50000 + +DEFAULT_EXTENSIONS = (".html", ".htm") + +_GLOB_CHARS = re.compile(r"[*?\[]") +_SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://") +_WINDOWS = os.name == "nt" + + +# --------------------------------------------------------------------------- +# Small helpers +# --------------------------------------------------------------------------- + +def _as_path_string(path): + """Return *path* as a ``str`` (accepts ``os.PathLike`` objects).""" + if hasattr(os, "fspath"): + path = os.fspath(path) + if isinstance(path, bytes): + path = os.fsdecode(path) + return path + + +def _segment_matches(segment, pattern): + """Compare one path segment with an exclusion pattern (glob aware).""" + if _GLOB_CHARS.search(pattern): + if fnmatch.fnmatchcase(segment, pattern): + return True + if _WINDOWS: + return fnmatch.fnmatchcase(segment.lower(), pattern.lower()) + return False + if _WINDOWS: + return segment.lower() == pattern.lower() + return segment == pattern - # If it's a simple name (no separators) - if os.sep not in excl_norm: - if excl_norm in path_segments: - return True - continue - # If it's a path (contains separators), we resolve it - # Try both assuming it's an absolute path or relative to root_path - if os.path.isabs(excl_norm): - abs_excl = os.path.abspath(excl_norm) - else: - abs_excl = os.path.abspath(os.path.join(root_path, excl_norm)) +def _normalise_extension(extension): + extension = str(extension).strip() + if not extension: + return "" + if not extension.startswith("."): + extension = "." + extension + return extension.lower() + - excl_segments = abs_excl.split(os.sep) - # Check if excl_segments is a prefix of path_segments - if len(path_segments) >= len(excl_segments): - if path_segments[:len(excl_segments)] == excl_segments: - return True +def _format_priority(value, default=None): + """Format a priority in the ``0.00`` - ``1.00`` range used by sitemaps.""" + try: + priority = float(value) + except (TypeError, ValueError) as error: + if default is None: + raise ValueError( + "priority values must be numbers, got %r" % (value,) + ) from error + priority = float(default) + if math.isnan(priority): + priority = float(default if default is not None else 0.5) + # The sitemap protocol only allows values between 0.0 and 1.0. + priority = min(1.0, max(0.0, priority)) + return "%.2f" % priority - return False def get_utc_lastmod(path): - """ - Returns the UTC last modified time of the path in YYYY-MM-DDThh:mm:ss+00:00 format. + """Return the UTC modification time of *path* formatted for ````. + + The value is an ISO-8601 date/time in UTC (``YYYY-MM-DDThh:mm:ss+00:00``). + If the timestamp cannot be read (for example because the file vanished + while the tree was being walked) the current UTC time is used instead. """ try: mtime = os.path.getmtime(path) - dt = datetime.datetime.fromtimestamp(mtime, tz=datetime.timezone.utc) - return dt.strftime("%Y-%m-%dT%H:%M:%S+00:00") - except Exception: - return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S+00:00") + moment = datetime.datetime.fromtimestamp(mtime, tz=datetime.timezone.utc) + except (OSError, OverflowError, ValueError): + moment = datetime.datetime.now(datetime.timezone.utc) + return moment.strftime("%Y-%m-%dT%H:%M:%S+00:00") -def get_priority(relative_path, is_dir, default_priority, priority_mapping=None): + +# --------------------------------------------------------------------------- +# Exclusions +# --------------------------------------------------------------------------- + +class _PathFilter(object): + """Decides whether a path inside the site root is excluded. + + Rules are resolved relative to the site root, so an exclusion can never + accidentally match a *parent* directory of the site (for example a site + checked out in a folder called ``media`` while ``media`` is excluded). """ - Computes priority based on the relative path, directory status, default, and mapping. + + def __init__(self, root_path, excluded_paths=None, exclude_hidden=False): + self.root_path = os.path.abspath(_as_path_string(root_path)) + self.exclude_hidden = bool(exclude_hidden) + if isinstance(excluded_paths, str): + excluded_paths = [excluded_paths] + self.rules = self._build_rules(excluded_paths or []) + + def _build_rules(self, excluded_paths): + rules = [] + seen = set() + for raw in excluded_paths: + if raw is None: + continue + text = str(raw).strip() + if not text: + continue + candidate = os.path.abspath( + text if os.path.isabs(text) + else os.path.join(self.root_path, text.replace("\\", "/")) + ) + try: + relative = os.path.relpath(candidate, self.root_path) + except ValueError: # pragma: no cover - different drives on Windows + continue + if relative == os.curdir: + # Excluding the root itself is never useful: the site root is + # always kept so the homepage is still emitted. + continue + parts = [part for part in relative.split(os.sep) if part not in ("", os.curdir)] + if not parts or parts[0] == os.pardir: + # Outside of the site root, therefore irrelevant. + continue + # A single relative name matches at any depth (e.g. "node_modules"); + # multi-segment patterns are anchored at the site root. + anchor = len(parts) > 1 or os.path.isabs(text) + key = (tuple(parts), anchor) + if key in seen: + continue + seen.add(key) + rules.append((parts, anchor)) + return rules + + def _segments(self, path): + absolute = os.path.abspath(_as_path_string(path)) + try: + relative = os.path.relpath(absolute, self.root_path) + except ValueError: # pragma: no cover - different drives on Windows + return None + if relative == os.curdir: + return [] + parts = [part for part in relative.split(os.sep) if part not in ("", os.curdir)] + if parts and parts[0] == os.pardir: + return None + return parts + + def excludes(self, path): + segments = self._segments(path) + if segments is None: + return False + if not segments: + return False + + if self.exclude_hidden: + for segment in segments: + if segment.startswith("."): + return True + + for pattern, anchor in self.rules: + if anchor: + if len(segments) < len(pattern): + continue + if all( + _segment_matches(segment, part) + for segment, part in zip(segments, pattern) + ): + return True + elif len(pattern) == 1: + if any(_segment_matches(segment, pattern[0]) for segment in segments): + return True + return False + + +def is_excluded(path, root_path, excluded_paths): + """Return ``True`` when *path* should be skipped. + + ``path`` and ``root_path`` may be absolute or relative. Exclusions are + interpreted relative to ``root_path``: + + * a plain name (``"node_modules"``) matches a file or folder with that + name at any depth, + * a relative path (``"blog/posts"``) matches that path and everything + below it, + * an absolute path matches only that path (and its contents) when it is + inside ``root_path``, + * shell-style wildcards (``"draft-*"``, ``"*.bak"``) are supported. """ + if not excluded_paths: + return False + return _PathFilter(root_path, excluded_paths).excludes(path) + + +# --------------------------------------------------------------------------- +# Priorities +# --------------------------------------------------------------------------- + +def _normalise_priority_mapping(mapping): + normalised = {} + for key, value in mapping.items(): + if key is None: + continue + text = str(key).strip().replace("\\", "/").strip("/") + if not text: + continue + normalised[text] = value + if text.startswith(".") and "/" not in text: + # Extension keys are matched case-insensitively. + normalised.setdefault(text.lower(), value) + return normalised + + +def _lookup_mapping_priority(mapping, relative_path, is_dir): + """Look up ``relative_path`` in an already normalised mapping.""" + rel = str(relative_path).replace("\\", "/").strip("/") + if not rel: + return None, None + basename = rel.rsplit("/", 1)[-1] + _, extension = os.path.splitext(basename) + candidates = [rel] + if is_dir or not extension: + candidates.append(rel + "/") + candidates.extend([basename, extension]) + if extension: + candidates.append(extension.lower()) + candidates.append(extension.lstrip(".").lower()) + for candidate in candidates: + if candidate and candidate in mapping: + return mapping[candidate], candidate + return None, None + + +def _resolve_priority(relative_path, is_dir, default_priority, priority_mapping): + """Resolve a priority; *priority_mapping* must already be normalised.""" if priority_mapping: if callable(priority_mapping): + value = None try: - p = priority_mapping(relative_path, is_dir) - if p is not None: - return f"{float(p):.2f}" - except Exception: - pass + value = priority_mapping(relative_path, is_dir) + except Exception: # noqa: BLE001 - user supplied callables may fail + value = None + if value is not None: + return _format_priority(value, default=default_priority) elif isinstance(priority_mapping, dict): - # Standardize relative_path to use forward slashes - norm_rel = relative_path.replace("\\", "/") - if norm_rel in priority_mapping: - return f"{float(priority_mapping[norm_rel]):.2f}" - # Also try matching file/dir name (basename) - basename = os.path.basename(relative_path) - if basename in priority_mapping: - return f"{float(priority_mapping[basename]):.2f}" - # Try matching by extension - _, ext = os.path.splitext(basename) - if ext in priority_mapping: - return f"{float(priority_mapping[ext]):.2f}" - return f"{float(default_priority):.2f}" + value, _key = _lookup_mapping_priority(priority_mapping, relative_path, is_dir) + if value is not None: + return _format_priority(value, default=default_priority) + return _format_priority(default_priority) -def write_sitemap_file(file_path, entries): - """ - Writes a list of entries to a sitemap XML file. - """ - urlset = Element("urlset") - urlset.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9") - urlset.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") - urlset.set("xsi:schemaLocation", "http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd") +def get_priority(relative_path, is_dir, default_priority, priority_mapping=None): + """Return the formatted priority for *relative_path*. + + ``priority_mapping`` may be a callable (``f(relative_path, is_dir)``) or a + dict. Dict keys may be a relative path (``"about/index.html"``), a folder + name, a file name, or an extension (``".php"``); folder keys may be + written with or without a trailing slash and Windows separators are + accepted. Priorities are clamped to the ``0.00`` - ``1.00`` range allowed + by the sitemap protocol. + """ + if isinstance(priority_mapping, dict): + priority_mapping = _normalise_priority_mapping(priority_mapping) + return _resolve_priority(relative_path, is_dir, default_priority, priority_mapping) + + +# --------------------------------------------------------------------------- +# URL helpers +# --------------------------------------------------------------------------- + +def _normalise_base_url(base_url): + base = str(base_url).strip() + if not _SCHEME_RE.match(base): + raise ValueError( + "base_url must be an absolute URL including its scheme " + "(for example 'https://example.com'), got %r" % (base_url,) + ) + return base.rstrip("/") + + +def _quote_segment(segment): + # ``quote`` never escapes unreserved characters, so "index.html" is left + # untouched while spaces, "#", "&" and non-ASCII characters are encoded. + return quote(segment, safe="") + + +def _url_for(base, relative_path, is_dir): + """Build an absolute, percent-encoded URL for a relative filesystem path.""" + posix = str(relative_path).replace("\\", "/") + parts = [part for part in posix.split("/") if part not in ("", ".")] + encoded = "/".join(_quote_segment(part) for part in parts) + if not encoded: + return base + url = base + "/" + encoded + if is_dir: + url += "/" + return url + + +# --------------------------------------------------------------------------- +# XML writers +# --------------------------------------------------------------------------- + +def _xml_escape(value): + return ( + str(value) + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +def _atomic_write(file_path, content): + """Write *content* to *file_path* atomically (create parent directories).""" + target = os.path.abspath(_as_path_string(file_path)) + directory = os.path.dirname(target) or os.curdir + os.makedirs(directory, exist_ok=True) + handle_fd, temporary = tempfile.mkstemp( + prefix=".localsitemap-", suffix=".tmp", dir=directory + ) + try: + with os.fdopen(handle_fd, "w", encoding="utf-8", newline="\n") as handle: + handle.write(content) + os.replace(temporary, target) + except BaseException: + try: + os.remove(temporary) + except OSError: + pass + raise + + +def _render_urlset(entries): + lines = [ + '', + '' + % (SITEMAP_NS, XSI_NS, SITEMAP_NS, SITEMAP_SCHEMA), + ] for entry in entries: - url_entry = SubElement(urlset, "url") - SubElement(url_entry, "loc").text = entry["loc"] - SubElement(url_entry, "lastmod").text = entry["lastmod"] - SubElement(url_entry, "priority").text = entry["priority"] + lines.append(" ") + lines.append(" %s" % _xml_escape(entry["loc"])) + if entry.get("lastmod"): + lines.append(" %s" % _xml_escape(entry["lastmod"])) + if entry.get("priority") is not None: + lines.append(" %s" % _xml_escape(entry["priority"])) + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + + +def _render_sitemapindex(sitemaps_info): + lines = [ + '', + '' % SITEMAP_NS, + ] + for info in sitemaps_info: + lines.append(" ") + lines.append(" %s" % _xml_escape(info["loc"])) + if info.get("lastmod"): + lines.append(" %s" % _xml_escape(info["lastmod"])) + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + - xml_string = tostring(urlset, "utf-8") - reparsed = minidom.parseString(xml_string) - pretty_xml = reparsed.toprettyxml(indent=" ") +def write_sitemap_file(file_path, entries): + """Write *entries* to *file_path* as a sitemap ````.""" + _atomic_write(file_path, _render_urlset(entries)) - with open(file_path, "w", encoding="utf-8") as f: - f.write(pretty_xml) def write_sitemap_index_file(file_path, sitemaps_info): - """ - Writes a list of sitemap files to a sitemap index XML file. - """ - sitemapindex = Element("sitemapindex") - sitemapindex.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9") + """Write *sitemaps_info* to *file_path* as a ````.""" + _atomic_write(file_path, _render_sitemapindex(sitemaps_info)) + + +# --------------------------------------------------------------------------- +# Sitemap generation +# --------------------------------------------------------------------------- + +def _walk_entries(root_path, base_url, path_filter, extensions, priority_mapping, + default_priority, show_progress): + entries = [] + seen_locations = set() + + def add(loc, source_path, is_dir, priority=None): + if loc in seen_locations: + return + seen_locations.add(loc) + entries.append({ + "loc": loc, + "lastmod": get_utc_lastmod(source_path), + "priority": priority if priority is not None else _resolve_priority( + _relative_posix(source_path, root_path), + is_dir=is_dir, + default_priority=default_priority, + priority_mapping=priority_mapping, + ), + }) + + for current_root, dirnames, filenames in os.walk(root_path): + # Keep the walk deterministic and prune excluded directories in place + # so their contents are never visited. + dirnames[:] = sorted( + name for name in dirnames + if not path_filter.excludes(os.path.join(current_root, name)) + ) + filenames = sorted(filenames) + + for dir_name in dirnames: + full_path = os.path.join(current_root, dir_name) + if show_progress: + print("Mapping directory: %s" % full_path) + add(_url_for(base_url, _relative_posix(full_path, root_path), is_dir=True), + full_path, True) + + for file_name in filenames: + full_path = os.path.join(current_root, file_name) + if path_filter.excludes(full_path): + continue + if not _has_allowed_extension(file_name, extensions): + continue + if show_progress: + print("Found file: %s" % full_path) + add(_url_for(base_url, _relative_posix(full_path, root_path), is_dir=False), + full_path, False) + + return entries + + +def _relative_posix(path, root_path): + relative = os.path.relpath(path, root_path) + return relative.replace(os.sep, "/") + + +def _has_allowed_extension(file_name, extensions): + lowered = file_name.lower() + return any(lowered.endswith(extension) for extension in extensions) - for info in sitemaps_info: - sitemap_entry = SubElement(sitemapindex, "sitemap") - SubElement(sitemap_entry, "loc").text = info["loc"] - SubElement(sitemap_entry, "lastmod").text = info["lastmod"] - xml_string = tostring(sitemapindex, "utf-8") - reparsed = minidom.parseString(xml_string) - pretty_xml = reparsed.toprettyxml(indent=" ") +def _split_sitemaps(entries, output_file, base_url, max_urls, show_progress): + """Write numbered sitemaps plus an index; return the created file paths.""" + output_file = _as_path_string(output_file) + directory = os.path.dirname(output_file) + base_name = os.path.basename(output_file) + name, extension = os.path.splitext(base_name) + + part_count = int(math.ceil(len(entries) / float(max_urls))) + created = [] + sitemaps_info = [] + + for part_number in range(1, part_count + 1): + chunk = entries[(part_number - 1) * max_urls:part_number * max_urls] + part_filename = "%s_%d%s" % (name, part_number, extension) + part_path = os.path.join(directory, part_filename) if directory else part_filename + if show_progress: + print("Saving split sitemap part %d to %s..." % (part_number, part_path)) + write_sitemap_file(part_path, chunk) + created.append(part_path) + sitemaps_info.append({ + "loc": base_url + "/" + _quote_segment(part_filename), + "lastmod": get_utc_lastmod(part_path), + }) + + # Remove leftovers from a previous, larger run so stale sitemaps are not + # left behind next to the freshly written index. + stale_pattern = re.compile(r"^%s_([0-9]+)%s$" % (re.escape(name), re.escape(extension))) + search_directory = directory or os.curdir + if os.path.isdir(search_directory): + for existing in sorted(os.listdir(search_directory)): + match = stale_pattern.match(existing) + if not match or int(match.group(1)) <= part_count: + continue + stale_path = os.path.join(search_directory, existing) + try: + os.remove(stale_path) + except OSError: + continue + if show_progress: + print("Removed stale sitemap part: %s" % stale_path) + + if show_progress: + print("Saving sitemap index to %s..." % output_file) + write_sitemap_index_file(output_file, sitemaps_info) + created.append(output_file) + return created - with open(file_path, "w", encoding="utf-8") as f: - f.write(pretty_xml) def generate_sitemap( root_path, @@ -131,126 +528,123 @@ def generate_sitemap( default_priority=0.80, homepage_priority=1.00, priority_mapping=None, - max_urls=50000 + max_urls=50000, + exclude_hidden=False, + include_homepage=True, + sort_entries=True, ): + """Generate a sitemap for a local website directory. + + Parameters + ---------- + root_path: + Root directory of the website. + base_url: + Public base URL of the website (for example ``https://example.com``). + output_file: + Sitemap path to write. When the URL limit is exceeded numbered parts + (``sitemap_1.xml``, ...) plus a ``sitemap.xml`` index are written. + excluded_paths: + File/folder names, relative paths, absolute paths or glob patterns to + skip. Matching is relative to ``root_path``. + show_progress: + Print the traversal progress. + allowed_extensions: + Extensions to include (default: ``[".html", ".htm"]``). Matching is + case-insensitive and the leading dot is optional. + default_priority: + Priority used when no mapping matches (default ``0.80``). + homepage_priority: + Priority for the site root (default ``1.00``). + priority_mapping: + Dict or callable mapping paths/folders/extensions to priorities. + max_urls: + Maximum number of URLs per sitemap file before splitting (max 50000). + exclude_hidden: + Skip files and folders starting with a dot. + include_homepage: + Emit the site root URL (default ``True``). + sort_entries: + Sort entries by URL for reproducible output (default ``True``). + + Returns + ------- + list of str + The paths that were written, in the order they were written. When the + sitemap is split, the numbered parts come first and the index last. """ - Generates a sitemap XML file or multiple files plus an index if URLs exceed max_urls. - - Parameters: - root_path: The root directory of your website. - base_url: The base URL of your website (e.g., "https://example.com"). - output_file: The name of the output XML file (default: "sitemap.xml"). - excluded_paths: A list of file paths or directories to exclude from the sitemap. - show_progress: Boolean to print mapping progress. - allowed_extensions: List of allowed file extensions (default: [".html", ".htm"]). - default_priority: Default priority for paths (default: 0.80). - homepage_priority: Priority for the root directory (default: 1.00). - priority_mapping: Dict or callable to map specific paths to priorities. - max_urls: Max URLs per single sitemap file before splitting (default: 50000). - """ - if excluded_paths is None: - excluded_paths = [] + root_path = _as_path_string(root_path) + if not os.path.isdir(root_path): + raise NotADirectoryError( + "root_path must point to an existing directory, got %r" % (root_path,) + ) + + base_url = _normalise_base_url(base_url) + + max_urls = int(max_urls) + if max_urls < 1: + raise ValueError("max_urls must be a positive integer, got %r" % (max_urls,)) + if max_urls > MAX_URLS_PER_SITEMAP: + warnings.warn( + "max_urls=%d exceeds the sitemap protocol limit of %d; " + "the limit will be used instead." % (max_urls, MAX_URLS_PER_SITEMAP), + RuntimeWarning, + stacklevel=2, + ) + max_urls = MAX_URLS_PER_SITEMAP if allowed_extensions is None: - allowed_extensions = [".html", ".htm"] + extensions = list(DEFAULT_EXTENSIONS) else: - # Normalize allowed_extensions to start with "." - allowed_extensions = [ext if ext.startswith(".") else f".{ext}" for ext in allowed_extensions] - - entries = [] - - for root, dirs, files in os.walk(root_path): - # Optimize by pruning excluded directories in-place - dirs[:] = [d for d in dirs if not is_excluded(os.path.join(root, d), root_path, excluded_paths)] - - # Add directories to entries - for dir_name in dirs: - full_dir_path = os.path.join(root, dir_name) - if show_progress: - print(f"Mapping directory: {full_dir_path}") - - relative_dir_path = os.path.relpath(full_dir_path, root_path) - base = base_url.rstrip("/") - url = (base + "/" + relative_dir_path.replace("\\", "/")).rstrip("/") + "/" - - lastmod = get_utc_lastmod(full_dir_path) - priority = get_priority(relative_dir_path, is_dir=True, default_priority=default_priority, priority_mapping=priority_mapping) - - entries.append({ - "loc": url, - "lastmod": lastmod, - "priority": priority - }) - - for file in files: - if file.endswith(tuple(allowed_extensions)): - full_path = os.path.join(root, file) - - if is_excluded(full_path, root_path, excluded_paths): - continue - - if show_progress: - print(f"Found file: {full_path}") - - relative_path = os.path.relpath(full_path, root_path) - base = base_url.rstrip("/") - url = base + "/" + relative_path.replace("\\", "/") - - lastmod = get_utc_lastmod(full_path) - priority = get_priority(relative_path, is_dir=False, default_priority=default_priority, priority_mapping=priority_mapping) + if isinstance(allowed_extensions, str): + allowed_extensions = [ + item.strip() for item in allowed_extensions.split(",") if item.strip() + ] + extensions = [ext for ext in (_normalise_extension(e) for e in allowed_extensions) if ext] + if not extensions: + raise ValueError("allowed_extensions must contain at least one extension") + + if isinstance(priority_mapping, dict): + priority_mapping = _normalise_priority_mapping(priority_mapping) + + path_filter = _PathFilter(root_path, excluded_paths, exclude_hidden=exclude_hidden) + + entries = _walk_entries( + root_path=root_path, + base_url=base_url, + path_filter=path_filter, + extensions=extensions, + priority_mapping=priority_mapping, + default_priority=default_priority, + show_progress=show_progress, + ) + + if include_homepage: + homepage = { + "loc": base_url, + "lastmod": get_utc_lastmod(root_path), + "priority": _format_priority(homepage_priority), + } + if any(entry["loc"] == homepage["loc"] for entry in entries): + for entry in entries: + if entry["loc"] == homepage["loc"]: + entry.update(homepage) + break + else: + entries.append(homepage) - entries.append({ - "loc": url, - "lastmod": lastmod, - "priority": priority - }) + if sort_entries: + entries.sort(key=lambda entry: entry["loc"]) - # Add homepage to entries - homepage_url = base_url - homepage_lastmod = get_utc_lastmod(root_path) - entries.append({ - "loc": homepage_url, - "lastmod": homepage_lastmod, - "priority": f"{float(homepage_priority):.2f}" - }) + if show_progress: + print("Total pages mapped: %d" % len(entries)) - # Write the XML output if len(entries) <= max_urls: if show_progress: - print(f"Total pages mapped: {len(entries)}") - print(f"Saving sitemap to {output_file}...") + print("Saving sitemap to %s..." % output_file) write_sitemap_file(output_file, entries) - else: - if show_progress: - print(f"Total pages mapped: {len(entries)}. Limit is {max_urls}. Splitting into multiple sitemaps...") - - dir_name = os.path.dirname(output_file) - base_name = os.path.basename(output_file) - name, ext = os.path.splitext(base_name) + return [_as_path_string(output_file)] - sitemaps_info = [] - - # Split into chunks of size max_urls - for i in range(0, len(entries), max_urls): - chunk = entries[i:i + max_urls] - part_num = (i // max_urls) + 1 - part_filename = f"{name}_{part_num}{ext}" - part_file_path = os.path.join(dir_name, part_filename) if dir_name else part_filename - - if show_progress: - print(f"Saving split sitemap part {part_num} to {part_file_path}...") - - write_sitemap_file(part_file_path, chunk) - - base = base_url.rstrip("/") - part_url = f"{base}/{part_filename}" - part_lastmod = get_utc_lastmod(part_file_path) - - sitemaps_info.append({ - "loc": part_url, - "lastmod": part_lastmod - }) - - if show_progress: - print(f"Saving sitemap index to {output_file}...") - write_sitemap_index_file(output_file, sitemaps_info) + if show_progress: + print("URL limit is %d. Splitting into multiple sitemaps..." % max_urls) + return _split_sitemaps(entries, output_file, base_url, max_urls, show_progress) diff --git a/localsitemap/__main__.py b/localsitemap/__main__.py new file mode 100644 index 0000000..63321a6 --- /dev/null +++ b/localsitemap/__main__.py @@ -0,0 +1,8 @@ +"""Allow ``python -m localsitemap`` to behave like the ``localsitemap`` CLI.""" + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/localsitemap/_version.py b/localsitemap/_version.py new file mode 100644 index 0000000..dd446de --- /dev/null +++ b/localsitemap/_version.py @@ -0,0 +1,7 @@ +"""Single source of truth for the LocalSiteMap version. + +The release workflow rewrites this file from the pushed git tag, so the +version published to PyPI always matches the tag. +""" + +__version__ = "2.1.0" diff --git a/localsitemap/cli.py b/localsitemap/cli.py new file mode 100644 index 0000000..64292a1 --- /dev/null +++ b/localsitemap/cli.py @@ -0,0 +1,135 @@ +"""Command line interface for LocalSiteMap. + +Usage:: + + localsitemap path/to/website https://example.com -o public/sitemap.xml + python -m localsitemap path/to/website https://example.com --exclude auth +""" + +import argparse +import json +import sys + +from . import __version__, generate_sitemap + + +def _comma_separated(value): + if value is None: + return None + return [item.strip() for item in value.split(",") if item.strip()] + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="localsitemap", + description=( + "Generate a sitemap (and sitemap index when needed) from a local " + "directory that mirrors a website." + ), + ) + parser.add_argument("root_path", help="root directory of the website") + parser.add_argument( + "base_url", + help="public base URL of the website, e.g. https://example.com", + ) + parser.add_argument( + "-o", "--output", default="sitemap.xml", + help="sitemap file to write (default: sitemap.xml)", + ) + parser.add_argument( + "-e", "--exclude", action="append", default=[], metavar="PATTERN", + help=( + "name, relative/absolute path or glob pattern to exclude; " + "repeat for multiple exclusions" + ), + ) + parser.add_argument( + "--extensions", default=".html,.htm", metavar="LIST", + help="comma separated extensions to include (default: .html,.htm)", + ) + parser.add_argument( + "--default-priority", type=float, default=0.80, + help="priority used when no mapping matches (default: 0.80)", + ) + parser.add_argument( + "--homepage-priority", type=float, default=1.00, + help="priority for the site root (default: 1.00)", + ) + parser.add_argument( + "--max-urls", type=int, default=50000, + help="maximum URLs per sitemap file before splitting (default: 50000)", + ) + parser.add_argument( + "--priority-map", metavar="FILE", + help="JSON file containing a priority mapping object", + ) + parser.add_argument( + "--exclude-hidden", action="store_true", + help="skip files and folders whose name starts with a dot", + ) + parser.add_argument( + "--no-homepage", action="store_true", + help="do not emit the site root URL", + ) + parser.add_argument( + "--no-sort", action="store_true", + help="keep filesystem order instead of sorting URLs", + ) + parser.add_argument( + "-p", "--progress", action="store_true", + help="print the traversal progress", + ) + parser.add_argument( + "--version", action="version", version="%(prog)s " + __version__, + ) + return parser + + +def _load_priority_map(parser, path): + try: + with open(path, "r", encoding="utf-8") as handle: + mapping = json.load(handle) + except OSError as error: + parser.error("could not read priority map %s: %s" % (path, error)) + except ValueError as error: + parser.error("priority map %s is not valid JSON: %s" % (path, error)) + if not isinstance(mapping, dict): + parser.error("priority map %s must contain a JSON object" % path) + return mapping + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + + priority_mapping = None + if args.priority_map: + priority_mapping = _load_priority_map(parser, args.priority_map) + + try: + created = generate_sitemap( + root_path=args.root_path, + base_url=args.base_url, + output_file=args.output, + excluded_paths=args.exclude, + show_progress=args.progress, + allowed_extensions=_comma_separated(args.extensions), + default_priority=args.default_priority, + homepage_priority=args.homepage_priority, + priority_mapping=priority_mapping, + max_urls=args.max_urls, + exclude_hidden=args.exclude_hidden, + include_homepage=not args.no_homepage, + sort_entries=not args.no_sort, + ) + except (OSError, ValueError) as error: + print("error: %s" % error, file=sys.stderr) + return 1 + + for path in created: + print("Wrote %s" % path) + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/test.py b/test.py index 34ecb6b..4ee6a48 100644 --- a/test.py +++ b/test.py @@ -1,14 +1,30 @@ +"""Small manual demo of LocalSiteMap. + +Point ``root_directory`` at any local website checkout and run: + + python test.py + +For automated checks use ``python -m unittest -v test_localsitemap`` instead. +""" + from localsitemap import generate_sitemap -# Example usage +# Root site directory root_directory = r"path/to/your/website/directory" # Domain of your website (where it is hosted) base_url_of_your_website = "https://example.com" -# List of file paths or directories to exclude from the sitemap -excluded = ["auth", "forms", "template.html", "media", ".git", ".vscode", "node_modules"] # Example exclusions +# Only exact folder names, relative paths, absolute paths or glob patterns +# will be skipped +excluded = ["auth", "forms", "template.html", "media", ".git", ".vscode", "node_modules"] # Generate the sitemap -generate_sitemap(root_directory, base_url_of_your_website, "sitemap.xml", excluded) -print("Sitemap generated in sitemap.xml") \ No newline at end of file +written = generate_sitemap( + root_path=root_directory, + base_url=base_url_of_your_website, + output_file="sitemap.xml", + excluded_paths=excluded, + show_progress=True, +) +print("Sitemap generated:", ", ".join(written)) diff --git a/test_localsitemap.py b/test_localsitemap.py index db2521e..46d1b01 100644 --- a/test_localsitemap.py +++ b/test_localsitemap.py @@ -1,9 +1,23 @@ +import json import os import shutil +import subprocess +import sys import tempfile import unittest import xml.etree.ElementTree as ET -from localsitemap import generate_sitemap + +from localsitemap import ( + __version__, + generate_sitemap, + get_priority, + is_excluded, + write_sitemap_file, +) + +SITEMAP_NS = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} +REPO_ROOT = os.path.dirname(os.path.abspath(__file__)) + class TestLocalSitemap(unittest.TestCase): def setUp(self): @@ -31,46 +45,58 @@ def setUp(self): self.output_file = os.path.join(self.test_dir, "sitemap.xml") def tearDown(self): - shutil.rmtree(self.test_dir) + shutil.rmtree(self.test_dir, ignore_errors=True) - def write_dummy_file(self, path): + # -- helpers --------------------------------------------------------- + + def write_dummy_file(self, path, content="Test"): + directory = os.path.dirname(path) + if directory and not os.path.isdir(directory): + os.makedirs(directory) with open(path, "w", encoding="utf-8") as f: - f.write("Test") + f.write(content) + + def urls(self, path=None): + tree = ET.parse(path or self.output_file) + return [loc.text for loc in tree.getroot().findall("sm:url/sm:loc", SITEMAP_NS)] + + def entries(self, path=None): + tree = ET.parse(path or self.output_file) + result = {} + for url_node in tree.getroot().findall("sm:url", SITEMAP_NS): + loc = url_node.find("sm:loc", SITEMAP_NS).text + priority_node = url_node.find("sm:priority", SITEMAP_NS) + lastmod_node = url_node.find("sm:lastmod", SITEMAP_NS) + result[loc] = { + "priority": priority_node.text if priority_node is not None else None, + "lastmod": lastmod_node.text if lastmod_node is not None else None, + } + return result + + # -- existing behaviour ---------------------------------------------- def test_basic_sitemap_generation(self): - # Generate with defaults generate_sitemap( root_path=self.test_dir, base_url="https://example.com", output_file=self.output_file ) - # Parse XML and verify entries - tree = ET.parse(self.output_file) - root = tree.getroot() - - # Standard sitemap namespace - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - urls = [loc.text for loc in root.findall("sm:url/sm:loc", ns)] - - # We expect root directory, subdirectories, html/htm files - # Directories included: about, blog, auth, authority, blog/posts - # Files included: index.html, about/index.html, blog/first-post.html, blog/second-post.htm, blog/posts/post1.html, auth/login.html, authority/page.html - # Plus the homepage itself + urls = self.urls() self.assertIn("https://example.com", urls) self.assertIn("https://example.com/about/", urls) self.assertIn("https://example.com/about/index.html", urls) self.assertIn("https://example.com/blog/second-post.htm", urls) - # Custom.php and custom.txt should NOT be included by default + # custom.php and custom.txt should NOT be included by default self.assertNotIn("https://example.com/custom.php", urls) self.assertNotIn("https://example.com/custom.txt", urls) + # No duplicate entries are allowed in a sitemap + self.assertEqual(len(urls), len(set(urls))) + def test_exclusion_exact_match(self): - # Exclude "auth" only. - # "authority" and "authority/page.html" should NOT be excluded. - # "auth" directory and "auth/login.html" SHOULD be excluded. generate_sitemap( root_path=self.test_dir, base_url="https://example.com", @@ -78,21 +104,15 @@ def test_exclusion_exact_match(self): excluded_paths=["auth"] ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - urls = [loc.text for loc in root.findall("sm:url/sm:loc", ns)] + urls = self.urls() - # "auth" segments should be excluded self.assertNotIn("https://example.com/auth/", urls) self.assertNotIn("https://example.com/auth/login.html", urls) - # "authority" segments should remain self.assertIn("https://example.com/authority/", urls) self.assertIn("https://example.com/authority/page.html", urls) def test_custom_extensions(self): - # Generate with custom extensions [".php", "txt"] generate_sitemap( root_path=self.test_dir, base_url="https://example.com", @@ -100,18 +120,13 @@ def test_custom_extensions(self): allowed_extensions=[".php", "txt"] ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - urls = [loc.text for loc in root.findall("sm:url/sm:loc", ns)] + urls = self.urls() self.assertIn("https://example.com/custom.php", urls) self.assertIn("https://example.com/custom.txt", urls) - # HTML files should not be present now self.assertNotIn("https://example.com/index.html", urls) def test_custom_priority_and_mapping(self): - # Test default values and priority mapping as dict pm = { "about/index.html": 0.95, "blog/second-post.htm": 0.40, @@ -127,21 +142,13 @@ def test_custom_priority_and_mapping(self): priority_mapping=pm ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - - entries = {} - for url_node in root.findall("sm:url", ns): - loc = url_node.find("sm:loc", ns).text - prio = url_node.find("sm:priority", ns).text - entries[loc] = prio + entries = self.entries() - self.assertEqual(entries["https://example.com"], "0.99") - self.assertEqual(entries["https://example.com/about/index.html"], "0.95") - self.assertEqual(entries["https://example.com/blog/second-post.htm"], "0.40") - self.assertEqual(entries["https://example.com/blog/posts/"], "0.15") - self.assertEqual(entries["https://example.com/index.html"], "0.55") + self.assertEqual(entries["https://example.com"]["priority"], "0.99") + self.assertEqual(entries["https://example.com/about/index.html"]["priority"], "0.95") + self.assertEqual(entries["https://example.com/blog/second-post.htm"]["priority"], "0.40") + self.assertEqual(entries["https://example.com/blog/posts/"]["priority"], "0.15") + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.55") def test_custom_priority_callable(self): def my_priority_mapper(relative_path, is_dir): @@ -159,24 +166,13 @@ def my_priority_mapper(relative_path, is_dir): priority_mapping=my_priority_mapper ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} + entries = self.entries() - entries = {} - for url_node in root.findall("sm:url", ns): - loc = url_node.find("sm:loc", ns).text - prio = url_node.find("sm:priority", ns).text - entries[loc] = prio - - self.assertEqual(entries["https://example.com/about/"], "0.11") - self.assertEqual(entries["https://example.com/blog/first-post.html"], "0.99") - self.assertEqual(entries["https://example.com/index.html"], "0.75") + self.assertEqual(entries["https://example.com/about/"]["priority"], "0.11") + self.assertEqual(entries["https://example.com/blog/first-post.html"]["priority"], "0.99") + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.75") def test_sitemap_splitting_and_index(self): - # Set max_urls to 3. Since there are more than 3 entries total, - # it should generate split sitemaps (sitemap_1.xml, sitemap_2.xml, etc) - # and a sitemap.xml index. generate_sitemap( root_path=self.test_dir, base_url="https://example.com", @@ -184,51 +180,461 @@ def test_sitemap_splitting_and_index(self): max_urls=3 ) - # Verify index sitemap.xml is created self.assertTrue(os.path.exists(self.output_file)) - tree = ET.parse(self.output_file) - root = tree.getroot() - - # It should be a sitemapindex root + root = ET.parse(self.output_file).getroot() self.assertEqual(root.tag, "{http://www.sitemaps.org/schemas/sitemap/0.9}sitemapindex") - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - sitemap_locs = [loc.text for loc in root.findall("sm:sitemap/sm:loc", ns)] + sitemap_locs = [loc.text for loc in root.findall("sm:sitemap/sm:loc", SITEMAP_NS)] self.assertGreater(len(sitemap_locs), 1) self.assertIn("https://example.com/sitemap_1.xml", sitemap_locs) - # Check that part 1 contains url entries part1_path = os.path.join(self.test_dir, "sitemap_1.xml") self.assertTrue(os.path.exists(part1_path)) - part_tree = ET.parse(part1_path) - part_root = part_tree.getroot() + part_root = ET.parse(part1_path).getroot() self.assertEqual(part_root.tag, "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset") - part_urls = [loc.text for loc in part_root.findall("sm:url/sm:loc", ns)] + part_urls = [loc.text for loc in part_root.findall("sm:url/sm:loc", SITEMAP_NS)] self.assertEqual(len(part_urls), 3) def test_utc_timezone_format(self): - # Ensure that timestamps end in +00:00 and are in correct ISO 8601 format generate_sitemap( root_path=self.test_dir, base_url="https://example.com", output_file=self.output_file ) - tree = ET.parse(self.output_file) - root = tree.getroot() - ns = {"sm": "http://www.sitemaps.org/schemas/sitemap/0.9"} - - lastmods = [lm.text for lm in root.findall("sm:url/sm:lastmod", ns)] + lastmods = [value["lastmod"] for value in self.entries().values()] for lm in lastmods: self.assertTrue(lm.endswith("+00:00")) - # Format: YYYY-MM-DDThh:mm:ss+00:00 self.assertEqual(len(lm), 25) self.assertEqual(lm[10], 'T') self.assertEqual(lm[19], '+') + # -- bug fixes / new behaviour --------------------------------------- + + def test_exclusion_does_not_match_parent_directories(self): + """A site checked out below a folder named like an exclusion still works.""" + parent = tempfile.mkdtemp() + try: + site = os.path.join(parent, "media", "website") + os.makedirs(site) + self.write_dummy_file(os.path.join(site, "index.html")) + output = os.path.join(site, "sitemap.xml") + + generate_sitemap( + root_path=site, + base_url="https://example.com", + output_file=output, + excluded_paths=["media"], + ) + + urls = self.urls(output) + self.assertIn("https://example.com/index.html", urls) + finally: + shutil.rmtree(parent, ignore_errors=True) + + def test_exclusion_of_nested_relative_path(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=[os.path.join("blog", "posts")], + ) + + urls = self.urls() + self.assertNotIn("https://example.com/blog/posts/", urls) + self.assertNotIn("https://example.com/blog/posts/post1.html", urls) + self.assertIn("https://example.com/blog/", urls) + + def test_exclusion_of_file_name(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=["second-post.htm"], + ) + self.assertNotIn("https://example.com/blog/second-post.htm", self.urls()) + + def test_absolute_exclusion_path(self): + excluded_dir = os.path.join(self.test_dir, "blog", "posts") + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=[excluded_dir], + ) + self.assertNotIn("https://example.com/blog/posts/post1.html", self.urls()) + + def test_glob_exclusions(self): + self.write_dummy_file(os.path.join(self.test_dir, "draft-page.html")) + self.write_dummy_file(os.path.join(self.test_dir, "blog", "draft-note.html")) + self.write_dummy_file(os.path.join(self.test_dir, "notes.tmp")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths=["draft-*", "*.tmp"], + allowed_extensions=[".html", ".tmp"], + ) + + urls = self.urls() + self.assertNotIn("https://example.com/draft-page.html", urls) + self.assertNotIn("https://example.com/blog/draft-note.html", urls) + self.assertNotIn("https://example.com/notes.tmp", urls) + self.assertIn("https://example.com/index.html", urls) + + def test_is_excluded_helper(self): + self.assertTrue(is_excluded(os.path.join(self.test_dir, "auth"), self.test_dir, ["auth"])) + self.assertFalse( + is_excluded(os.path.join(self.test_dir, "authority"), self.test_dir, ["auth"]) + ) + self.assertFalse(is_excluded(self.test_dir, self.test_dir, ["website"])) + + def test_extensions_are_case_insensitive(self): + self.write_dummy_file(os.path.join(self.test_dir, "UPPER.HTML")) + self.write_dummy_file(os.path.join(self.test_dir, "upper.PHP")) + self.write_dummy_file(os.path.join(self.test_dir, "page.html.bak")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + allowed_extensions=[".HTML", "PHP"], + ) + + urls = self.urls() + self.assertIn("https://example.com/UPPER.HTML", urls) + self.assertIn("https://example.com/upper.PHP", urls) + self.assertIn("https://example.com/custom.php", urls) + self.assertNotIn("https://example.com/page.html.bak", urls) + + def test_urls_are_percent_encoded(self): + self.write_dummy_file(os.path.join(self.test_dir, "my page.html")) + self.write_dummy_file(os.path.join(self.test_dir, "a&b.html")) + self.write_dummy_file(os.path.join(self.test_dir, "hash#1.html")) + self.write_dummy_file(os.path.join(self.test_dir, "café", "index.html")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + + urls = self.urls() + self.assertIn("https://example.com/my%20page.html", urls) + self.assertIn("https://example.com/a%26b.html", urls) + self.assertIn("https://example.com/hash%231.html", urls) + self.assertIn("https://example.com/caf%C3%A9/", urls) + self.assertIn("https://example.com/caf%C3%A9/index.html", urls) + for url in urls: + self.assertNotIn(" ", url) + self.assertNotIn("#", url) + + def test_priority_mapping_accepts_folder_and_extension_variants(self): + mapping = { + "blog/": 0.70, # documented trailing slash folder key + "posts\\": 0.20, # Windows separator + ".PHP": 0.35, # upper case extension + "about/index.html": 0.90, + } + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + priority_mapping=mapping, + allowed_extensions=[".html", ".php"], + ) + + entries = self.entries() + self.assertEqual(entries["https://example.com/blog/"]["priority"], "0.70") + self.assertEqual(entries["https://example.com/blog/posts/"]["priority"], "0.20") + self.assertEqual(entries["https://example.com/custom.php"]["priority"], "0.35") + self.assertEqual(entries["https://example.com/about/index.html"]["priority"], "0.90") + + def test_priorities_are_clamped_to_the_protocol_range(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + default_priority=0.5, + priority_mapping={"about/index.html": 5, "index.html": -2}, + ) + entries = self.entries() + self.assertEqual(entries["https://example.com/about/index.html"]["priority"], "1.00") + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.00") + + def test_get_priority_public_helper(self): + self.assertEqual( + get_priority("blog/posts", True, 0.5, {"posts": 0.3}), "0.30" + ) + self.assertEqual(get_priority("blog/posts", True, 0.5), "0.50") + self.assertEqual( + get_priority("blog/posts", True, 0.5, lambda rel, is_dir: 0.25), "0.25" + ) + + def test_missing_root_raises(self): + with self.assertRaises(NotADirectoryError): + generate_sitemap( + os.path.join(self.test_dir, "does-not-exist"), + "https://example.com", + self.output_file, + ) + + def test_invalid_base_url_raises(self): + with self.assertRaises(ValueError): + generate_sitemap(self.test_dir, "example.com", self.output_file) + + def test_invalid_max_urls_raises(self): + for value in (0, -5): + with self.assertRaises(ValueError): + generate_sitemap( + self.test_dir, "https://example.com", self.output_file, max_urls=value + ) + + def test_max_urls_above_protocol_limit_warns(self): + with self.assertWarns(RuntimeWarning): + generate_sitemap( + self.test_dir, "https://example.com", self.output_file, max_urls=60000 + ) + self.assertEqual( + ET.parse(self.output_file).getroot().tag, + "{http://www.sitemaps.org/schemas/sitemap/0.9}urlset", + ) + + def test_output_directory_is_created(self): + nested = os.path.join(self.test_dir, "public", "seo", "sitemap.xml") + created = generate_sitemap(self.test_dir, "https://example.com", nested) + self.assertTrue(os.path.exists(nested)) + self.assertEqual(created, [nested]) + + def test_string_arguments_are_accepted(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + excluded_paths="auth", + allowed_extensions="html,php", + ) + urls = self.urls() + self.assertNotIn("https://example.com/auth/login.html", urls) + self.assertIn("https://example.com/custom.php", urls) + self.assertIn("https://example.com/index.html", urls) + + def test_exclude_hidden(self): + self.write_dummy_file(os.path.join(self.test_dir, ".hidden.html")) + self.write_dummy_file(os.path.join(self.test_dir, ".git", "config.html")) + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + exclude_hidden=True, + ) + + urls = self.urls() + for url in urls: + self.assertNotIn("/.", url) + self.assertIn("https://example.com/index.html", urls) + + def test_include_homepage_false(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com/", + output_file=self.output_file, + include_homepage=False, + ) + self.assertNotIn("https://example.com", self.urls()) + + def test_base_url_trailing_slash_is_normalised(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com/", + output_file=self.output_file, + ) + urls = self.urls() + self.assertIn("https://example.com", urls) + self.assertIn("https://example.com/about/", urls) + self.assertEqual(len(urls), len(set(urls))) + + def test_output_is_deterministic(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + with open(self.output_file, "rb") as handle: + first = handle.read() + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + with open(self.output_file, "rb") as handle: + second = handle.read() + + self.assertEqual(first, second) + + def test_output_is_pretty_and_parsable(self): + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + ) + with open(self.output_file, "r", encoding="utf-8") as handle: + content = handle.read() + + self.assertTrue(content.startswith('')) + self.assertNotIn("\n\n", content) + self.assertIn('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"', content) + ET.fromstring(content) # must be well-formed + + def collect_urls(self, paths): + collected = [] + for path in paths: + collected.extend(self.urls(path)) + return collected + + def reference_urls(self): + """Generate an unsplit sitemap elsewhere and return its URLs.""" + reference_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, reference_dir, ignore_errors=True) + reference = os.path.join(reference_dir, "sitemap.xml") + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=reference, + ) + return set(self.urls(reference)) + + def test_split_covers_every_url_exactly_once(self): + created = generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + max_urls=2, + ) + + self.assertEqual(created[-1], self.output_file) + parts = [path for path in created if path != self.output_file] + self.assertGreater(len(parts), 1) + self.assertTrue(all(os.path.exists(part) for part in parts)) + + collected = self.collect_urls(parts) + self.assertEqual(len(collected), len(set(collected))) + self.assertEqual(set(collected), self.reference_urls()) + + def test_split_index_lists_every_part(self): + created = generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + max_urls=2, + ) + parts = [path for path in created if path != self.output_file] + + root = ET.parse(self.output_file).getroot() + index_locs = [loc.text for loc in root.findall("sm:sitemap/sm:loc", SITEMAP_NS)] + + expected = ["https://example.com/" + os.path.basename(part) for part in parts] + self.assertEqual(index_locs, expected) + self.assertEqual(len(index_locs), len(parts)) + + def test_stale_split_parts_are_removed(self): + stale = os.path.join(self.test_dir, "sitemap_9.xml") + self.write_dummy_file(stale, content="") + + generate_sitemap( + root_path=self.test_dir, + base_url="https://example.com", + output_file=self.output_file, + max_urls=3, + ) + + self.assertFalse(os.path.exists(stale)) + self.assertTrue(os.path.exists(os.path.join(self.test_dir, "sitemap_1.xml"))) + + def test_write_sitemap_file_helper(self): + target = os.path.join(self.test_dir, "manual.xml") + write_sitemap_file( + target, + [ + { + "loc": "https://example.com/a?b=1&c=2", + "lastmod": "2024-01-01T00:00:00+00:00", + "priority": "0.50", + } + ], + ) + urls = self.urls(target) + self.assertEqual(urls, ["https://example.com/a?b=1&c=2"]) + + def test_version_is_exposed(self): + self.assertRegex(__version__, r"^\d+\.\d+\.\d+") + + # -- CLI ------------------------------------------------------------- + + def run_cli(self, *args): + environment = os.environ.copy() + environment["PYTHONPATH"] = os.pathsep.join( + [REPO_ROOT, environment.get("PYTHONPATH", "")] + ).strip(os.pathsep) + return subprocess.run( + [sys.executable, "-m", "localsitemap"] + list(args), + cwd=self.test_dir, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + + def test_cli_generates_sitemap(self): + result = self.run_cli( + self.test_dir, + "https://example.com", + "-o", + "cli-sitemap.xml", + "--exclude", + "auth", + ) + self.assertEqual(result.returncode, 0, result.stderr) + target = os.path.join(self.test_dir, "cli-sitemap.xml") + self.assertTrue(os.path.exists(target)) + urls = self.urls(target) + self.assertIn("https://example.com/blog/first-post.html", urls) + self.assertNotIn("https://example.com/auth/login.html", urls) + + def test_cli_priority_map_and_version(self): + mapping_path = os.path.join(self.test_dir, "priorities.json") + with open(mapping_path, "w", encoding="utf-8") as handle: + json.dump({".html": 0.42}, handle) + + result = self.run_cli( + self.test_dir, + "https://example.com", + "-o", + "cli-sitemap.xml", + "--priority-map", + mapping_path, + ) + self.assertEqual(result.returncode, 0, result.stderr) + entries = self.entries(os.path.join(self.test_dir, "cli-sitemap.xml")) + self.assertEqual(entries["https://example.com/index.html"]["priority"], "0.42") + + version_result = self.run_cli("--version") + self.assertEqual(version_result.returncode, 0) + self.assertIn(__version__, version_result.stdout) + + def test_cli_reports_errors(self): + result = self.run_cli("missing-directory", "https://example.com") + self.assertEqual(result.returncode, 1) + self.assertIn("error:", result.stderr) + + if __name__ == "__main__": unittest.main() From eb8d490cc9fdb04009863f7bf41edd3f287200b0 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:46:05 +0000 Subject: [PATCH 2/6] Move packaging to pyproject.toml and bump to 2.1.0 Metadata now lives in pyproject.toml (PEP 621 + PEP 639 license expression) and the version is read from localsitemap/_version.py, so the release workflow no longer has to sed setup.py. setup.py is kept as a compatibility shim. Python 3.9+ is required to build; 3.6-3.8 remain served by 2.0.0. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .gitignore | 17 ++++++++++--- MANIFEST.in | 6 +++++ pyproject.toml | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 35 ++++++-------------------- 4 files changed, 92 insertions(+), 32 deletions(-) create mode 100644 MANIFEST.in create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 2f92536..47c03d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,15 @@ -__pycache__ -*.pyc -dist/ +__pycache__/ +*.py[cod] +*.egg-info/ build/ +dist/ venv/ -.venv/ \ No newline at end of file +.venv/ +.tox/ +.nox/ +.pytest_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +.ruff_cache/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..60bbc8c --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include CONTRIBUTING.md +include LICENSE +include README.md +include test.py +include test_localsitemap.py +recursive-include .github/scripts *.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b3c59f5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["setuptools>=77.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "localsitemap" +dynamic = ["version"] +description = "Generate sitemaps and sitemap indexes from local files by crawling directories recursively." +readme = "README.md" +requires-python = ">=3.9" +license = "MIT" +license-files = ["LICENSE"] +authors = [ + { name = "Infinitode Pty Ltd", email = "infinitode.ltd@gmail.com" }, +] +keywords = [ + "sitemap", + "sitemap-index", + "seo", + "static-site", + "xml", + "crawler", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Internet :: WWW/HTTP :: Site Management", + "Topic :: Text Processing :: Markup :: XML", +] +dependencies = [] + +[project.urls] +Homepage = "https://github.com/Infinitode/LocalSiteMap" +Repository = "https://github.com/Infinitode/LocalSiteMap" +Issues = "https://github.com/Infinitode/LocalSiteMap/issues" +Changelog = "https://github.com/Infinitode/LocalSiteMap/releases" +PyPI = "https://pypi.org/project/localsitemap/" + +[project.scripts] +localsitemap = "localsitemap.cli:main" + +[tool.setuptools.dynamic] +version = { attr = "localsitemap._version.__version__" } + +[tool.setuptools.packages.find] +include = ["localsitemap*"] +namespaces = false + +[tool.ruff] +line-length = 100 +target-version = "py39" +exclude = ["build", "dist"] + +[tool.ruff.lint] +select = ["E", "F", "W", "B"] +ignore = ["B905"] # zip(strict=...) requires Python 3.10 diff --git a/setup.py b/setup.py index f73a221..a8a5213 100644 --- a/setup.py +++ b/setup.py @@ -1,30 +1,9 @@ -from setuptools import setup, find_packages +"""Compatibility shim. -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() +All project metadata lives in ``pyproject.toml``; this file only exists so +that legacy ``python setup.py ...`` invocations keep working. +""" -setup( - name='localsitemap', - version='{{VERSION_PLACEHOLDER}}', - author='Infinitode Pty Ltd', - author_email='infinitode.ltd@gmail.com', - description="LocalSiteMap is an open-source Python package designed for generating sitemaps and sitemap indexes from local files by crawling directories recursively.", - long_description=long_description, - long_description_content_type='text/markdown', - url='https://github.com/infinitode/localsitemap', - packages=find_packages(), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - 'Programming Language :: Python :: 3.13', - ], - python_requires='>=3.6', -) +from setuptools import setup + +setup() From fb4e0196d6a7c63ada04fdff455abd043e807009 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:46:08 +0000 Subject: [PATCH 3/6] Publish with PyPI Trusted Publishing and PEP 740 attestations - build job: run the tests, derive the version from the tag, build and twine-check the distributions, then upload them as artifacts - publish job: no API token and no 'environment:' key, only 'id-token: write', using pypa/gh-action-pypi-publish with attestations - new verify-provenance job queries the PyPI Integrity API and fails the release if any published file lacks provenance - add a tests workflow (lint, 3.9-3.14 matrix on Linux plus Windows, and a build/install smoke test) Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/scripts/verify_provenance.py | 142 ++++++++++++++++++++++ .github/workflows/build_and_publish.yml | 153 ++++++++++++++++++++---- .github/workflows/tests.yml | 88 ++++++++++++++ 3 files changed, 358 insertions(+), 25 deletions(-) create mode 100644 .github/scripts/verify_provenance.py create mode 100644 .github/workflows/tests.yml diff --git a/.github/scripts/verify_provenance.py b/.github/scripts/verify_provenance.py new file mode 100644 index 0000000..c768517 --- /dev/null +++ b/.github/scripts/verify_provenance.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Verify that every file of a PyPI release carries PEP 740 provenance. + +The script talks to the public PyPI Integrity API: + + https://pypi.org/integrity/{project}/{version}/{filename}/provenance + +For each distribution file of the given release it checks that at least one +attestation bundle is present and prints the trusted publisher identity the +attestations are bound to. This is what makes the "provenance" indicator on +the PyPI project page show up, so the release workflow runs it after every +upload. + +Exit codes: + 0 - every file is attested (or --allow-missing was given) + 1 - at least one file has no provenance + 2 - the release could not be queried at all + +Usage: + python .github/scripts/verify_provenance.py --project localsitemap --version 2.1.0 +""" + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request + +PYPI_JSON = "https://pypi.org/pypi/{project}/{version}/json" +PYPI_INTEGRITY = "https://pypi.org/integrity/{project}/{version}/{filename}/provenance" +USER_AGENT = "localsitemap-release-check/1.0" + + +def _get_json(url, timeout=30): + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def _release_files(project, version): + data = _get_json(PYPI_JSON.format(project=project, version=version)) + return [item["filename"] for item in data.get("urls", [])] + + +def _provenance(project, version, filename): + url = PYPI_INTEGRITY.format(project=project, version=version, filename=filename) + try: + return _get_json(url) + except urllib.error.HTTPError as error: + if error.code == 404: + return None + raise + + +def _describe(bundles): + descriptions = [] + for bundle in bundles: + publisher = bundle.get("publisher") or {} + parts = [ + str(publisher.get("kind", "unknown")), + str(publisher.get("repository", "")), + str(publisher.get("workflow", "")), + ] + environment = publisher.get("environment") + if environment: + parts.append("env=%s" % environment) + descriptions.append(" ".join(part for part in parts if part)) + return "; ".join(descriptions) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", default="localsitemap") + parser.add_argument("--version", required=True) + parser.add_argument( + "--allow-missing", + action="store_true", + help="do not fail when a file has no provenance", + ) + parser.add_argument( + "--attempts", type=int, default=5, + help="how many times to retry while PyPI propagates a new release", + ) + parser.add_argument("--delay", type=float, default=6.0, help="seconds between retries") + args = parser.parse_args(argv) + + filenames = [] + for attempt in range(1, max(1, args.attempts) + 1): + try: + filenames = _release_files(args.project, args.version) + if filenames: + break + except (urllib.error.URLError, ValueError) as error: + print("could not query PyPI (%s), attempt %d" % (error, attempt)) + if attempt < args.attempts: + time.sleep(args.delay) + + if not filenames: + print( + "error: no files found for %s %s on PyPI" % (args.project, args.version), + file=sys.stderr, + ) + return 2 + + missing = [] + for filename in filenames: + data = None + for attempt in range(1, max(1, args.attempts) + 1): + try: + data = _provenance(args.project, args.version, filename) + if data: + break + except (urllib.error.URLError, ValueError) as error: + print(" %s: query failed (%s), attempt %d" % (filename, error, attempt)) + if attempt < args.attempts: + time.sleep(args.delay) + + bundles = (data or {}).get("attestation_bundles") or [] + if bundles: + print("OK %s -> %s" % (filename, _describe(bundles))) + else: + print("MISS %s has no provenance on PyPI" % filename) + missing.append(filename) + + if missing: + if args.allow_missing: + print("warning: %d file(s) without provenance" % len(missing)) + return 0 + print( + "error: %d file(s) were published without provenance: %s" + % (len(missing), ", ".join(missing)), + file=sys.stderr, + ) + return 1 + + print("All %d file(s) are attested with verifiable provenance." % len(filenames)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 4a9ec6b..90731c9 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -1,34 +1,137 @@ name: Publish Python 🐍 distributions 📦 to PyPI +# Publishing uses PyPI Trusted Publishing (OIDC) - no API token is stored in +# this repository. The publisher configured on PyPI must match: +# owner/repository : Infinitode/LocalSiteMap +# workflow : build_and_publish.yml +# environment : (left blank - therefore no `environment:` key below) +# +# Every upload is signed with PEP 740 attestations, which is what powers the +# "provenance" indicator on the PyPI project page. + on: push: tags: - - '*' + - 'v[0-9]*' + - '[0-9]*' + workflow_dispatch: + +permissions: + contents: read jobs: - build-n-publish: - name: Build and publish Python 🐍 distributions 📦 to PyPI + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tooling + run: >- + python -m pip install --upgrade + build + twine + setuptools + wheel + + - name: Run the test suite + run: python -m unittest -v test_localsitemap + + - name: Determine the release version + id: version + run: | + set -euo pipefail + if [[ "${GITHUB_REF}" == refs/tags/* ]]; then + VERSION="${GITHUB_REF_NAME#v}" + else + VERSION="$(python -c 'from localsitemap._version import __version__; print(__version__)')" + fi + python - "${VERSION}" <<'PY' + import pathlib + import re + import sys + + version = sys.argv[1].strip() + pattern = r"^[0-9]+(\.[0-9]+)*((a|b|rc)[0-9]+)?(\.post[0-9]+)?(\.dev[0-9]+)?$" + if not re.match(pattern, version): + sys.exit("Invalid release version: %r" % version) + + target = pathlib.Path("localsitemap/_version.py") + target.write_text( + '"""Single source of truth for the LocalSiteMap version.\n\n' + "The release workflow rewrites this file from the pushed git tag, so the\n" + 'version published to PyPI always matches the tag.\n"""\n\n' + '__version__ = "%s"\n' % version, + encoding="utf-8", + ) + print("Building version", version) + PY + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" + + - name: Build sdist and wheel + run: python -m build + + - name: Check the distribution metadata + run: python -m twine check --strict dist/* + + - name: Upload the distribution artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + + publish: + name: Publish distribution 📦 to PyPI + needs: build + runs-on: ubuntu-latest + # NOTE: intentionally no `environment:` key. The PyPI trusted publisher is + # configured with a blank environment name, and adding one here would make + # the OIDC token claim mismatch and the upload fail. + permissions: + id-token: write # required for Trusted Publishing and attestations + steps: + - name: Download the distribution artifacts + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + attestations: true + print-hash: true + + verify-provenance: + name: Verify published provenance 🛡️ + needs: + - build + - publish runs-on: ubuntu-latest + permissions: + contents: read steps: - - uses: actions/checkout@master - - name: Set up Python 3.12 - uses: actions/setup-python@v3 - with: - python-version: '3.12' - - name: Install pypa/setuptools - run: >- - python -m - pip install setuptools wheel - - name: Extract tag name - id: tag - run: echo ::set-output name=TAG_NAME::$(echo $GITHUB_REF | cut -d / -f 3) - - name: Update version in setup.py - run: >- - sed -i "s/{{VERSION_PLACEHOLDER}}/${{ steps.tag.outputs.TAG_NAME }}/g" setup.py - - name: Build a binary wheel - run: >- - python setup.py sdist bdist_wheel - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@master - with: - password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Confirm every published file carries provenance + env: + VERSION: ${{ needs.build.outputs.version }} + run: >- + python .github/scripts/verify_provenance.py + --project localsitemap + --version "${VERSION}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..11b92ac --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,88 @@ +name: Tests + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install ruff + run: python -m pip install --upgrade ruff + + - name: Run ruff + run: ruff check . + + test: + name: Test on Python ${{ matrix.python-version }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + python-version: + - '3.9' + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + include: + - os: windows-latest + python-version: '3.12' + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Run the test suite + run: python -m unittest -v test_localsitemap + + build: + name: Build the distributions + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tooling + run: python -m pip install --upgrade build twine + + - name: Build sdist and wheel + run: python -m build + + - name: Check the distribution metadata + run: python -m twine check --strict dist/* + + - name: Install the built wheel and smoke test it + run: | + python -m pip install --force-reinstall dist/*.whl + cd "${RUNNER_TEMP}" + python -m localsitemap --version + python -c "import localsitemap; print(localsitemap.__version__)" From 133d2589025aa13f50535fa55b85814b3b716a31 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:46:08 +0000 Subject: [PATCH 4/6] Document the 2.1.0 release, CLI and release provenance Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- CONTRIBUTING.md | 48 ++++++++++++++++-- README.md | 132 ++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 149 insertions(+), 31 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bee6980..e77c1ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,13 +8,19 @@ Thank you for considering contributing to **LocalSiteMap**! Contributions, wheth Create a fork of this repository to your GitHub account and clone it locally. 2. **Set Up the Environment** - Ensure you have Python 3.6+ installed. You should also install compatible versions of the necessary dependencies for the project. + Ensure you have Python 3.9+ installed. LocalSiteMap has no runtime dependencies, so there is nothing else to install for development (a virtual environment is still recommended). 3. **Run Tests** - Before making changes, run tests to confirm that everything works. + Before making changes, run tests to confirm that everything works: + + ```bash + python -m unittest -v test_localsitemap + ``` + + The same suite runs on GitHub Actions for Python 3.9 – 3.14 on Linux, plus Windows. > [!TIP] -> We sometimes include testing scripts in our projects for easy testing. +> We sometimes include testing scripts in our projects for easy testing. `test.py` is a small manual demo you can point at any directory. ## How to Contribute @@ -33,6 +39,38 @@ Thank you for considering contributing to **LocalSiteMap**! Contributions, wheth - Submit a pull request with a detailed explanation of what you've changed or added. - Ensure your PR passes all automated tests and adheres to the contribution guidelines. +## Building Locally + +All project metadata lives in `pyproject.toml`, and the version is read from `localsitemap/_version.py`. + +```bash +python -m pip install --upgrade build twine +python -m build +python -m twine check --strict dist/* +``` + +## Releasing + +1. Update `__version__` in `localsitemap/_version.py` and the changelog in `README.md`. +2. Commit and push, then push a matching tag (`2.1.0` or `v2.1.0`): + + ```bash + git tag 2.1.0 + git push origin 2.1.0 + ``` + +3. The `Publish Python 🐍 distributions 📦 to PyPI` workflow builds the sdist and wheel, runs the tests, and publishes to PyPI using **Trusted Publishing** — no API token or repository secret is required. + + The publisher configured on PyPI must match this repository: + + | Field | Value | + | --- | --- | + | Owner / Repository | `Infinitode/LocalSiteMap` | + | Workflow | `build_and_publish.yml` | + | Environment | *(leave blank — the workflow does not set `environment:`)* | + +4. The workflow signs every uploaded file with a PEP 740 attestation and then runs `.github/scripts/verify_provenance.py`, which fails the release if any published file is missing provenance. That is what makes the provenance badge and the PyPI "provenance" indicator accurate. + ## Community Guidelines To maintain a positive and collaborative community, we ask that all contributors adhere to the following principles: @@ -44,6 +82,6 @@ To maintain a positive and collaborative community, we ask that all contributors Offer helpful and actionable suggestions during code reviews. 3. **Follow Licensing Requirements** - Ensure that any derivative works comply with the [license](https://github.com/infinitode/localsitemap/blob/main/LICENSE). + Ensure that any derivative works comply with the [license](https://github.com/Infinitode/LocalSiteMap/blob/main/LICENSE). -By contributing, you agree to abide by these guidelines and this project's [license](https://github.com/infinitode/localsitemap/blob/main/LICENSE). Thank you for helping make **LocalSiteMap** better! +By contributing, you agree to abide by these guidelines and this project's [license](https://github.com/Infinitode/LocalSiteMap/blob/main/LICENSE). Thank you for helping make **LocalSiteMap** better! diff --git a/README.md b/README.md index 33c7e1c..b61dd7e 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,44 @@ # LocalSiteMap -![Python Version](https://img.shields.io/badge/python-3.13-blue.svg) -[![Code Size](https://img.shields.io/github/languages/code-size/infinitode/localsitemap)](https://github.com/infinitode/localsitemap) -![Downloads](https://pepy.tech/badge/localsitemap) -![License Compliance](https://img.shields.io/badge/license-compliance-brightgreen.svg) -![PyPI Version](https://img.shields.io/pypi/v/localsitemap) -LocalSiteMap is an open-source Python package designed for generating sitemaps and sitemap indexes from local files. It crawls your root website directory and automatically builds standard-compliant sitemaps. +[![PyPI Version](https://img.shields.io/pypi/v/localsitemap)](https://pypi.org/project/localsitemap/) +[![PyPI - Python Versions](https://img.shields.io/pypi/pyversions/localsitemap)](https://pypi.org/project/localsitemap/) +[![PyPI provenance](https://img.shields.io/badge/PyPI%20provenance-attested-blueviolet?logo=pypi&logoColor=white)](https://pypi.org/project/localsitemap/#files) +[![Tests](https://github.com/Infinitode/LocalSiteMap/actions/workflows/tests.yml/badge.svg)](https://github.com/Infinitode/LocalSiteMap/actions/workflows/tests.yml) +[![Downloads](https://pepy.tech/badge/localsitemap)](https://pepy.tech/project/localsitemap) +[![Code Size](https://img.shields.io/github/languages/code-size/infinitode/localsitemap)](https://github.com/Infinitode/LocalSiteMap) +![License](https://img.shields.io/badge/license-MIT%20(modified)-brightgreen.svg) -### Changes in version 2.0.0 (Major Release): +LocalSiteMap is an open-source Python package designed for generating sitemaps and sitemap indexes from local files. It crawls your root website directory and automatically builds standard-compliant sitemaps. No third-party dependencies are required. + +> [!IMPORTANT] +> `LocalSiteMap` crawls any directory found in the root folder, be sure to add exclusions for important directories, or hidden/resource/file directories. + +## Changes in version 2.1.0 + +### Breaking change +- **Python 3.9 or newer is now required.** Python 3.6 – 3.8 are long past end-of-life and are no longer supported by modern packaging tooling. Version `2.0.0` on PyPI still serves those interpreters (pip selects it automatically), and the runtime code has no 3.9-only syntax. + +### Bug fixes +- **Exclusions no longer match parent folders**: exclusions are now resolved *inside* the site root, so a site checked out in (for example) a folder called `media` is not wiped out when `media` is excluded. +- **Priority map keys now behave as documented**: folder keys may be written with a trailing slash (`"blog/"`), Windows separators are accepted, and extension keys are matched case-insensitively (`".PHP"` matches `.php`). +- **URLs are percent-encoded**: file and folder names containing spaces, `#`, `&`, `?` or non-ASCII characters (e.g. `my page.html`, `café/`) now produce valid, crawlable URLs. +- **Extension matching is case-insensitive**: `PAGE.HTML` is picked up by the default configuration, and `page.html.bak` is not mistaken for `page.html`. +- **Inputs are validated**: a missing `root_path`, a `base_url` without a scheme, or a non-positive `max_urls` now raise a clear error instead of silently writing a broken sitemap. `max_urls` above the protocol limit of 50 000 is clamped with a warning. +- **Priorities are clamped** to the `0.00`–`1.00` range required by the sitemap protocol. +- **Deterministic, deduplicated output**: entries are sorted, duplicate `` values are dropped, and repeated runs produce byte-identical files. +- **Stale split sitemaps are removed**: shrinking a site no longer leaves orphaned `sitemap_N.xml` parts behind. +- **Output is written atomically** and missing output directories are created, so an interrupted run cannot leave a truncated sitemap behind. +- **Much faster XML writing**: the DOM round-trip (`minidom`) was replaced with a streaming writer, which also removes the stray blank lines from the output. + +### New features +- **Command line interface**: `localsitemap` / `python -m localsitemap` with exclusions, extensions, priorities and a JSON priority map. +- **Glob exclusions**: `["draft-*", "*.bak"]`. +- **New options**: `exclude_hidden`, `include_homepage`, `sort_entries`. +- **`generate_sitemap()` returns the written files**, which is handy for CI and build scripts. +- **Modern packaging**: metadata moved to `pyproject.toml` with the version read from `localsitemap/_version.py`. +- **Release pipeline**: published from GitHub Actions with PyPI Trusted Publishing (OIDC) and PEP 740 attestations — see [Release provenance](#release-provenance). + +### Changes in version 2.0.0 (Major Release) - **Precise Exclusions**: Rewrote exclusion matching so only exact folder or filename matches (not partial substring matches) are excluded. - **Traversal Optimization**: Subdirectories matching the exclusion list are pruned in-place, eliminating the overhead of traversing skipped directories. - **UTC Timestamps**: Fixed timezone bug to retrieve and format all file modification timestamps as true, accurate UTC standard dates. @@ -15,18 +46,15 @@ LocalSiteMap is an open-source Python package designed for generating sitemaps a - **Flexible Priority Mappings**: Custom mapping for directory/page priorities using a dictionary or callable mapping function. - **Sitemap Splitting & Sitemap Indexing**: Automatically splits large sitemaps into chunks and creates a standard `` parent if the URL limit (`max_urls`, defaults to 50,000) is exceeded. -### Changes in version 1.0.2: +### Changes in version 1.0.2 - Removed the unnecessary `install_requires` from the `setup.py` as the package utilizes none of the required packages. -### Changes in version 1.0.1: +### Changes in version 1.0.1 - Added `show_progress` boolean to print out mapping progress, pages mapped, and files found. -### Changes in version 1.0.0: +### Changes in version 1.0.0 - Added initial package code, with automatic directory crawling to generate the `sitemap.xml` file. -> [!IMPORTANT] -> `LocalSiteMap` crawls any directory found in the root folder, be sure to add exclusions for important directories, or hidden/resource/file directories. - ## Installation You can install LocalSiteMap using pip: @@ -37,14 +65,14 @@ pip install localsitemap ## Supported Python Versions -LocalSiteMap supports the following Python versions: +LocalSiteMap supports Python 3.9 and newer, and every release is tested on CPython 3.9 – 3.14 on Linux, plus Windows: -- Python 3.6 -- Python 3.7 -- Python 3.8 - Python 3.9 - Python 3.10 -- Python 3.11/Later (Preferred) +- Python 3.11 +- Python 3.12 +- Python 3.13/Later (Preferred) +- Python 3.14 ## Features @@ -73,8 +101,14 @@ base_url_of_your_website = "https://example.com" excluded = ["auth", "forms", "template.html", "media", ".git", ".vscode", "node_modules"] # Generate the sitemap -generate_sitemap(root_directory, base_url_of_your_website, "sitemap.xml", excluded, show_progress=True) -print("Sitemap generated in sitemap.xml") +written = generate_sitemap( + root_directory, + base_url_of_your_website, + "sitemap.xml", + excluded, + show_progress=True, +) +print("Sitemap generated:", ", ".join(written)) ``` ### Advanced Features (Extensions, Mappings & Splitting) @@ -82,33 +116,79 @@ print("Sitemap generated in sitemap.xml") ```python from localsitemap import generate_sitemap -# Custom priority dictionary or callable mapper +# Custom priority dictionary or callable mapper. +# Keys may be a relative path, a folder name (trailing slash optional), +# a file name, or an extension (case-insensitive). priority_mapping = { "about/index.html": 0.95, "blog/": 0.70, - ".php": 0.50 + ".php": 0.50, } generate_sitemap( root_path=r"path/to/website", base_url="https://example.com", output_file="sitemap.xml", - excluded_paths=["node_modules", ".git"], + excluded_paths=["node_modules", ".git", "draft-*", "*.bak"], allowed_extensions=[".html", ".htm", ".php"], default_priority=0.80, homepage_priority=1.00, priority_mapping=priority_mapping, - max_urls=1000 # Will automatically split and output a sitemap index if URLs exceed 1000 + max_urls=1000, # Automatically splits and writes a sitemap index above 1000 URLs ) ``` +### Exclusions + +Exclusions are always interpreted relative to `root_path`: + +| Pattern | Meaning | +| --- | --- | +| `"node_modules"` | any file or folder with that name, at any depth | +| `"blog/posts"` | that relative path and everything below it | +| `"/srv/site/private"` | an absolute path inside the site root | +| `"draft-*"`, `"*.bak"` | shell-style glob patterns | + +`excluded_paths=["auth"]` therefore excludes `auth/` but keeps `authority/`. + +### Command line + +```bash +# After pip install localsitemap (or with `python -m localsitemap`) +localsitemap path/to/website https://example.com \ + -o public/sitemap.xml \ + --exclude auth --exclude "draft-*" \ + --extensions .html,.htm,.php \ + --max-urls 1000 \ + --progress +``` + +Run `localsitemap --help` for every option, including `--priority-map priorities.json` and `--exclude-hidden`. + +### Release provenance + +Releases are built and published by GitHub Actions through [PyPI Trusted Publishing](https://docs.pypi.org/trusted-publishers/) (OIDC) — there is no long-lived PyPI API token in this repository. Every uploaded file is signed with a [PEP 740](https://peps.python.org/pep-0740/) digital attestation, which is what makes PyPI display the *provenance* indicator for the release. You can verify it yourself with the bundled script: + +```bash +python .github/scripts/verify_provenance.py --project localsitemap --version 2.1.0 +``` + +or straight from the public Integrity API: + +```bash +curl -s "https://pypi.org/integrity/localsitemap/2.1.0/localsitemap-2.1.0-py3-none-any.whl/provenance" \ + | python -m json.tool +``` + +A non-`404` response containing an `attestation_bundles` entry proves the file was built by this repository's workflow. + ## Contributing -Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to LocalSiteMap, please open an issue or submit a pull request on [GitHub](https://github.com/infinitode/localsitemap). +Contributions are welcome! If you encounter any issues, have suggestions, or want to contribute to LocalSiteMap, please open an issue or submit a pull request on [GitHub](https://github.com/Infinitode/LocalSiteMap). See [CONTRIBUTING.md](CONTRIBUTING.md) for the test and release workflow. ## License -LocalSiteMap is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/infinitode/localsitemap/blob/main/LICENSE) file for the full text. +LocalSiteMap is released under the terms of the **MIT License (Modified)**. Please see the [LICENSE](https://github.com/Infinitode/LocalSiteMap/blob/main/LICENSE) file for the full text. **Modified License Clause** From 8c66a498693bb0a3b5904e126194bcbb5cb4bebb Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:47:54 +0000 Subject: [PATCH 5/6] Run the Python 3.9 job on ubuntu-22.04 where it is available Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/tests.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 11b92ac..fa6269b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,16 +35,20 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - python-version: - - '3.9' - - '3.10' - - '3.11' - - '3.12' - - '3.13' - - '3.14' include: + # Python 3.9 ships in the tool cache of the 22.04 image. + - os: ubuntu-22.04 + python-version: '3.9' + - os: ubuntu-latest + python-version: '3.10' + - os: ubuntu-latest + python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' + - os: ubuntu-latest + python-version: '3.13' + - os: ubuntu-latest + python-version: '3.14' - os: windows-latest python-version: '3.12' steps: From f15fc99d902ce50a27b7f879d207be8ee0794ff1 Mon Sep 17 00:00:00 2001 From: johanbeytell1 <107624055+johanbeytell1@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:48:44 +0000 Subject: [PATCH 6/6] Skip publishing on manual workflow runs Manual 'Run workflow' executions now only build and validate the distributions, so the pipeline can be exercised safely before the PyPI trusted publisher is configured. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/build_and_publish.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 90731c9..12efaa0 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -92,6 +92,10 @@ jobs: publish: name: Publish distribution 📦 to PyPI needs: build + # Only real tag pushes publish. A manual "Run workflow" therefore builds, + # tests and validates the distributions without touching PyPI, which also + # makes it safe to run before Trusted Publishing is configured. + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest # NOTE: intentionally no `environment:` key. The PyPI trusted publisher is # configured with a blank environment name, and adding one here would make @@ -116,6 +120,7 @@ jobs: needs: - build - publish + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest permissions: contents: read