ignore import of fields in properties file that are calculated

.gitignore

@@ -1 +1,2 @@
 **/__pycache__/
+.venv

contributions.yaml

@@ -7011,7 +7011,7 @@ contributions:
   download: https://github.com/domizai/Keyed/releases/latest/download/Keyed.zip
   previousVersions:
   - 1.0.0
-- id: org.rednoise.rita
+- id: 314
   status: VALID
   dateAdded: 2026-10-25T10:54:12+0000
   type: library

scripts/add_new_contribution_to_yaml.py

@@ -9,6 +9,9 @@
 from ruamel.yaml import YAML
 
 
+FIELDS_TO_IGNORE = ['id', 'dateAdded', 'lastUpdated', 'status']
+
+
 def split_categories(categories):
     categories = sorted(categories.replace('"', '').split(','))
     categories = [category.strip() for category in categories]
@@ -25,6 +28,10 @@ def postprocess_properties(properties_dict):
     if 'download' not in properties_dict:
         properties_dict['download'] = properties_dict['source'][:properties_dict['source'].rfind('.')] + '.zip'
 
+    # remove fields that are set by our scripts, not by the contributor
+    for field in FIELDS_TO_IGNORE:
+        properties_dict.pop(field, None)
+
 
 if __name__ == "__main__":
     if len(argv) < 2:

scripts/parse_and_validate_properties_txt.py

@@ -10,11 +10,11 @@
 from tenacity import retry, stop_after_attempt, wait_fixed
 import re
 import os
-from typing import Optional, Union
-from pydantic import BaseModel, Field, ConfigDict, field_validator, AliasChoices
+from typing import Any, Optional, Union
+from pydantic import BaseModel, Field, ConfigDict, field_validator, model_validator, AliasChoices
 import javaproperties as jp
 
-
+FIELDS_TO_IGNORE = ['id', 'download', 'dateAdded', 'lastUpdated', 'status', 'source', 'type']
 
 class PropertiesBase(BaseModel):
     name: str
@@ -33,6 +33,14 @@ class PropertiesBase(BaseModel):
         extra='allow',
     )
 
+    @model_validator(mode="before")
+    @classmethod
+    def check_raw_input(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            for field in FIELDS_TO_IGNORE:
+                data.pop(field, None)
+        return data
+
 class PropertiesExisting(PropertiesBase):
     version: Union[int, str]
     prettyVersion: Optional[str] = None

tests/unit/conftest.py

@@ -130,3 +130,25 @@ def properties_with_extra_fields():
         "customField": "custom value",
         "anotherExtra": "42"
     }
+
+
+@pytest.fixture
+def properties_with_fields_to_ignore():
+    """Valid library data that also contains fields which are set by our scripts,
+    and so must be stripped if they appear in a contributor's properties file."""
+    return {
+        "name": "Ignored Fields Library",
+        "authors": "Some Author",
+        "url": "https://ignored.com",
+        "categories": "ignored,fields",
+        "sentence": "Library whose properties file contains fields we set ourselves",
+        "version": "3",
+        "prettyVersion": "3.0",
+        "id": "999",
+        "download": "https://ignored.com/wrong.zip",
+        "dateAdded": "2001-01-01T00:00:00+0000",
+        "lastUpdated": "2001-01-01T00:00:00+0000",
+        "status": "DEPRECATED",
+        "source": "https://ignored.com/wrong.txt",
+        "type": "tool",
+    }

tests/unit/test_add_new_contribution_to_yaml.py

@@ -0,0 +1,47 @@
+import pytest
+from scripts.add_new_contribution_to_yaml import postprocess_properties
+
+
+@pytest.fixture
+def new_contribution_props():
+    """What parse_and_validate_properties_txt.py outputs, plus fields the database sets."""
+    return {
+        "type": "library",
+        "source": "https://example.com/releases/library.txt",
+        "name": "Test Library",
+        "authors": "Some Author",
+        "url": "https://example.com",
+        "categories": "b,a",
+        "sentence": "A test library",
+        "version": 1,
+        "prettyVersion": "1.0",
+        "id": 999,
+        "dateAdded": "2001-01-01T00:00:00+0000",
+        "lastUpdated": "2001-01-01T00:00:00+0000",
+        "status": "DEPRECATED",
+    }
+
+
+class TestPostprocessProperties:
+
+    @pytest.mark.parametrize("field", ['id', 'dateAdded', 'lastUpdated', 'status'])
+    def test_removes_database_fields(self, field, new_contribution_props):
+        postprocess_properties(new_contribution_props)
+
+        assert field not in new_contribution_props
+
+    def test_keeps_type_and_source(self, new_contribution_props):
+        postprocess_properties(new_contribution_props)
+
+        assert new_contribution_props['type'] == 'library'
+        assert new_contribution_props['source'] == 'https://example.com/releases/library.txt'
+
+    def test_download_derived_from_source(self, new_contribution_props):
+        postprocess_properties(new_contribution_props)
+
+        assert new_contribution_props['download'] == 'https://example.com/releases/library.zip'
+
+    def test_categories_split_and_sorted(self, new_contribution_props):
+        postprocess_properties(new_contribution_props)
+
+        assert new_contribution_props['categories'] == ['a', 'b']

tests/unit/test_fields_to_ignore.py

@@ -0,0 +1,52 @@
+"""
+Fields like id, dateAdded, source, type etc. are set by our scripts, not by the
+contributor. If they appear in a properties file, they must be dropped before
+validation so they never reach the database.
+"""
+import pytest
+from scripts.parse_and_validate_properties_txt import (
+    PropertiesBase, PropertiesExisting, LibraryPropertiesNew,
+    validate_new, validate_existing, validate_new_library,
+)
+
+IGNORED_FIELDS = ['id', 'download', 'dateAdded', 'lastUpdated', 'status', 'source', 'type']
+
+
+class TestFieldsToIgnore:
+
+    @pytest.mark.parametrize("model", [PropertiesBase, PropertiesExisting, LibraryPropertiesNew])
+    @pytest.mark.parametrize("field", IGNORED_FIELDS)
+    def test_model_drops_ignored_field(self, model, field, properties_with_fields_to_ignore):
+        props = model.model_validate(properties_with_fields_to_ignore)
+
+        assert field not in props.model_dump()
+        assert field not in (props.model_extra or {})
+
+    @pytest.mark.parametrize("validate", [validate_new, validate_existing, validate_new_library])
+    @pytest.mark.parametrize("field", IGNORED_FIELDS)
+    def test_validate_drops_ignored_field(self, validate, field, properties_with_fields_to_ignore):
+        props = validate(properties_with_fields_to_ignore)
+
+        assert field not in props
+
+    @pytest.mark.parametrize("validate", [validate_new, validate_existing, validate_new_library])
+    def test_validate_keeps_other_fields(self, validate, properties_with_fields_to_ignore):
+        props = validate(properties_with_fields_to_ignore)
+
+        assert props['name'] == properties_with_fields_to_ignore['name']
+        assert props['authors'] == properties_with_fields_to_ignore['authors']
+        assert props['categories'] == properties_with_fields_to_ignore['categories']
+
+    def test_ignored_fields_do_not_overwrite_type_and_source(self, properties_with_fields_to_ignore):
+        """Mirrors __main__ in parse_and_validate_properties_txt.py: type and source come
+        from the issue form, and must not be overwritten by values in the properties file."""
+        contribution = {
+            "type": "library",
+            "source": "https://correct.com/library.txt",
+        }
+        contribution.update(validate_new_library(properties_with_fields_to_ignore))
+
+        assert contribution['type'] == 'library'
+        assert contribution['source'] == 'https://correct.com/library.txt'
+        assert 'download' not in contribution
+        assert 'id' not in contribution