From ec5c1322ad47657854d52ec35203eb03349e0249 Mon Sep 17 00:00:00 2001 From: Claudine Chen Date: Fri, 13 Nov 2026 19:12:00 -0500 Subject: [PATCH 1/2] ignore import of fields in properties file that are calculated --- .gitignore | 1 + .python-version | 1 + contributions.yaml | 2 +- scripts/add_new_contribution_to_yaml.py | 7 +++ scripts/parse_and_validate_properties_txt.py | 14 +++-- tests/unit/conftest.py | 22 ++++++++ .../unit/test_add_new_contribution_to_yaml.py | 47 +++++++++++++++++ tests/unit/test_fields_to_ignore.py | 52 +++++++++++++++++++ 8 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 .python-version create mode 100644 tests/unit/test_add_new_contribution_to_yaml.py create mode 100644 tests/unit/test_fields_to_ignore.py diff --git a/.gitignore b/.gitignore index 61f2dc9..820b4e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ **/__pycache__/ +.venv \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..4e4afc1 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11.18 diff --git a/contributions.yaml b/contributions.yaml index 31db960..f9b5b7f 100644 --- a/contributions.yaml +++ b/contributions.yaml @@ -7011,7 +7011,7 @@ contributions: download: https://github.com/domizai/Keyed/releases/latest/download/Keyed.zip previousVersions: - 1.0.0 -- id: org.rednoise.rita +- id: 314 status: VALID dateAdded: 2026-10-25T10:54:12+0000 type: library diff --git a/scripts/add_new_contribution_to_yaml.py b/scripts/add_new_contribution_to_yaml.py index 568313f..15edef4 100644 --- a/scripts/add_new_contribution_to_yaml.py +++ b/scripts/add_new_contribution_to_yaml.py @@ -9,6 +9,9 @@ from ruamel.yaml import YAML +FIELDS_TO_IGNORE = ['id', 'dateAdded', 'lastUpdated', 'status'] + + def split_categories(categories): categories = sorted(categories.replace('"', '').split(',')) categories = [category.strip() for category in categories] @@ -25,6 +28,10 @@ def postprocess_properties(properties_dict): if 'download' not in properties_dict: properties_dict['download'] = properties_dict['source'][:properties_dict['source'].rfind('.')] + '.zip' + # remove fields that are set by our scripts, not by the contributor + for field in FIELDS_TO_IGNORE: + properties_dict.pop(field, None) + if __name__ == "__main__": if len(argv) < 2: diff --git a/scripts/parse_and_validate_properties_txt.py b/scripts/parse_and_validate_properties_txt.py index 34a8aad..7a41639 100644 --- a/scripts/parse_and_validate_properties_txt.py +++ b/scripts/parse_and_validate_properties_txt.py @@ -10,11 +10,11 @@ from tenacity import retry, stop_after_attempt, wait_fixed import re import os -from typing import Optional, Union -from pydantic import BaseModel, Field, ConfigDict, field_validator, AliasChoices +from typing import Any, Optional, Union +from pydantic import BaseModel, Field, ConfigDict, field_validator, model_validator, AliasChoices import javaproperties as jp - +FIELDS_TO_IGNORE = ['id', 'download', 'dateAdded', 'lastUpdated', 'status', 'source', 'type'] class PropertiesBase(BaseModel): name: str @@ -33,6 +33,14 @@ class PropertiesBase(BaseModel): extra='allow', ) + @model_validator(mode="before") + @classmethod + def check_raw_input(cls, data: Any) -> Any: + if isinstance(data, dict): + for field in FIELDS_TO_IGNORE: + data.pop(field, None) + return data + class PropertiesExisting(PropertiesBase): version: Union[int, str] prettyVersion: Optional[str] = None diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index fe27af9..81e2340 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -130,3 +130,25 @@ def properties_with_extra_fields(): "customField": "custom value", "anotherExtra": "42" } + + +@pytest.fixture +def properties_with_fields_to_ignore(): + """Valid library data that also contains fields which are set by our scripts, + and so must be stripped if they appear in a contributor's properties file.""" + return { + "name": "Ignored Fields Library", + "authors": "Some Author", + "url": "https://ignored.com", + "categories": "ignored,fields", + "sentence": "Library whose properties file contains fields we set ourselves", + "version": "3", + "prettyVersion": "3.0", + "id": "999", + "download": "https://ignored.com/wrong.zip", + "dateAdded": "2001-01-01T00:00:00+0000", + "lastUpdated": "2001-01-01T00:00:00+0000", + "status": "DEPRECATED", + "source": "https://ignored.com/wrong.txt", + "type": "tool", + } diff --git a/tests/unit/test_add_new_contribution_to_yaml.py b/tests/unit/test_add_new_contribution_to_yaml.py new file mode 100644 index 0000000..37f7a43 --- /dev/null +++ b/tests/unit/test_add_new_contribution_to_yaml.py @@ -0,0 +1,47 @@ +import pytest +from scripts.add_new_contribution_to_yaml import postprocess_properties + + +@pytest.fixture +def new_contribution_props(): + """What parse_and_validate_properties_txt.py outputs, plus fields the database sets.""" + return { + "type": "library", + "source": "https://example.com/releases/library.txt", + "name": "Test Library", + "authors": "Some Author", + "url": "https://example.com", + "categories": "b,a", + "sentence": "A test library", + "version": 1, + "prettyVersion": "1.0", + "id": 999, + "dateAdded": "2001-01-01T00:00:00+0000", + "lastUpdated": "2001-01-01T00:00:00+0000", + "status": "DEPRECATED", + } + + +class TestPostprocessProperties: + + @pytest.mark.parametrize("field", ['id', 'dateAdded', 'lastUpdated', 'status']) + def test_removes_database_fields(self, field, new_contribution_props): + postprocess_properties(new_contribution_props) + + assert field not in new_contribution_props + + def test_keeps_type_and_source(self, new_contribution_props): + postprocess_properties(new_contribution_props) + + assert new_contribution_props['type'] == 'library' + assert new_contribution_props['source'] == 'https://example.com/releases/library.txt' + + def test_download_derived_from_source(self, new_contribution_props): + postprocess_properties(new_contribution_props) + + assert new_contribution_props['download'] == 'https://example.com/releases/library.zip' + + def test_categories_split_and_sorted(self, new_contribution_props): + postprocess_properties(new_contribution_props) + + assert new_contribution_props['categories'] == ['a', 'b'] diff --git a/tests/unit/test_fields_to_ignore.py b/tests/unit/test_fields_to_ignore.py new file mode 100644 index 0000000..1803c74 --- /dev/null +++ b/tests/unit/test_fields_to_ignore.py @@ -0,0 +1,52 @@ +""" +Fields like id, dateAdded, source, type etc. are set by our scripts, not by the +contributor. If they appear in a properties file, they must be dropped before +validation so they never reach the database. +""" +import pytest +from scripts.parse_and_validate_properties_txt import ( + PropertiesBase, PropertiesExisting, LibraryPropertiesNew, + validate_new, validate_existing, validate_new_library, +) + +IGNORED_FIELDS = ['id', 'download', 'dateAdded', 'lastUpdated', 'status', 'source', 'type'] + + +class TestFieldsToIgnore: + + @pytest.mark.parametrize("model", [PropertiesBase, PropertiesExisting, LibraryPropertiesNew]) + @pytest.mark.parametrize("field", IGNORED_FIELDS) + def test_model_drops_ignored_field(self, model, field, properties_with_fields_to_ignore): + props = model.model_validate(properties_with_fields_to_ignore) + + assert field not in props.model_dump() + assert field not in (props.model_extra or {}) + + @pytest.mark.parametrize("validate", [validate_new, validate_existing, validate_new_library]) + @pytest.mark.parametrize("field", IGNORED_FIELDS) + def test_validate_drops_ignored_field(self, validate, field, properties_with_fields_to_ignore): + props = validate(properties_with_fields_to_ignore) + + assert field not in props + + @pytest.mark.parametrize("validate", [validate_new, validate_existing, validate_new_library]) + def test_validate_keeps_other_fields(self, validate, properties_with_fields_to_ignore): + props = validate(properties_with_fields_to_ignore) + + assert props['name'] == properties_with_fields_to_ignore['name'] + assert props['authors'] == properties_with_fields_to_ignore['authors'] + assert props['categories'] == properties_with_fields_to_ignore['categories'] + + def test_ignored_fields_do_not_overwrite_type_and_source(self, properties_with_fields_to_ignore): + """Mirrors __main__ in parse_and_validate_properties_txt.py: type and source come + from the issue form, and must not be overwritten by values in the properties file.""" + contribution = { + "type": "library", + "source": "https://correct.com/library.txt", + } + contribution.update(validate_new_library(properties_with_fields_to_ignore)) + + assert contribution['type'] == 'library' + assert contribution['source'] == 'https://correct.com/library.txt' + assert 'download' not in contribution + assert 'id' not in contribution From 84eea4ea242754627372eb8a10de12a6579615cb Mon Sep 17 00:00:00 2001 From: Claudine Chen Date: Fri, 13 Nov 2026 19:17:10 -0500 Subject: [PATCH 2/2] fix --- .python-version | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .python-version diff --git a/.python-version b/.python-version deleted file mode 100644 index 4e4afc1..0000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.11.18