diff --git a/components/api_server/src/initialization/migrations.py b/components/api_server/src/initialization/migrations.py index 88ffed79c9..3dabd776f3 100644 --- a/components/api_server/src/initialization/migrations.py +++ b/components/api_server/src/initialization/migrations.py @@ -2,7 +2,8 @@ from typing import TYPE_CHECKING -from model.iterators import metrics, subjects +from shared.model.iterators import metrics, subjects + from utils.log import get_logger if TYPE_CHECKING: diff --git a/components/api_server/src/model/iterators.py b/components/api_server/src/model/iterators.py index 3b1144652f..8a2c2742a3 100644 --- a/components/api_server/src/model/iterators.py +++ b/components/api_server/src/model/iterators.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING +from shared.model.iterators import issue_trackers, sources + from .queries import is_password_parameter if TYPE_CHECKING: @@ -10,31 +12,6 @@ from shared.model.source import Source -def subjects(*reports) -> Iterator: - """Return all subjects in the reports.""" - for report in reports: - yield from report.get("subjects", {}).values() - - -def metrics(*reports) -> Iterator: - """Return all metrics in the reports.""" - for subject in subjects(*reports): - yield from subject.get("metrics", {}).values() - - -def sources(*reports) -> Iterator: - """Return all sources in the reports.""" - for metric in metrics(*reports): - yield from metric.get("sources", {}).values() - - -def issue_trackers(*reports) -> Iterator: - """Return all issue trackers in the reports.""" - for report in reports: - if issue_tracker := report.get("issue_tracker"): - yield issue_tracker - - def credential_holders(*reports, data_model: dict | None = None) -> Iterator[tuple[dict, list[str]]]: """Yield (parameters, keys) for each item in the reports holding credentials. Keys with empty values are skipped.""" for source in sources(*reports): diff --git a/components/api_server/src/routes/search.py b/components/api_server/src/routes/search.py index a13b287939..d02128db76 100644 --- a/components/api_server/src/routes/search.py +++ b/components/api_server/src/routes/search.py @@ -4,8 +4,9 @@ import bottle +from shared.model.iterators import metrics, sources, subjects + from database.reports import latest_reports -from model.iterators import metrics, sources, subjects from utils.log import get_logger if TYPE_CHECKING: diff --git a/components/collector/.vulture-whitelist.py b/components/collector/.vulture-whitelist.py index 03d3abf16f..95f5aea976 100644 --- a/components/collector/.vulture-whitelist.py +++ b/components/collector/.vulture-whitelist.py @@ -79,7 +79,6 @@ GrafanaK6SlowTransactions # unused class (src/source_collectors/grafana_k6/slow_transactions.py:44) GrafanaK6SourceUpToDateness # unused class (src/source_collectors/grafana_k6/source_up_to_dateness.py:17) GrafanaK6SourceVersion # unused class (src/source_collectors/grafana_k6/source_version.py:16) -GrafanaK6Tests # unused class (src/source_collectors/grafana_k6/tests.py:12) fixable # unused variable (src/source_collectors/harbor/security_warnings.py:68) scan_status # unused variable (src/source_collectors/harbor/security_warnings.py:76) addition_links # unused variable (src/source_collectors/harbor/security_warnings.py:90) @@ -151,7 +150,7 @@ PipDependencies # unused class (src/source_collectors/pip/dependencies.py:10) PyupioSafetySecurityWarnings # unused class (src/source_collectors/pyupio_safety/security_warnings.py:14) QualityTimeMetrics # unused class (src/source_collectors/quality_time/metrics.py:20) -QualityTimeSourceUpToDateness # unused class (src/source_collectors/quality_time/source_up_to_dateness.py:15) +QualityTimeSourceUpToDateness # unused class (src/source_collectors/quality_time/source_up_to_dateness.py:17) QualityTimeSourceVersion # unused class (src/source_collectors/quality_time/source_version.py:9) RobotFrameworkSourceUpToDateness # unused class (src/source_collectors/robot_framework/source_up_to_dateness.py:17) RobotFrameworkSourceVersion # unused class (src/source_collectors/robot_framework/source_version.py:16) diff --git a/components/collector/src/source_collectors/quality_time/missing_metrics.py b/components/collector/src/source_collectors/quality_time/missing_metrics.py index 52eb4c4aa6..82fef4c381 100644 --- a/components/collector/src/source_collectors/quality_time/missing_metrics.py +++ b/components/collector/src/source_collectors/quality_time/missing_metrics.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING +from shared.model.iterators import metrics, subjects + from collector_utilities.functions import match_string_or_regular_expression from collector_utilities.type import URL from model import SourceMeasurement, SourceResponses @@ -80,8 +82,7 @@ def __nr_of_possible_metric_types(self, data_model: dict, reports: list[dict]) - def __report_nr_of_possible_metric_types(self, data_model: dict, report: dict) -> int: """Return the number of possible metric types in the report.""" - subjects = report.get("subjects", {}).values() - return sum(len(self.supported_metric_types(data_model, subject)) for subject in subjects) + return sum(len(self.supported_metric_types(data_model, subject)) for subject in subjects(report)) def __missing_metric_type_entities(self, data_model: dict, reports: list[dict], landing_url: URL) -> Entities: """Return the reports' missing metric types as entities.""" @@ -92,12 +93,8 @@ def __missing_metric_type_entities(self, data_model: dict, reports: list[dict], def __used_metric_types(self, report_uuid: ReportId) -> set[str]: """Return the metric types used in the report with the given report UUID.""" - metric_types = set() report = next(report for report in self.reports if report["report_uuid"] == report_uuid) - for subject in report.get("subjects", {}).values(): - for metric in subject.get("metrics", {}).values(): - metric_types.add(metric["type"]) - return metric_types + return {metric["type"] for metric in metrics(report)} def __report_missing_metric_type_entities(self, data_model: dict, report: dict, landing_url: URL) -> Entities: """Return the report's missing metric types as entities.""" diff --git a/components/collector/src/source_collectors/quality_time/source_up_to_dateness.py b/components/collector/src/source_collectors/quality_time/source_up_to_dateness.py index 61c58204b5..7fbbfa0d75 100644 --- a/components/collector/src/source_collectors/quality_time/source_up_to_dateness.py +++ b/components/collector/src/source_collectors/quality_time/source_up_to_dateness.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING +from shared.model.iterators import metrics + from base_collectors import TimePassedCollector from collector_utilities.date_time import MIN_DATETIME, parse_datetime from collector_utilities.type import URL, Response @@ -21,10 +23,10 @@ async def _api_url(self) -> URL: async def _parse_source_response_date_time(self, response: Response) -> datetime: """Override to parse the oldest datetime from the recent measurements.""" - measurement_dates = [] - for report in await self._get_reports(response): - for subject in report.get("subjects", {}).values(): - for metric in subject.get("metrics", {}).values(): - if recent_measurements := metric.get("recent_measurements", []): - measurement_dates.append(parse_datetime(recent_measurements[-1]["end"])) # noqa: PERF401 + reports = await self._get_reports(response) + measurement_dates = [ + parse_datetime(metric["recent_measurements"][-1]["end"]) + for metric in metrics(*reports) + if metric.get("recent_measurements", []) + ] return min(measurement_dates, default=MIN_DATETIME) diff --git a/components/shared_code/.vulture-whitelist.py b/components/shared_code/.vulture-whitelist.py index f69af1e5ec..c8aa77515f 100644 --- a/components/shared_code/.vulture-whitelist.py +++ b/components/shared_code/.vulture-whitelist.py @@ -32,7 +32,6 @@ _.check_api_values # unused method (src/shared_data_model/meta/parameter.py:82) documentation # unused variable (src/shared_data_model/meta/source.py:26) parameter_layout # unused variable (src/shared_data_model/meta/source.py:29) -issue_tracker # unused variable (src/shared_data_model/meta/source.py:31) supported_versions_description # unused variable (src/shared_data_model/meta/source.py:32) _.check_parameters # unused method (src/shared_data_model/meta/source.py:36) _.check_deprecation_url # unused method (src/shared_data_model/meta/source.py:59) diff --git a/components/shared_code/src/shared/model/iterators.py b/components/shared_code/src/shared/model/iterators.py new file mode 100644 index 0000000000..324dfa192d --- /dev/null +++ b/components/shared_code/src/shared/model/iterators.py @@ -0,0 +1,31 @@ +"""Model iterators.""" + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + +def subjects(*reports) -> Iterator: + """Return all subjects in the reports.""" + for report in reports: + yield from report.get("subjects", {}).values() + + +def metrics(*reports) -> Iterator: + """Return all metrics in the reports.""" + for subject in subjects(*reports): + yield from subject.get("metrics", {}).values() + + +def sources(*reports) -> Iterator: + """Return all sources in the reports.""" + for metric in metrics(*reports): + yield from metric.get("sources", {}).values() + + +def issue_trackers(*reports) -> Iterator: + """Return all issue trackers in the reports.""" + for report in reports: + if issue_tracker := report.get("issue_tracker"): + yield issue_tracker diff --git a/components/shared_code/tests/shared/model/test_iterators.py b/components/shared_code/tests/shared/model/test_iterators.py new file mode 100644 index 0000000000..e80f5f2377 --- /dev/null +++ b/components/shared_code/tests/shared/model/test_iterators.py @@ -0,0 +1,58 @@ +"""Unit tests for the model iterators.""" + +import unittest + +from shared.model.iterators import issue_trackers, metrics, sources, subjects + + +class SubjectIteratorTest(unittest.TestCase): + """Unit test for the subject iterator.""" + + def test_empty_report(self): + """Test that an empty report contains no subjects.""" + self.assertEqual([], list(subjects({}))) + + def test_one_subject(self): + """Test that the subject is returned.""" + self.assertEqual([{}], list(subjects({"subjects": {"subject_uuid": {}}}))) + + +class MetricIteratorTest(unittest.TestCase): + """Unit test for the metric iterator.""" + + def test_empty_report(self): + """Test that an empty report contains no metrics.""" + self.assertEqual([], list(metrics({}))) + + def test_one_metric(self): + """Test that the metric is returned.""" + self.assertEqual([{}], list(metrics({"subjects": {"subject_uuid": {"metrics": {"metric_uuid": {}}}}}))) + + +class SourceIteratorTest(unittest.TestCase): + """Unit test for the source iterator.""" + + def test_empty_report(self): + """Test that an empty report contains no sources.""" + self.assertEqual([], list(sources({}))) + + def test_one_source(self): + """Test that the source is returned.""" + self.assertEqual( + [{}], + list( + sources({"subjects": {"subject_uuid": {"metrics": {"metric_uuid": {"sources": {"source_uuid": {}}}}}}}) + ), + ) + + +class IssueTrackerIteratorTest(unittest.TestCase): + """Unit test for the issue tracker iterator.""" + + def test_empty_report(self): + """Test that an empty report contains no issue tracker.""" + self.assertEqual([], list(issue_trackers({}))) + + def test_one_issue_tracker(self): + """Test that the issue tracker is returned.""" + self.assertEqual([{"type": "jira"}], list(issue_trackers({"issue_tracker": {"type": "jira"}})))