Reduce duplication between collector and API-server

components/api_server/src/initialization/migrations.py

@@ -2,7 +2,8 @@
 
 from typing import TYPE_CHECKING
 
-from model.iterators import metrics, subjects
+from shared.model.iterators import metrics, subjects
+
 from utils.log import get_logger
 
 if TYPE_CHECKING:

components/api_server/src/model/iterators.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING
 
+from shared.model.iterators import issue_trackers, sources
+
 from .queries import is_password_parameter
 
 if TYPE_CHECKING:
@@ -10,31 +12,6 @@
     from shared.model.source import Source
 
 
-def subjects(*reports) -> Iterator:
-    """Return all subjects in the reports."""
-    for report in reports:
-        yield from report.get("subjects", {}).values()
-
-
-def metrics(*reports) -> Iterator:
-    """Return all metrics in the reports."""
-    for subject in subjects(*reports):
-        yield from subject.get("metrics", {}).values()
-
-
-def sources(*reports) -> Iterator:
-    """Return all sources in the reports."""
-    for metric in metrics(*reports):
-        yield from metric.get("sources", {}).values()
-
-
-def issue_trackers(*reports) -> Iterator:
-    """Return all issue trackers in the reports."""
-    for report in reports:
-        if issue_tracker := report.get("issue_tracker"):
-            yield issue_tracker
-
-
 def credential_holders(*reports, data_model: dict | None = None) -> Iterator[tuple[dict, list[str]]]:
     """Yield (parameters, keys) for each item in the reports holding credentials. Keys with empty values are skipped."""
     for source in sources(*reports):

components/api_server/src/routes/search.py

@@ -4,8 +4,9 @@
 
 import bottle
 
+from shared.model.iterators import metrics, sources, subjects
+
 from database.reports import latest_reports
-from model.iterators import metrics, sources, subjects
 from utils.log import get_logger
 
 if TYPE_CHECKING:

components/collector/.vulture-whitelist.py

@@ -79,7 +79,6 @@
 GrafanaK6SlowTransactions  # unused class (src/source_collectors/grafana_k6/slow_transactions.py:44)
 GrafanaK6SourceUpToDateness  # unused class (src/source_collectors/grafana_k6/source_up_to_dateness.py:17)
 GrafanaK6SourceVersion  # unused class (src/source_collectors/grafana_k6/source_version.py:16)
-GrafanaK6Tests  # unused class (src/source_collectors/grafana_k6/tests.py:12)
 fixable  # unused variable (src/source_collectors/harbor/security_warnings.py:68)
 scan_status  # unused variable (src/source_collectors/harbor/security_warnings.py:76)
 addition_links  # unused variable (src/source_collectors/harbor/security_warnings.py:90)
@@ -151,7 +150,7 @@
 PipDependencies  # unused class (src/source_collectors/pip/dependencies.py:10)
 PyupioSafetySecurityWarnings  # unused class (src/source_collectors/pyupio_safety/security_warnings.py:14)
 QualityTimeMetrics  # unused class (src/source_collectors/quality_time/metrics.py:20)
-QualityTimeSourceUpToDateness  # unused class (src/source_collectors/quality_time/source_up_to_dateness.py:15)
+QualityTimeSourceUpToDateness  # unused class (src/source_collectors/quality_time/source_up_to_dateness.py:17)
 QualityTimeSourceVersion  # unused class (src/source_collectors/quality_time/source_version.py:9)
 RobotFrameworkSourceUpToDateness  # unused class (src/source_collectors/robot_framework/source_up_to_dateness.py:17)
 RobotFrameworkSourceVersion  # unused class (src/source_collectors/robot_framework/source_version.py:16)

components/collector/src/source_collectors/quality_time/missing_metrics.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING
 
+from shared.model.iterators import metrics, subjects
+
 from collector_utilities.functions import match_string_or_regular_expression
 from collector_utilities.type import URL
 from model import SourceMeasurement, SourceResponses
@@ -80,8 +82,7 @@ def __nr_of_possible_metric_types(self, data_model: dict, reports: list[dict]) -
 
     def __report_nr_of_possible_metric_types(self, data_model: dict, report: dict) -> int:
         """Return the number of possible metric types in the report."""
-        subjects = report.get("subjects", {}).values()
-        return sum(len(self.supported_metric_types(data_model, subject)) for subject in subjects)
+        return sum(len(self.supported_metric_types(data_model, subject)) for subject in subjects(report))
 
     def __missing_metric_type_entities(self, data_model: dict, reports: list[dict], landing_url: URL) -> Entities:
         """Return the reports' missing metric types as entities."""
@@ -92,12 +93,8 @@ def __missing_metric_type_entities(self, data_model: dict, reports: list[dict],
 
     def __used_metric_types(self, report_uuid: ReportId) -> set[str]:
         """Return the metric types used in the report with the given report UUID."""
-        metric_types = set()
         report = next(report for report in self.reports if report["report_uuid"] == report_uuid)
-        for subject in report.get("subjects", {}).values():
-            for metric in subject.get("metrics", {}).values():
-                metric_types.add(metric["type"])
-        return metric_types
+        return {metric["type"] for metric in metrics(report)}
 
     def __report_missing_metric_type_entities(self, data_model: dict, report: dict, landing_url: URL) -> Entities:
         """Return the report's missing metric types as entities."""

components/collector/src/source_collectors/quality_time/source_up_to_dateness.py

@@ -2,6 +2,8 @@
 
 from typing import TYPE_CHECKING
 
+from shared.model.iterators import metrics
+
 from base_collectors import TimePassedCollector
 from collector_utilities.date_time import MIN_DATETIME, parse_datetime
 from collector_utilities.type import URL, Response
@@ -21,10 +23,10 @@ async def _api_url(self) -> URL:
 
     async def _parse_source_response_date_time(self, response: Response) -> datetime:
         """Override to parse the oldest datetime from the recent measurements."""
-        measurement_dates = []
-        for report in await self._get_reports(response):
-            for subject in report.get("subjects", {}).values():
-                for metric in subject.get("metrics", {}).values():
-                    if recent_measurements := metric.get("recent_measurements", []):
-                        measurement_dates.append(parse_datetime(recent_measurements[-1]["end"]))  # noqa: PERF401
+        reports = await self._get_reports(response)
+        measurement_dates = [
+            parse_datetime(metric["recent_measurements"][-1]["end"])
+            for metric in metrics(*reports)
+            if metric.get("recent_measurements", [])
+        ]
         return min(measurement_dates, default=MIN_DATETIME)

components/shared_code/.vulture-whitelist.py

@@ -32,7 +32,6 @@
 _.check_api_values  # unused method (src/shared_data_model/meta/parameter.py:82)
 documentation  # unused variable (src/shared_data_model/meta/source.py:26)
 parameter_layout  # unused variable (src/shared_data_model/meta/source.py:29)
-issue_tracker  # unused variable (src/shared_data_model/meta/source.py:31)
 supported_versions_description  # unused variable (src/shared_data_model/meta/source.py:32)
 _.check_parameters  # unused method (src/shared_data_model/meta/source.py:36)
 _.check_deprecation_url  # unused method (src/shared_data_model/meta/source.py:59)

components/shared_code/src/shared/model/iterators.py

@@ -0,0 +1,31 @@
+"""Model iterators."""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+def subjects(*reports) -> Iterator:
+    """Return all subjects in the reports."""
+    for report in reports:
+        yield from report.get("subjects", {}).values()
+
+
+def metrics(*reports) -> Iterator:
+    """Return all metrics in the reports."""
+    for subject in subjects(*reports):
+        yield from subject.get("metrics", {}).values()
+
+
+def sources(*reports) -> Iterator:
+    """Return all sources in the reports."""
+    for metric in metrics(*reports):
+        yield from metric.get("sources", {}).values()
+
+
+def issue_trackers(*reports) -> Iterator:
+    """Return all issue trackers in the reports."""
+    for report in reports:
+        if issue_tracker := report.get("issue_tracker"):
+            yield issue_tracker

components/shared_code/tests/shared/model/test_iterators.py

@@ -0,0 +1,58 @@
+"""Unit tests for the model iterators."""
+
+import unittest
+
+from shared.model.iterators import issue_trackers, metrics, sources, subjects
+
+
+class SubjectIteratorTest(unittest.TestCase):
+    """Unit test for the subject iterator."""
+
+    def test_empty_report(self):
+        """Test that an empty report contains no subjects."""
+        self.assertEqual([], list(subjects({})))
+
+    def test_one_subject(self):
+        """Test that the subject is returned."""
+        self.assertEqual([{}], list(subjects({"subjects": {"subject_uuid": {}}})))
+
+
+class MetricIteratorTest(unittest.TestCase):
+    """Unit test for the metric iterator."""
+
+    def test_empty_report(self):
+        """Test that an empty report contains no metrics."""
+        self.assertEqual([], list(metrics({})))
+
+    def test_one_metric(self):
+        """Test that the metric is returned."""
+        self.assertEqual([{}], list(metrics({"subjects": {"subject_uuid": {"metrics": {"metric_uuid": {}}}}})))
+
+
+class SourceIteratorTest(unittest.TestCase):
+    """Unit test for the source iterator."""
+
+    def test_empty_report(self):
+        """Test that an empty report contains no sources."""
+        self.assertEqual([], list(sources({})))
+
+    def test_one_source(self):
+        """Test that the source is returned."""
+        self.assertEqual(
+            [{}],
+            list(
+                sources({"subjects": {"subject_uuid": {"metrics": {"metric_uuid": {"sources": {"source_uuid": {}}}}}}})
+            ),
+        )
+
+
+class IssueTrackerIteratorTest(unittest.TestCase):
+    """Unit test for the issue tracker iterator."""
+
+    def test_empty_report(self):
+        """Test that an empty report contains no issue tracker."""
+        self.assertEqual([], list(issue_trackers({})))
+
+    def test_one_issue_tracker(self):
+        """Test that the issue tracker is returned."""
+        self.assertEqual([{"type": "jira"}], list(issue_trackers({"issue_tracker": {"type": "jira"}})))