fix: gerrit repositories filtering (CM-1079) (#3977)

joanagmaia · web-flow · commit e885a25c49d9 · 2026-04-02T11:53:58.000+01:00
Signed-off-by: Joana Maia &lt;jmaia@contractor.linuxfoundation.org&gt;
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_0.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_0.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_0.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_1.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_1.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_1.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_2.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_2.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_2.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_3.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_3.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_3.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_4.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_4.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_4.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_5.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_5.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_5.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_6.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_6.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_6.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_7.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_7.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_7.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_8.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_8.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_8.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe b/services/libs/tinybird/pipes/activityRelations_bucket_clean_enrich_copy_pipe_9.pipe
@@ -36,12 +36,8 @@ SQL >
         and (
             (
                 platform IN ('git', 'gerrit', 'github', 'gitlab')
-                AND (channel, activityRelations_bucket_MV_ds_9.segmentId) IN (
-                    SELECT r.url, r.segmentId
-                    FROM repositories r FINAL
-                    INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
-                    WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true
-                )
+                AND (channel, activityRelations_bucket_MV_ds_9.segmentId)
+                IN (SELECT channel, segmentId FROM repos_to_channels)
             )
             OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
         )
diff --git a/services/libs/tinybird/pipes/repos_to_channels.pipe b/services/libs/tinybird/pipes/repos_to_channels.pipe
@@ -6,7 +6,7 @@ DESCRIPTION >
     - Parameters:
     - `repos`: Optional array of repository URLs to expand. If not provided, defaults to all enabled repositories.
     - `excluded`: Optional boolean. When true, expands excluded repositories instead of enabled ones. Used to build exclusion filters.
-    - Response: `channel` - all possible channel URL formats for the given repos
+    - Response: `channel` - all possible channel URL formats for the given repos; `segmentId` - segment the repo belongs to (empty string when repos param is provided)
 
 TAGS "Repository URLs", "Gerrit"
 
@@ -20,13 +20,15 @@ SQL >
         SELECT
             arrayJoin(
                 {{ Array(repos, 'String', description="Repository URLs to expand", required=False) }}
-            ) AS url
+            ) AS url,
+            '' AS segmentId
     {% else %}
-        SELECT url
-        FROM repositories FINAL
+        SELECT r.url AS url, r.segmentId AS segmentId
+        FROM repositories r FINAL
+        INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id
         WHERE
-            isNull (deletedAt) AND enabled = true
-            {% if defined(excluded) and excluded %} AND excluded = true
+            isNull (r.deletedAt) AND r.enabled = true AND isNull (i.deletedAt)
+            {% if defined(excluded) and excluded %} AND r.excluded = true
             {% end %}
     {% end %}
 
@@ -35,18 +37,24 @@ DESCRIPTION >
     Identify Gerrit repositories by joining with integrations table
 
 SQL >
-    SELECT r.url
+    SELECT r.url AS url, r.segmentId AS segmentId
     FROM repositories r FINAL
     JOIN integrations i FINAL ON r.sourceIntegrationId = i.id
-    WHERE i.platform = 'gerrit' AND isNull (r.deletedAt) AND r.url IN (SELECT url FROM repos_to_expand)
+    INNER JOIN insightsProjects p FINAL ON r.insightsProjectId = p.id
+    WHERE
+        i.platform = 'gerrit'
+        AND isNull (r.deletedAt)
+        AND r.enabled = true
+        AND isNull (p.deletedAt)
+        AND r.url IN (SELECT url FROM repos_to_expand)
 
 NODE expanded_urls
 DESCRIPTION >
     Output original URLs plus Gerrit channel variants
 
 SQL >
     -- Original URLs (all repos)
-    SELECT url AS channel
+    SELECT url AS channel, segmentId
     FROM repos_to_expand
     UNION ALL
     -- Gerrit channel variants: insert q/project: after the base path
@@ -67,12 +75,13 @@ SQL >
                     '/q/project:',
                     if(path(url) = '/', '', substring(path(url), 2))
                 )
-        END AS channel
+        END AS channel,
+        segmentId
     FROM gerrit_repos
 
 NODE channels_deduplicated
 DESCRIPTION >
     Final deduplicated list of all possible channel URLs
 
 SQL >
-    SELECT DISTINCT channel FROM expanded_urls
+    SELECT DISTINCT channel, segmentId FROM expanded_urls

Original file line number	Diff line number	Diff line change
`@@ -36,12 +36,8 @@ SQL >`
`36`	`36`	`and (`
`37`	`37`	`(`
`38`	`38`	`platform IN ('git', 'gerrit', 'github', 'gitlab')`
`39`		`- AND (channel, activityRelations_bucket_MV_ds_0.segmentId) IN (`
`40`		`- SELECT r.url, r.segmentId`
`41`		`- FROM repositories r FINAL`
`42`		`- INNER JOIN insightsProjects i FINAL ON r.insightsProjectId = i.id`
`43`		`- WHERE isNull (r.deletedAt) AND isNull (i.deletedAt) AND r.enabled = true`
`44`		`- )`
	`39`	`+ AND (channel, activityRelations_bucket_MV_ds_0.segmentId)`
	`40`	`+ IN (SELECT channel, segmentId FROM repos_to_channels)`
`45`	`41`	`)`
`46`	`42`	`OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')`
`47`	`43`	`)`