feat: reflecting changes to collections in tinybird & formatting (IN-997) (#3893)

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

services/libs/tinybird/datasources/collections.datasource

@@ -8,6 +8,9 @@ DESCRIPTION >
     - `description` provides additional details about the collection's purpose (empty string if not provided).
     - `categoryId` links to the category this collection belongs to (empty string if no category association).
     - `starred` indicates whether this collection is featured or highlighted (Bool, 0 default for not starred).
+    - `isPrivate` indicates whether this collection is private (Bool, 0 default for public).
+    - `ssoUserId` links to the SSO user who created this community collection (Nullable, null for curated collections).
+    - `logoUrl` is the URL of the collection's logo image (empty string if not provided).
     - `createdAt` and `updatedAt` are standard timestamp fields for record lifecycle tracking.
 
 TAGS "Project collections"
@@ -19,6 +22,9 @@ SCHEMA >
     `description` String `json:$.record.description` DEFAULT '',
     `categoryId` String `json:$.record.categoryId` DEFAULT '',
     `starred` Bool `json:$.record.starred` DEFAULT 0,
+    `isPrivate` Bool `json:$.record.isPrivate` DEFAULT 0,
+    `ssoUserId` Nullable(String) `json:$.record.ssoUserId`,
+    `logoUrl` String `json:$.record.logoUrl` DEFAULT '',
     `createdAt` DateTime64(3) `json:$.record.createdAt`,
     `updatedAt` DateTime64(3) `json:$.record.updatedAt`,
     `deletedAt` Nullable(DateTime64(3)) `json:$.record.deletedAt`

services/libs/tinybird/pipes/activities_filtered.pipe

@@ -32,9 +32,7 @@ SQL >
             AND a.timestamp
             < {{ DateTime(endDate, description="Filter activity timestamp before", required=False) }}
         {% end %}
-        {% if defined(repos) %}
-            AND a.channel IN (SELECT channel FROM repos_to_channels)
-        {% end %}
+        {% if defined(repos) %} AND a.channel IN (SELECT channel FROM repos_to_channels) {% end %}
         {% if defined(platform) %}
             AND a.platform
             = {{ String(platform, description="Filter activity platform", required=False) }}

services/libs/tinybird/pipes/activities_filtered_historical_cutoff.pipe

@@ -34,9 +34,7 @@ SQL >
             AND a.timestamp
             < {{ DateTime(endDate, description="Filter activity timestamp before", required=False) }}
         {% end %}
-        {% if defined(repos) %}
-            AND a.channel IN (SELECT channel FROM repos_to_channels)
-        {% end %}
+        {% if defined(repos) %} AND a.channel IN (SELECT channel FROM repos_to_channels) {% end %}
         {% if defined(platform) %}
             AND a.platform
             = {{ String(platform, description="Filter activity platform", required=False) }}

services/libs/tinybird/pipes/activities_filtered_retention.pipe

@@ -50,9 +50,7 @@ SQL >
             AND a.timestamp
             < {{ DateTime(endDate, description="Filter activity timestamp before", required=False) }}
         {% end %}
-        {% if defined(repos) %}
-            AND a.channel IN (SELECT channel FROM repos_to_channels)
-        {% end %}
+        {% if defined(repos) %} AND a.channel IN (SELECT channel FROM repos_to_channels) {% end %}
         {% if defined(platform) %}
             AND a.platform
             = {{ String(platform, description="Filter activity platform", required=False) }}

services/libs/tinybird/pipes/activityTypes_by_project.pipe

@@ -19,8 +19,6 @@ SQL >
     INNER JOIN activityTypes at ON a.type = at.activityType AND a.platform = at.platform
     WHERE
         a.segmentId = (SELECT segmentId FROM segments_filtered)
-        {% if defined(repos) %}
-            AND a.channel IN (SELECT channel FROM repos_to_channels)
-        {% end %}
+        {% if defined(repos) %} AND a.channel IN (SELECT channel FROM repos_to_channels) {% end %}
         AND (a.type, a.platform) IN (SELECT activityType, platform FROM activityTypes_filtered)
     ORDER BY activityType, platform

services/libs/tinybird/pipes/activity_heatmap_by_weekday_and_2hours_blocks.pipe

@@ -27,9 +27,7 @@ SQL >
             AND a.timestamp
             < {{ DateTime(endDate, description="Filter activity timestamp before", required=False) }}
         {% end %}
-        {% if defined(repos) %}
-            AND a.channel IN (SELECT channel FROM repos_to_channels)
-        {% end %}
+        {% if defined(repos) %} AND a.channel IN (SELECT channel FROM repos_to_channels) {% end %}
     group by weekday, two_hours_block
 
 NODE weekday_hours

services/libs/tinybird/pipes/categories_oss_index.pipe

@@ -43,7 +43,9 @@ DESCRIPTION >
     Returns only needed columns from collections
 
 SQL >
-    SELECT collections.id, collections.name FROM collections FINAL where isNull (deletedAt)
+    SELECT collections.id, collections.name
+    FROM collections FINAL
+    where isNull (deletedAt) AND isNull (ssoUserId)
 
 NODE categories_oss_index_projects_ranked_by_category
 DESCRIPTION >

services/libs/tinybird/pipes/category_groups_oss_index.pipe

@@ -40,7 +40,9 @@ DESCRIPTION >
     Returns only columns we need from collections table
 
 SQL >
-    SELECT collections.id, collections.name FROM collections FINAL where isNull (deletedAt)
+    SELECT collections.id, collections.name
+    FROM collections FINAL
+    where isNull (deletedAt) AND isNull (ssoUserId)
 
 NODE category_groups_oss_index_projects_ranked_by_category_group
 DESCRIPTION >

services/libs/tinybird/pipes/collections_filtered.pipe

@@ -15,7 +15,7 @@ SQL >
         on collectionsInsightsProjects.collectionId = collections.id
         and isNull (collectionsInsightsProjects.deletedAt)
     where
-        isNull (collections.deletedAt)
+        isNull (collections.deletedAt) AND isNull (collections.ssoUserId)
         {% if defined(slug) %}
             AND collections.slug
             = {{ String(slug, description="Filter collection by slug", required=False) }}
@@ -51,4 +51,7 @@ SQL >
         collections.createdAt,
         collections.updatedAt,
         collections.deletedAt,
-        collections.starred
+        collections.starred,
+        collections.isPrivate,
+        collections.ssoUserId,
+        collections.logoUrl

services/libs/tinybird/pipes/collections_oss_index.pipe

@@ -91,6 +91,7 @@ SQL >
     FROM collections FINAL
     WHERE
         isNull (collections.deletedAt)
+        AND isNull (collections.ssoUserId)
         and collections.categoryId IN (SELECT id from collections_oss_index_category_ids)
 
 NODE collections_oss_index_top_projects