Skip to content

Commit 17e3c54

Browse files
themaroltthemarolt
authored
chore: script to sync nango tokens and db installations (CM-922) (#3826)
Signed-off-by: Uroš Marolt <uros@marolt.me>
1 parent c871e11 commit 17e3c54

4 files changed

Lines changed: 405 additions & 1 deletion

File tree

‎services/apps/nango_worker/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@
1010
"format": "npx prettier --write \"src/**/*.ts\"",
1111
"format-check": "npx prettier --check .",
1212
"tsc-check": "tsc --noEmit",
13-
"check-nango-mapping": "tsx src/bin/check-nango-mapping.ts"
13+
"check-nango-mapping": "tsx src/bin/check-nango-mapping.ts",
14+
"script:create-missing-github-token-connections": "tsx src/bin/create-missing-github-token-connections.ts",
15+
"script:sync-github-token-connections-metadata": "tsx src/bin/sync-github-token-connections-metadata.ts"
1416
},
1517
"dependencies": {
1618
"@crowd/archetype-standard": "workspace:*",

‎services/apps/nango_worker/src/bin/create-missing-github-token-connections.ts‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
import { READ_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database'
2+
import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor'
3+
import { getServiceLogger } from '@crowd/logging'
4+
import {
5+
NangoIntegration,
6+
createNangoGithubTokenConnection,
7+
getNangoConnections,
8+
initNangoCloudClient,
9+
} from '@crowd/nango'
10+
11+
const log = getServiceLogger()
12+
13+
// Required environment variables
14+
const REQUIRED_ENV_VARS = [
15+
'NANGO_CLOUD_SECRET_KEY',
16+
'NANGO_CLOUD_INTEGRATIONS',
17+
'CROWD_GITHUB_APP_ID',
18+
'CROWD_GITHUB_CLIENT_ID',
19+
]
20+
21+
function validateEnvVars(): void {
22+
const missing: string[] = []
23+
for (const envVar of REQUIRED_ENV_VARS) {
24+
if (!process.env[envVar]) {
25+
missing.push(envVar)
26+
}
27+
}
28+
29+
if (missing.length > 0) {
30+
log.error(`Missing required environment variables: ${missing.join(', ')}`)
31+
process.exit(1)
32+
}
33+
}
34+
35+
const processArguments = process.argv.slice(2)
36+
37+
const executeMode = processArguments.includes('--execute')
38+
const dryRunMode = !executeMode
39+
40+
if (dryRunMode) {
41+
log.info('Running in DRY-RUN mode. Use --execute flag to actually create connections.')
42+
}
43+
44+
setImmediate(async () => {
45+
validateEnvVars()
46+
47+
const appId = process.env.CROWD_GITHUB_APP_ID
48+
const clientId = process.env.CROWD_GITHUB_CLIENT_ID
49+
50+
const db = await getDbConnection(READ_DB_CONFIG())
51+
const qx = pgpQx(db)
52+
53+
await initNangoCloudClient()
54+
55+
// Get all github-token-* connections from Nango
56+
const allConnections = await getNangoConnections()
57+
const tokenConnections = allConnections.filter(
58+
(c) =>
59+
c.provider_config_key === NangoIntegration.GITHUB &&
60+
c.connection_id.toLowerCase().startsWith('github-token-'),
61+
)
62+
63+
log.info(`Found ${tokenConnections.length} github-token-* connections in Nango`)
64+
65+
// Extract installation IDs from connection names (e.g., github-token-52165842 -> 52165842)
66+
const nangoInstallationIds = tokenConnections
67+
.map((c) => {
68+
const match = c.connection_id.match(/^github-token-(\d+)$/i)
69+
return match ? match[1] : null
70+
})
71+
.filter((id): id is string => id !== null)
72+
73+
log.info(`Extracted ${nangoInstallationIds.length} installation IDs from Nango connections`)
74+
75+
// Find all GitHub integrations in the database that are NOT in Nango
76+
const missingInstallations = await qx.select(
77+
`
78+
SELECT
79+
id,
80+
"integrationIdentifier",
81+
"tenantId",
82+
"segmentId",
83+
status,
84+
"createdAt"
85+
FROM integrations
86+
WHERE platform = 'github'
87+
AND "deletedAt" IS NULL
88+
AND "integrationIdentifier" IS NOT NULL
89+
${nangoInstallationIds.length > 0 ? `AND "integrationIdentifier" NOT IN ($(nangoInstallationIds:csv))` : ''}
90+
ORDER BY "createdAt" DESC
91+
`,
92+
{ nangoInstallationIds },
93+
)
94+
95+
log.info(
96+
`Found ${missingInstallations.length} GitHub installations in database that are NOT in Nango`,
97+
)
98+
99+
// Report existing github-token-* connections
100+
log.info('='.repeat(80))
101+
log.info('EXISTING GITHUB TOKEN CONNECTIONS IN NANGO')
102+
log.info('='.repeat(80))
103+
log.info(`Total: ${tokenConnections.length}`)
104+
105+
log.info('')
106+
log.info('='.repeat(80))
107+
log.info('MISSING GITHUB INSTALLATIONS (in DB but not in Nango)')
108+
log.info('='.repeat(80))
109+
110+
if (missingInstallations.length === 0) {
111+
log.info('All GitHub installations in the database are present in Nango.')
112+
log.info('='.repeat(80))
113+
process.exit(0)
114+
}
115+
116+
log.info(`Total missing: ${missingInstallations.length}`)
117+
log.info('')
118+
119+
if (dryRunMode) {
120+
log.info('DRY-RUN: The following connections would be created:')
121+
for (const installation of missingInstallations) {
122+
log.info(
123+
{
124+
connectionId: `github-token-${installation.integrationIdentifier}`,
125+
installationId: installation.integrationIdentifier,
126+
integrationId: installation.id,
127+
tenantId: installation.tenantId,
128+
segmentId: installation.segmentId,
129+
status: installation.status,
130+
},
131+
`Would create: github-token-${installation.integrationIdentifier}`,
132+
)
133+
}
134+
log.info('')
135+
log.info('='.repeat(80))
136+
log.info('To actually create these connections, run with --execute flag')
137+
log.info('='.repeat(80))
138+
} else {
139+
log.info('EXECUTE MODE: Creating missing connections...')
140+
log.info('')
141+
142+
let successCount = 0
143+
let errorCount = 0
144+
let lastLoggedPercent = -1
145+
146+
for (let i = 0; i < missingInstallations.length; i++) {
147+
const installation = missingInstallations[i]
148+
const installationId = installation.integrationIdentifier
149+
150+
const percent = Math.floor(((i + 1) / missingInstallations.length) * 100)
151+
if (percent % 5 === 0 && percent !== lastLoggedPercent) {
152+
lastLoggedPercent = percent
153+
log.info(`Progress: ${i + 1}/${missingInstallations.length} (${percent}%)`)
154+
}
155+
156+
try {
157+
log.info(
158+
{
159+
installationId,
160+
integrationId: installation.id,
161+
tenantId: installation.tenantId,
162+
},
163+
`Creating connection: github-token-${installationId}`,
164+
)
165+
166+
const connectionId = await createNangoGithubTokenConnection(installationId, appId, clientId)
167+
168+
log.info(
169+
{ connectionId, installationId },
170+
`Successfully created connection: ${connectionId}`,
171+
)
172+
successCount++
173+
} catch (err) {
174+
log.error(
175+
{
176+
installationId,
177+
integrationId: installation.id,
178+
err,
179+
},
180+
`Failed to create connection for installation ${installationId}`,
181+
)
182+
errorCount++
183+
}
184+
}
185+
186+
log.info('')
187+
log.info('='.repeat(80))
188+
log.info('SUMMARY')
189+
log.info('='.repeat(80))
190+
log.info(`Successfully created: ${successCount}`)
191+
log.info(`Failed: ${errorCount}`)
192+
log.info(`Total attempted: ${missingInstallations.length}`)
193+
log.info('='.repeat(80))
194+
}
195+
196+
process.exit(0)
197+
})

‎services/apps/nango_worker/src/bin/sync-github-token-connections-metadata.ts‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import { getServiceLogger } from '@crowd/logging'
2+
import {
3+
NangoIntegration,
4+
getNangoConnectionData,
5+
getNangoConnections,
6+
initNangoCloudClient,
7+
setNangoMetadata,
8+
} from '@crowd/nango'
9+
10+
const log = getServiceLogger()
11+
12+
const REQUIRED_ENV_VARS = ['NANGO_CLOUD_SECRET_KEY', 'NANGO_CLOUD_INTEGRATIONS']
13+
14+
function validateEnvVars(): void {
15+
const missing: string[] = []
16+
for (const envVar of REQUIRED_ENV_VARS) {
17+
if (!process.env[envVar]) {
18+
missing.push(envVar)
19+
}
20+
}
21+
22+
if (missing.length > 0) {
23+
log.error(`Missing required environment variables: ${missing.join(', ')}`)
24+
process.exit(1)
25+
}
26+
}
27+
28+
const processArguments = process.argv.slice(2)
29+
30+
const executeMode = processArguments.includes('--execute')
31+
const dryRunMode = !executeMode
32+
33+
if (dryRunMode) {
34+
log.info('Running in DRY-RUN mode. Use --execute flag to actually update connection metadata.')
35+
}
36+
37+
setImmediate(async () => {
38+
validateEnvVars()
39+
40+
await initNangoCloudClient()
41+
42+
const allConnections = await getNangoConnections()
43+
44+
// Get all github-token-* connection IDs
45+
const tokenConnectionIds = allConnections
46+
.filter(
47+
(c) =>
48+
c.provider_config_key === NangoIntegration.GITHUB &&
49+
c.connection_id.toLowerCase().startsWith('github-token-'),
50+
)
51+
.map((c) => c.connection_id)
52+
53+
log.info(`Found ${tokenConnectionIds.length} github-token-* connections in Nango`)
54+
55+
if (tokenConnectionIds.length === 0) {
56+
log.error('No github-token-* connections found in Nango. Nothing to do.')
57+
process.exit(1)
58+
}
59+
60+
// Get all repo connections (non-token GitHub connections)
61+
const repoConnections = allConnections.filter(
62+
(c) =>
63+
c.provider_config_key === NangoIntegration.GITHUB &&
64+
!c.connection_id.toLowerCase().startsWith('github-token-'),
65+
)
66+
67+
log.info(`Found ${repoConnections.length} GitHub repo connections to check`)
68+
69+
let updatedCount = 0
70+
let skippedCount = 0
71+
let errorCount = 0
72+
let lastLoggedPercent = -1
73+
74+
for (let i = 0; i < repoConnections.length; i++) {
75+
const repoConnection = repoConnections[i]
76+
77+
const percent = Math.floor(((i + 1) / repoConnections.length) * 100)
78+
if (percent % 5 === 0 && percent !== lastLoggedPercent) {
79+
lastLoggedPercent = percent
80+
log.info(`Progress: ${i + 1}/${repoConnections.length} (${percent}%)`)
81+
}
82+
83+
try {
84+
const data = await getNangoConnectionData(
85+
NangoIntegration.GITHUB,
86+
repoConnection.connection_id,
87+
)
88+
89+
const metadata = data.metadata
90+
const existingTokenIds = (metadata.connection_ids as string[]) || []
91+
92+
// Find token connections that are missing from this repo connection's metadata
93+
const missingTokenIds = tokenConnectionIds.filter((id) => !existingTokenIds.includes(id))
94+
95+
if (missingTokenIds.length === 0) {
96+
skippedCount++
97+
continue
98+
}
99+
100+
log.info(
101+
{
102+
connectionId: repoConnection.connection_id,
103+
existingCount: existingTokenIds.length,
104+
missingCount: missingTokenIds.length,
105+
missingTokenIds,
106+
},
107+
`Connection ${repoConnection.connection_id} is missing ${missingTokenIds.length} token connection(s)`,
108+
)
109+
110+
if (executeMode) {
111+
const newMetadata = {
112+
...metadata,
113+
connection_ids: [...existingTokenIds, ...missingTokenIds],
114+
}
115+
116+
await setNangoMetadata(NangoIntegration.GITHUB, repoConnection.connection_id, newMetadata)
117+
118+
log.info(
119+
{ connectionId: repoConnection.connection_id },
120+
`Updated metadata for connection ${repoConnection.connection_id}`,
121+
)
122+
}
123+
124+
updatedCount++
125+
} catch (err) {
126+
log.error(
127+
{ connectionId: repoConnection.connection_id, err },
128+
`Failed to process connection ${repoConnection.connection_id}`,
129+
)
130+
errorCount++
131+
}
132+
}
133+
134+
log.info('')
135+
log.info('='.repeat(80))
136+
log.info('SUMMARY')
137+
log.info('='.repeat(80))
138+
log.info(`Total repo connections: ${repoConnections.length}`)
139+
log.info(`${dryRunMode ? 'Would update' : 'Updated'}: ${updatedCount}`)
140+
log.info(`Already up-to-date: ${skippedCount}`)
141+
log.info(`Errors: ${errorCount}`)
142+
log.info('='.repeat(80))
143+
144+
if (dryRunMode && updatedCount > 0) {
145+
log.info('To actually update these connections, run with --execute flag')
146+
}
147+
148+
process.exit(0)
149+
})

0 commit comments

Comments
 (0)