chore: enable crustdata member enrichment source and bump cadence (CM-886) (#3790)

Author: skwowet

services/apps/members_enrichment_worker/src/schedules/getMembersToEnrich.ts

@@ -14,7 +14,7 @@ export const scheduleMembersEnrichment = async () => {
     await svc.temporal.schedule.create({
       scheduleId: 'members-enrichment-multiple-sources',
       spec: {
-        cronExpressions: ['0 6 * * *'],
+        cronExpressions: ['0 * * * *'],
       },
       policies: {
         overlap: ScheduleOverlapPolicy.SKIP,

services/apps/members_enrichment_worker/src/sources/serp/service.ts

@@ -32,7 +32,7 @@ export default class EnrichmentServiceSerpApi extends LoggerBase implements IEnr
   // bust cache after 120 days
   public cacheObsoleteAfterSeconds = 60 * 60 * 24 * 120
 
-  public maxConcurrentRequests = 300
+  public maxConcurrentRequests = 10
 
   constructor(public readonly log: Logger) {
     super(log)

services/apps/members_enrichment_worker/src/workflows/enrichMember.ts

@@ -27,10 +27,10 @@ const {
 } = proxyActivities<typeof activities>({
   startToCloseTimeout: '7 minutes',
   retry: {
-    maximumAttempts: 3,
+    maximumAttempts: 4,
     initialInterval: '5 minutes',
     backoffCoefficient: 2,
-    maximumInterval: '10 minutes',
+    maximumInterval: '20 minutes',
   },
 })
 

services/apps/members_enrichment_worker/src/workflows/getMembersToEnrich.ts

@@ -23,6 +23,7 @@ export async function getMembersToEnrich(): Promise<void> {
     MemberEnrichmentSource.PROGAI,
     MemberEnrichmentSource.CLEARBIT,
     MemberEnrichmentSource.PROGAI_LINKEDIN_SCRAPER,
+    MemberEnrichmentSource.CRUSTDATA,
   ]
 
   const members = await getEnrichableMembers(QUERY_FOR_ENRICHABLE_MEMBERS_PER_RUN, sources)