Implement Gmail activity tracking with optional metadata extraction

ActivityWatch is used in timesheet tracking, and knowing just "reading" or "composing" email info is not very useful on its own. Extracting involved email metadata (Sender, Recipients, Subject) helps determine the context of the activity relative to project models and workflows in the used software.

A new setting has been added to allow users to enable or disable Gmail tracking.

Also added a build.sh script to simplify the build and test process for Chrome and Firefox.

ui related changes: ActivityWatch/aw-webui#796

Author: RaoufGhrissi

build.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+make build-chrome && \
+mkdir -p artifacts/chrome && \
+unzip -o artifacts/chrome.zip -d artifacts/chrome
+
+make build-firefox && \
+mkdir -p artifacts/firefox && \
+unzip -o artifacts/firefox.zip -d artifacts/firefox

src/background/client.ts

@@ -6,7 +6,10 @@ import { emitNotification, getBrowser, logHttpError } from './helpers'
 import { getHostname, getSyncStatus, setSyncStatus } from '../storage'
 
 export const getClient = () =>
-  new AWClient('aw-client-web', { testing: config.isDevelopment })
+  new AWClient('aw-client-web', {
+    testing: config.isDevelopment,
+    baseURL: (config as any).baseURL, // for testing locally, will be removed once PR ready
+  })
 
 // TODO: We might want to get the hostname somehow, maybe like this:
 // https://stackoverflow.com/questions/28223087/how-can-i-allow-firefox-or-chrome-to-read-a-pcs-hostname-or-other-assignable

src/background/heartbeat.ts

@@ -3,9 +3,73 @@ import { getActiveWindowTab, getTab, getTabs } from './helpers'
 import config from '../config'
 import { AWClient, IEvent } from 'aw-client'
 import { getBucketId, sendHeartbeat } from './client'
-import { getEnabled, getHeartbeatData, setHeartbeatData } from '../storage'
+import { getEnabled, getHeartbeatData, setHeartbeatData, getGmailEnabled } from '../storage'
 import deepEqual from 'deep-equal'
 
+export function setupMessageListener(client: AWClient) {
+  browser.runtime.onMessage.addListener(
+    async (message: any) => {
+      const enabled = await getEnabled();
+      const gmailEnabled = await getGmailEnabled();
+      if (!enabled || !gmailEnabled) return;
+
+      if (message.type === 'AW_GMAIL_HEARTBEAT') {
+        const tab = await getActiveWindowTab()
+        if (!tab || !tab.url || !tab.title) return;
+        if (!tab.url.includes('mail.google.com')) return;
+        const tabs = await getTabs();
+
+        const data: IEvent['data'] = {
+          url: tab.url,
+          title: tab.title,
+          audible: tab.audible ?? false,
+          incognito: tab.incognito,
+          tabCount: tabs.length,
+          ...message.data,
+        };
+        await performHeartbeat(client, data);
+      }
+    },
+  )
+}
+
+async function performHeartbeat(
+  client: AWClient,
+  data: IEvent['data'],
+  options: { finalizeOnly?: boolean } = {}
+) {
+  const bucketId = await getBucketId()
+  const now = new Date()
+  const previousData = await getHeartbeatData()
+  if (previousData && !deepEqual(previousData, data)) {
+    console.debug('[Background] Activity changed, finalizing previous session', previousData)
+    await sendHeartbeat(
+      client,
+      bucketId,
+      new Date(now.getTime() - 1),
+      previousData,
+      config.heartbeat.intervalInSeconds + 20,
+    ).catch(() => {})
+  }
+
+  if (options.finalizeOnly) {
+     return;
+  }
+
+  console.debug('[Background] Sending heartbeat', data)
+  await sendHeartbeat(
+    client,
+    bucketId,
+    now,
+    data,
+    config.heartbeat.intervalInSeconds + 20,
+  ).catch((err: unknown) => {
+    console.error('[Background] Failed to send heartbeat:', err);
+  })
+  
+  await setHeartbeatData(data)
+}
+
 async function heartbeat(
   client: AWClient,
   tab: browser.Tabs.Tab | undefined,
@@ -27,34 +91,23 @@ async function heartbeat(
     return
   }
 
-  const now = new Date()
   const data: IEvent['data'] = {
     url: tab.url,
     title: tab.title,
     audible: tab.audible ?? false,
     incognito: tab.incognito,
     tabCount: tabCount,
   }
-  const previousData = await getHeartbeatData()
-  if (previousData && !deepEqual(previousData, data)) {
-    console.debug('Sending heartbeat for previous data', previousData)
-    await sendHeartbeat(
-      client,
-      await getBucketId(),
-      new Date(now.getTime() - 1),
-      previousData,
-      config.heartbeat.intervalInSeconds + 20,
-    )
+
+  const gmailEnabled = await getGmailEnabled();
+  if (gmailEnabled && tab.url.includes('mail.google.com')) {
+    // Sharp cut: finalize the previous activity (e.g. if we came from Google Search)
+    // but don't start the 'Generic' Gmail event. Gmail.ts will do that with metadata.
+    await performHeartbeat(client, data, { finalizeOnly: true });
+    return;
   }
-  console.debug('Sending heartbeat', data)
-  await sendHeartbeat(
-    client,
-    await getBucketId(),
-    now,
-    data,
-    config.heartbeat.intervalInSeconds + 20,
-  )
-  await setHeartbeatData(data)
+
+  await performHeartbeat(client, data);
 }
 
 export const sendInitialHeartbeat = async (client: AWClient) => {
@@ -76,9 +129,9 @@ export const heartbeatAlarmListener =
 
 export const tabActivatedListener =
   (client: AWClient) =>
-  async (activeInfo: browser.Tabs.OnActivatedActiveInfoType) => {
-    const tab = await getTab(activeInfo.tabId)
-    const tabs = await getTabs()
-    console.debug('Sending heartbeat for tab activation', tab)
-    await heartbeat(client, tab, tabs.length)
-  }
+    async (activeInfo: browser.Tabs.OnActivatedActiveInfoType) => {
+      const tab = await getTab(activeInfo.tabId)
+      const tabs = await getTabs()
+      console.debug('Sending heartbeat for tab activation', tab)
+      await heartbeat(client, tab, tabs.length)
+    }

src/background/main.ts

@@ -4,6 +4,7 @@ import {
   heartbeatAlarmListener,
   sendInitialHeartbeat,
   tabActivatedListener,
+  setupMessageListener,
 } from './heartbeat'
 import { getClient, detectHostname } from './client'
 import {
@@ -15,6 +16,7 @@ import {
   setHostname,
   waitForEnabled,
 } from '../storage'
+import { AWClient } from 'aw-client'
 
 async function getIsConsentRequired() {
   if (!config.requireConsent) return false
@@ -24,7 +26,7 @@ async function getIsConsentRequired() {
     .catch(() => true)
 }
 
-async function autodetectHostname() {
+async function autodetectHostname(client: AWClient) {
   const hostname = await getHostname()
   if (hostname === undefined) {
     const detectedHostname = await detectHostname(client)
@@ -57,13 +59,17 @@ browser.runtime.onInstalled.addListener(async () => {
     })
   }
 
-  await autodetectHostname()
+  await autodetectHostname(client)
 })
 
 console.debug('Creating alarms and tab listeners')
 browser.alarms.create(config.heartbeat.alarmName, {
   periodInMinutes: Math.floor(config.heartbeat.intervalInSeconds / 60),
 })
+
+// Set up Gmail message listener (other watchers will be added later)
+setupMessageListener(client)
+
 browser.alarms.onAlarm.addListener(heartbeatAlarmListener(client))
 browser.tabs.onActivated.addListener(tabActivatedListener(client))
 

src/config.ts

@@ -5,6 +5,7 @@ const config = {
     alarmName: 'heartbeat',
     intervalInSeconds: 60,
   },
+  baseURL: 'http://127.0.0.1:5666', // for testing locally, will be removed once PR ready
 }
 
 export default config

src/content/gmail.ts

@@ -0,0 +1,163 @@
+import browser from 'webextension-polyfill'
+import deepEqual from 'deep-equal'
+import config from '../config'
+
+let lastData: any | null = null
+
+if (window.top === window.self) {
+
+function isExtensionValid() {
+  return typeof browser !== 'undefined' && !!browser.storage && !!browser.runtime?.id
+}
+
+
+function getComposeMetadata(form: HTMLElement) {
+  const getRecipients = (name: string) =>
+    Array.from(
+      form.querySelectorAll(`div[name="${name}"] [data-hovercard-id]`),
+    ).map((el) => el.getAttribute('data-hovercard-id'))
+      .filter(Boolean) as string[]
+
+  return {
+    gmail_activity: 'composing_email',
+    subject: (form.querySelector('input[name="subjectbox"]') as HTMLInputElement)?.value || '',
+    to: getRecipients('to'),
+    cc: getRecipients('cc'),
+    bcc: getRecipients('bcc'),
+  }
+}
+
+function sendGmailHeartbeat() {
+  if (!isExtensionValid()) {
+    stopTracking()
+    return
+  }
+  if (document.visibilityState === 'hidden') {
+    return
+  }
+
+  const hash = window.location.hash
+  // for simplity in MVP:
+  // - if many emails forms are open, we only track the first one
+  const form = document.querySelector('div[role="dialog"] form') as HTMLElement | null
+  
+  let activity = 'reading_inbox'
+  let meta: any = { gmail_activity: activity }
+
+  if (form) {
+    activity = 'composing_email'
+    meta = getComposeMetadata(form)
+  } else if (
+    hash.includes('inbox/') ||
+    hash.includes('sent/') ||
+    hash.includes('all/')
+  ) {
+    /**
+     * NOTE on Fragility: The selectors below (span.gD, .gE, h2.hP) are internal 
+     * Gmail class names. These are not part of a stable API and may change 
+     * during Gmail frontend updates. High-fidelity tracking may require 
+     * maintenance if these selectors break.
+     */
+    const fromEl = document.querySelector('span.gD')
+    const from =
+      fromEl?.getAttribute('email') ||
+      fromEl?.getAttribute('data-hovercard-id') ||
+      (fromEl as HTMLElement)?.innerText ||
+      ''
+    const to = Array.from(
+      document.querySelectorAll('.gE [email], .gE [data-hovercard-id]'),
+    )
+      .map(
+        (el) => el.getAttribute('email') || el.getAttribute('data-hovercard-id'),
+      )
+      .filter((e) => e && e !== from) as string[]
+
+    activity = 'reading_email'
+    meta = {
+      gmail_activity: activity,
+      subject: (document.querySelector('h2.hP') as HTMLElement)?.innerText || '',
+      from,
+      to,
+    }
+  }
+
+  if (!deepEqual(lastData, meta)) {
+    lastData = meta;
+    if (!isExtensionValid()) return;
+    browser.runtime.sendMessage({ 
+      type: 'AW_GMAIL_HEARTBEAT', 
+      data: meta
+    }).catch(() => {})
+  }
+}
+
+let detectIntervalId: ReturnType<typeof setInterval> | null = null
+let pulseIntervalId: ReturnType<typeof setInterval> | null = null
+
+function startTracking() {
+  if (detectIntervalId !== null) {
+    return
+  }
+  
+  detectIntervalId = setInterval(sendGmailHeartbeat, 1000)
+  pulseIntervalId = setInterval(() => {
+    if (!isExtensionValid()) {
+      stopTracking()
+      return
+    }
+    if (lastData && document.visibilityState === 'visible') {
+      try {
+        browser.runtime.sendMessage({ 
+          type: 'AW_GMAIL_HEARTBEAT', 
+          data: lastData
+        }).catch(() => {})
+      } catch (err) {
+        // Extension context invalidated
+      }
+    }
+  }, config.heartbeat.intervalInSeconds * 1000)
+  
+  sendGmailHeartbeat()
+}
+
+async function refreshTracking() {
+  if (!isExtensionValid()) {
+    return
+  }
+  try {
+    const settings = await browser.storage.local.get(['gmailEnabled', 'enabled'])
+    const shouldTrack = Boolean(settings.gmailEnabled && settings.enabled)
+    
+    if (shouldTrack) {
+      startTracking()
+    } else {
+      stopTracking()
+    }
+  } catch (err) {
+    console.error('[Gmail Content] Failed to refresh tracking state', err)
+  }
+}
+
+function stopTracking() {
+  if (detectIntervalId !== null) {
+    clearInterval(detectIntervalId)
+    detectIntervalId = null
+  }
+  if (pulseIntervalId !== null) {
+    clearInterval(pulseIntervalId)
+    pulseIntervalId = null
+  }
+  lastData = null
+}
+
+if (isExtensionValid()) {
+  refreshTracking()
+
+  browser.storage.onChanged.addListener((changes) => {
+    if ('gmailEnabled' in changes || 'enabled' in changes) {
+      refreshTracking()
+    }
+  })
+}
+
+} // if (window.top === window.self)

src/manifest.json

@@ -64,5 +64,15 @@
         "gecko": {
             "id": "{ef87d84c-2127-493f-b952-5b4e744245bc}"
         }
-    }
+    },
+    "content_scripts": [
+        {
+            "matches": [
+                "*://mail.google.com/*"
+            ],
+            "js": [
+                "src/content/gmail.ts"
+            ]
+        }
+    ]
 }