Skip to content

Commit 195f0bc

Browse files
David Engelhartclaude
andcommitted
lumopolis: no sync fs or code-graph build on main; compile re-reads only what changed
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 5a6fc9d commit 195f0bc

31 files changed

Lines changed: 1886 additions & 616 deletions

‎src/main/codeFileFilter.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Which repository files belong in a code index: no secrets (the sensitive-path rules), no binaries
2+
// or media, no minified bundles or lock files. Pure, so the world utility process applies the same
3+
// rule as the main-process code ingest.
4+
import { matchSensitiveFile } from './sensitivePaths'
5+
6+
// Non-text / generated artifacts that pollute a code index.
7+
const SKIP_EXT =
8+
/\.(png|jpe?g|gif|webp|ico|bmp|svg|pdf|zip|gz|tgz|tar|7z|rar|exe|dll|so|dylib|node|wasm|onnx|bin|woff2?|ttf|eot|otf|mp[34]|mov|avi|webm|class|jar|pyc|map)$/i
9+
const SKIP_NAME = /(\.min\.(js|css)|package-lock\.json|yarn\.lock|pnpm-lock\.yaml)$/i
10+
11+
export function isIndexableCodeFile(filePath: string): boolean {
12+
if (!filePath || typeof filePath !== 'string') return false
13+
if (matchSensitiveFile(filePath) !== null) return false // secrets — never index
14+
if (SKIP_EXT.test(filePath)) return false
15+
if (SKIP_NAME.test(filePath)) return false
16+
return true
17+
}

‎src/main/codeIngest.ts‎

Lines changed: 211 additions & 221 deletions
Large diffs are not rendered by default.

‎src/main/index.ts‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ import { configureConversationIngest, getIngestRuntime, makeDrainRun, passResult
326326
import { noteTyping, whenTypingQuiet } from './typingQuiet'
327327
import { runCodeIngest, discoverRepoFiles } from './codeIngest'
328328
import { outlineHint, countLinesIn, OUTLINE_MAX_FILE_BYTES, OUTLINE_MIN_FILE_BYTES } from './outlineHint'
329-
import { initCodeGraph, buildCodeGraph, reindexWatchedChange, fileSymbols, codeExplore, codeCallers, codeCallees, codeImpact, codeSymbols, codeGraphStats, graphKeyForRoot, graphSnapshot, resolveCodeRefs, resolveToken, ALL_REPOS, type CodeRef } from './codeGraph'
329+
import { initCodeGraph, buildCodeGraph, reindexWatchedChange, fileSymbols, codeExplore, codeCallers, codeCallees, codeImpact, codeSymbols, codeGraphStats, graphKeyForRoot, graphSnapshot, resolveCodeRefs, resolveToken, ALL_REPOS, type CodeRef, type GraphSnapshot } from './codeGraph'
330330
import type { ExecFn } from './mnemeDistiller'
331331
import { createWorldClient } from './world/worldClient'
332332
import { confinedReader, isCloneRoot, isFolderEmpty } from './world/folderSafety'
@@ -2595,11 +2595,29 @@ ipcMain.handle('memory:ingest-code', async (_, opts: { repoRoot: string }) => {
25952595
} catch (e: any) { return err(e.message) }
25962596
})
25972597

2598+
/** Indexes `root` into the code graph (async reads, paused while the app sheds load). A cloned repository is
2599+
* someone else's content: a committed symlink never leads a read out of the clone. */
2600+
function buildGraphFor(root: string) {
2601+
const read = (f: string): Promise<string> => fsPromises.readFile(f, 'utf8')
2602+
return buildCodeGraph({ listFiles: () => discoverRepoFiles(root), readFile: isCloneRoot(worldUserData, root) ? confinedReader(root, read) : read, pause: () => whenNotShed() }, graphKeyForRoot(root))
2603+
}
2604+
/** Background indexes started for a first world, one per graph key at a time. */
2605+
const worldGraphBuilds = new Map<string, Promise<void>>()
2606+
function indexInBackground(root: string): void {
2607+
const key = graphKeyForRoot(root)
2608+
if (worldGraphBuilds.has(key)) return
2609+
const run = buildGraphFor(root)
2610+
.then(() => undefined)
2611+
.catch((e: unknown) => console.error('[world] background index failed:', (e as Error)?.message ?? e))
2612+
.finally(() => worldGraphBuilds.delete(key))
2613+
worldGraphBuilds.set(key, run)
2614+
}
2615+
25982616
// Native code-graph IPC (for the app UI / on-demand build + structural queries).
25992617
ipcMain.handle('code-graph:build', async (_, opts: { repoRoot: string }) => {
26002618
try {
26012619
if (!opts?.repoRoot) return err('repoRoot required')
2602-
return ok(await buildCodeGraph({ listFiles: () => discoverRepoFiles(opts.repoRoot), readFile: (f) => fsPromises.readFile(f, 'utf8'), pause: () => whenNotShed() }, graphKeyForRoot(opts.repoRoot)))
2620+
return ok(await buildGraphFor(opts.repoRoot))
26032621
} catch (e: any) { return err(e.message) }
26042622
})
26052623
ipcMain.handle('memory:anomalies', async (_, opts?: { limit?: number }) => { try { return ok({ anomalies: getAnomalies(opts?.limit ?? 100), total: anomalyCount() }) } catch (e: any) { return err(e.message) } })
@@ -2615,6 +2633,8 @@ ipcMain.handle('code-graph:impact', async (_, opts: { name: string }) => { try {
26152633
// environment only main owns. world:get / world:compile / world:relayout, plus world:progress and
26162634
// world:updated pushes and the world:settings-get/-set pair, are registered by world/ipc.ts.
26172635
const worldSettings = createWorldSettings(() => join(app.getPath('userData'), WORLD_SETTINGS_FILE))
2636+
// Loaded off the main thread at startup; until it lands get() answers the defaults, and a labelling call waits for it.
2637+
void worldSettings.ready()
26182638
/** `claude -p` for labelling, with the same launch environment (token proxy included) an agent gets in
26192639
* `root`. Spawned by the proc host, never on the main thread: a watcher-driven recompile can label. */
26202640
const worldClaudeExec = (root: string): ExecFn => async (cmd, args, opts) => {
@@ -2677,7 +2697,9 @@ const worldClient = createWorldClient(undefined, {
26772697
// The cheapest model of the Lumopolis agent (worldAgent): Haiku for Claude, a mini or flash model for the
26782698
// others. Claude goes through worldClaudeExec; Codex and Gemini through runHeadless, read-only, with the
26792699
// same run environment (Codex's proxy URL) a question gets.
2680-
llm: (root, prompt) => {
2700+
llm: async (root, prompt) => {
2701+
// The saved agent and consent, never the defaults, decide a model call.
2702+
await worldSettings.ready()
26812703
const agent = worldSettings.get().worldAgent
26822704
return makeWorldLlm(agent, cheapModelFor(agent, modelCatalog), {
26832705
claudeExec: worldClaudeExec(root),
@@ -2699,17 +2721,18 @@ const worldClient = createWorldClient(undefined, {
26992721
return { head: '', dirty: false }
27002722
}
27012723
},
2702-
// A project nobody indexed yet is indexed first, so a first world never compiles from nothing. Still
2703-
// nothing (an unmounted drive, an emptied folder) is an error, never an empty world saved over the
2704-
// stored one: that would throw away every stored position. The ipc handler answers err(message).
2705-
snapshot: async (root) => {
2724+
// A project the code graph has not indexed yet compiles from a snapshot the world process makes (it
2725+
// lists and parses the files there, never on the main thread), and main's own index starts in the
2726+
// background (not awaited) for the next compile and the code tools. Still nothing (an unmounted
2727+
// drive, an emptied folder) is an error, never an empty world saved over the stored one: that would
2728+
// throw away every stored position. The ipc handler answers err(message).
2729+
snapshot: async (root): Promise<GraphSnapshot> => {
27062730
const key = graphKeyForRoot(root)
2731+
// graphSnapshot reads the in-memory graph only: no file or database access on this path.
27072732
const snap = graphSnapshot(key, root, { structureOnly: true })
27082733
if (snap.files.length > 0) return snap
2709-
// A cloned repository is someone else's content: a committed symlink never leads a read out of the clone.
2710-
const read = (f: string): Promise<string> => fsPromises.readFile(f, 'utf8')
2711-
await buildCodeGraph({ listFiles: () => discoverRepoFiles(root), readFile: isCloneRoot(worldUserData, root) ? confinedReader(root, read) : read, pause: () => whenNotShed() }, key)
2712-
const built = graphSnapshot(key, root, { structureOnly: true })
2734+
const built = await worldClient.snapshotFromFiles(root)
2735+
indexInBackground(root)
27132736
// A Build it folder starts empty: its world is the linked design's plan alone until files arrive.
27142737
if (built.files.length === 0) {
27152738
const source = (await worldSources.get(key))?.source

‎src/main/sensitiveFileWatcher.ts‎

Lines changed: 2 additions & 245 deletions
Original file line numberDiff line numberDiff line change
@@ -38,260 +38,17 @@
3838
// - Reverse-DNS or otherwise enrich the path. We just record the basename.
3939

4040
import * as path from 'path'
41-
import * as os from 'os'
4241
import { subscribe, type AgentEvent } from './agentEventBus'
42+
import { matchSensitiveFile, type SensitiveMatch } from './sensitivePaths'
4343

44-
export interface SensitiveMatch {
45-
/** Original path the agent reported reading */
46-
filePath: string
47-
/** Which rule matched (e.g. 'dotenv', 'aws-credentials') */
48-
rule: string
49-
/** Human-readable label (e.g. '.env file', 'AWS credentials') */
50-
label: string
51-
/** Tool name that triggered (Read, Bash, Edit, …) */
52-
tool: string
53-
/** Source heuristic: 'path' (direct file_path argument) or 'command' (parsed from shell command) */
54-
source: 'path' | 'command'
55-
}
44+
export { RULES, matchSensitiveFile, type SensitiveMatch } from './sensitivePaths'
5645

5746
export interface SensitiveReadEvent extends SensitiveMatch {
5847
ts: number
5948
terminalId: string
6049
agent: string
6150
}
6251

63-
interface RuleDef {
64-
id: string
65-
label: string
66-
/** test the basename / full path; return true if it matches */
67-
match: (basename: string, fullPath: string) => boolean
68-
}
69-
70-
// We test against the BASENAME first (cheap), then the FULL PATH (for
71-
// rules anchored to ~/.ssh, ~/.aws, ~/.config, etc). Both tests get
72-
// normalised to lower-case on Windows because filesystems there are
73-
// case-insensitive — agents will sometimes report "C:\Users\…\.AWS\credentials"
74-
// with mixed case.
75-
76-
const HOME = (() => {
77-
try { return os.homedir() } catch { return '' }
78-
})()
79-
80-
function norm(p: string): string {
81-
if (!p) return ''
82-
let q = p.trim()
83-
// strip surrounding quotes the agent may have included
84-
if ((q.startsWith('"') && q.endsWith('"')) || (q.startsWith("'") && q.endsWith("'"))) {
85-
q = q.slice(1, -1)
86-
}
87-
// expand ~ and ~/ to homedir for matching purposes
88-
if (q === '~' || q.startsWith('~/') || q.startsWith('~\\')) {
89-
q = path.join(HOME, q.slice(2))
90-
}
91-
// unify path sep on Windows for full-path checks
92-
return q.replace(/\\/g, '/')
93-
}
94-
95-
const KEY_BASENAMES_CASE_SENSITIVE = new Set([
96-
'id_rsa', 'id_ed25519', 'id_ecdsa', 'id_dsa', 'identity', 'authorized_keys',
97-
])
98-
99-
export const RULES: RuleDef[] = [
100-
// --- .env family ---
101-
// Match .env, .env.local, .env.production, etc.
102-
// EXCLUDE .env.example, .env.sample, .env.template
103-
{
104-
id: 'dotenv',
105-
label: '.env file',
106-
match: (base) => {
107-
const b = base.toLowerCase()
108-
if (b === '.env') return true
109-
if (!b.startsWith('.env.')) return false
110-
const tail = b.slice(5)
111-
if (!tail) return false
112-
// Common safe-to-share suffixes
113-
if (['example', 'sample', 'template', 'dist', 'tpl'].includes(tail)) return false
114-
return true
115-
},
116-
},
117-
118-
// --- Private keys (OpenSSH, OpenSSL, PGP) ---
119-
{
120-
id: 'private-key-pem',
121-
label: 'PEM-format private key',
122-
match: (base) => /\.(pem|key)$/i.test(base) && !/\.pub$/i.test(base),
123-
},
124-
{
125-
id: 'pkcs12',
126-
label: 'PKCS#12 keystore',
127-
match: (base) => /\.(p12|pfx)$/i.test(base),
128-
},
129-
{
130-
id: 'jks',
131-
label: 'Java keystore',
132-
match: (base) => /\.(jks|keystore)$/i.test(base),
133-
},
134-
{
135-
id: 'ssh-private-key',
136-
label: 'SSH private key',
137-
match: (base) => {
138-
// explicit names — id_rsa, id_ed25519, etc — but NOT *.pub
139-
if (/\.pub$/i.test(base)) return false
140-
const b = base
141-
if (KEY_BASENAMES_CASE_SENSITIVE.has(b)) return true
142-
// id_<algo>(_xxx) without .pub
143-
if (/^id_[a-z0-9]+(?:_[a-z0-9]+)*$/i.test(b) && !/\.pub$/i.test(b)) return true
144-
return false
145-
},
146-
},
147-
148-
// --- Cloud provider creds ---
149-
{
150-
id: 'aws-credentials',
151-
label: 'AWS credentials file',
152-
match: (base, full) => {
153-
const b = base.toLowerCase()
154-
if (b !== 'credentials' && b !== 'config') return false
155-
// Only flag when path includes `.aws`
156-
return /\/\.aws\//i.test(full) || /\\\.aws\\/i.test(full) || full.toLowerCase().includes('/.aws/')
157-
},
158-
},
159-
{
160-
id: 'gcp-service-account',
161-
label: 'GCP service-account key',
162-
match: (base) => {
163-
const b = base.toLowerCase()
164-
if (!b.endsWith('.json')) return false
165-
return (
166-
b === 'credentials.json' ||
167-
b === 'service-account.json' ||
168-
b === 'gcp-key.json' ||
169-
b.startsWith('service-account-') ||
170-
b.startsWith('gcp-key-') ||
171-
b.startsWith('google-credentials')
172-
)
173-
},
174-
},
175-
{
176-
id: 'azure-credentials',
177-
label: 'Azure credentials file',
178-
match: (base) => /^azure[-_]?(credentials|profile)\.(json|txt)$/i.test(base),
179-
},
180-
181-
// --- SSH directory contents (anything under ~/.ssh except .pub and 'config' alone) ---
182-
{
183-
id: 'ssh-dir-content',
184-
label: 'file inside ~/.ssh',
185-
match: (base, full) => {
186-
if (!/\/\.ssh\//i.test(full) && !/\\\.ssh\\/i.test(full)) return false
187-
const b = base.toLowerCase()
188-
if (b === 'known_hosts' || b === 'known_hosts.old') return false
189-
if (b === 'config') return false
190-
if (b.endsWith('.pub')) return false
191-
// Avoid double-flagging items already caught by ssh-private-key
192-
return true
193-
},
194-
},
195-
196-
// --- Auth tokens / configs in dotfiles ---
197-
{ id: 'netrc', label: '.netrc', match: (base) => /^_?\.?netrc(\.gpg)?$/i.test(base) },
198-
{ id: 'npmrc', label: '.npmrc', match: (base) => /^_?\.?npmrc$/i.test(base) },
199-
{ id: 'pypirc', label: '.pypirc', match: (base) => /^_?\.?pypirc$/i.test(base) },
200-
{
201-
id: 'docker-config',
202-
label: 'docker config (registry creds)',
203-
match: (base, full) => base.toLowerCase() === 'config.json' && /\/\.docker\//i.test(full),
204-
},
205-
{
206-
id: 'kube-config',
207-
label: 'kubeconfig',
208-
match: (base, full) => {
209-
const b = base.toLowerCase()
210-
if (b === 'config' && /\/\.kube\//i.test(full)) return true
211-
if (b === 'kubeconfig' || b.endsWith('.kubeconfig')) return true
212-
return false
213-
},
214-
},
215-
216-
// --- Generic secret-named files ---
217-
{
218-
id: 'secrets-file',
219-
label: 'secrets configuration file',
220-
match: (base) => /^secrets?\.(ya?ml|json|env|toml|ini)$/i.test(base),
221-
},
222-
{
223-
id: 'credentials-file',
224-
label: 'credentials configuration file',
225-
match: (base) => /^credentials?\.(ya?ml|json|env|toml|ini)$/i.test(base),
226-
},
227-
{
228-
id: 'database-url',
229-
label: 'database URL config',
230-
match: (base) => /^database(?:_|-)?(?:url|password)\.(?:env|txt|json)$/i.test(base),
231-
},
232-
233-
// --- GnuPG / KeePass ---
234-
{
235-
id: 'gpg-private',
236-
label: 'GnuPG private key store',
237-
match: (base, full) => {
238-
const b = base.toLowerCase()
239-
// ONLY the PUBLIC keyrings are excluded. `secring.gpg` used to be listed here too, which
240-
// made this rule DEAD for the one file it exists to catch: the exclusion returned false
241-
// before the match below could ever return true, so an agent reading your PRIVATE GnuPG
242-
// keyring was never flagged. It had been grouped in with the pubring.* entries by mistake,
243-
// and the failure mode was SILENCE — the worst possible failure mode for a watcher.
244-
if (b === 'pubring.kbx' || b === 'pubring.gpg') return false
245-
if (/\.gnupg\//i.test(full) && (b === 'secring.gpg' || b.endsWith('.key'))) return true
246-
return false
247-
},
248-
},
249-
{
250-
id: 'keepass-db',
251-
label: 'KeePass database',
252-
match: (base) => /\.(kdbx|kdb)$/i.test(base),
253-
},
254-
255-
// --- Browser cookies / session stores ---
256-
{
257-
id: 'browser-cookies',
258-
label: 'browser cookies database',
259-
match: (base, full) => {
260-
const b = base.toLowerCase()
261-
if (b !== 'cookies' && b !== 'cookies.sqlite' && b !== 'cookies.db') return false
262-
const f = full.toLowerCase()
263-
return /chrome|firefox|edge|safari|brave|chromium/.test(f)
264-
},
265-
},
266-
]
267-
268-
export function matchSensitiveFile(filePath: string): SensitiveMatch | null {
269-
if (!filePath || typeof filePath !== 'string') return null
270-
const normalized = norm(filePath)
271-
if (!normalized) return null
272-
const base = path.basename(normalized)
273-
if (!base) return null
274-
// Skip URL-shaped strings (http://, file://, …) — agents sometimes pass URLs
275-
if (/^[a-z]+:\/\//i.test(filePath)) return null
276-
277-
for (const rule of RULES) {
278-
try {
279-
if (rule.match(base, normalized)) {
280-
return {
281-
filePath,
282-
rule: rule.id,
283-
label: rule.label,
284-
tool: 'Read',
285-
source: 'path',
286-
}
287-
}
288-
} catch {
289-
// a rule throwing must never crash the watcher
290-
}
291-
}
292-
return null
293-
}
294-
29552
const FS_TOOLS = new Set([
29653
'Read', 'read_file', 'view', 'open_file', 'fs_read', 'fs.read',
29754
'Edit', 'Write', 'NotebookEdit', 'edit_file', 'write_file',

0 commit comments

Comments
 (0)