Skip to content

Commit 7b70a01

Browse files
David Engelhartclaude
andcommitted
lumopolis: untracked dependency, environment and cache folders are dropped before anything reads them, a repository nested inside the folder is mapped by its own git, names that can't be read are counted in one pane note, the walk honours the root .gitignore and streams each folder, the cap note counts the files actually mapped and survives Re-layout and export, submodules are found from the repository top, and the scenario e2e drives Open a folder and can fail on a bundled file
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 9a72171 commit 7b70a01

30 files changed

Lines changed: 1426 additions & 222 deletions

‎e2e/world-scenarios.spec.ts‎

Lines changed: 98 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,35 @@
11
/**
22
* Lumopolis over the edge-case scenario fixtures (tests/e2e/fixtures/worldScenarios.ts), in the real app with one
3-
* launch for the whole table. Each scenario is opened as the Open folder button opens a folder: main is asked to map
4-
* it as a project (world:map-project), then the pane opens on it. For each row the world must have the files a
5-
* person can see as parts (every building's members), none of the vendored ones, and no error in the pane. Last, the
6-
* renderer must have logged no console error and thrown no page error.
3+
* launch for the whole table. Most rows open their scenario the way the app opens a project's world: main is asked to
4+
* map it (world:map-project), then the pane opens on it (the termpolis:openWorld event). One row, nested-repo, goes
5+
* through the Lumopolis home's real Open a folder button instead (world:open-folder, its folder dialog answered from
6+
* TERMPOLIS_TEST_WORLD_PICK_FILE). For each row the world must have the files a person can see as parts (every
7+
* building's members), none of the vendored ones, and no error in the pane. Last, the renderer must have logged no
8+
* console error and thrown no page error.
9+
*
10+
* With WORLD_SCENARIO_HUGE=1 two more rows run: huge (50,000 files) as a plain folder, where the walk stops at its cap
11+
* and the pane says so, and as a repository, where git lists every file. Each prints the time to the first map and
12+
* every process's peak working set (app.getAppMetrics) for the record. CI does not set it: each fixture writes 50,000
13+
* files.
714
*
815
* The rows: unicode-names (a git repository, so git's quoting of non-ASCII names is in play), non-git (a plain
9-
* folder, walked), untracked (a file never added), vendored (dependency, build and minified files committed) and
10-
* submodule (recursed into).
16+
* folder, walked), untracked (a file never added), vendored (dependency, build, minified and bundled files committed),
17+
* vendored-untracked (dependency, environment, cache and build folders written and never added), submodule (recursed
18+
* into) and nested-repo (a repository inside the folder, listed by its own git).
1119
*/
1220
import { test, expect, type ElectronApplication, type Page } from '@playwright/test'
1321
import fs from 'fs'
1422
import os from 'os'
1523
import path from 'path'
1624
import { quitApp, ensureE2eBuild } from './helpers/launch'
1725
import { launchWorldApp, removeDirs, expectPaneOpens } from './helpers/world'
18-
import { makeScenario, type ScenarioName, type ScenarioOptions } from '../tests/e2e/fixtures/worldScenarios'
26+
import { makeScenario, UNTRACKED_VENDORED, type ScenarioName, type ScenarioOptions } from '../tests/e2e/fixtures/worldScenarios'
1927

2028
let app: ElectronApplication
2129
let page: Page
2230
let userDataDir: string
2331
let base: string
32+
let pickFile: string
2433
const consoleErrors: string[] = []
2534

2635
interface Row {
@@ -30,6 +39,8 @@ interface Row {
3039
parts: string[]
3140
/** Path prefixes no member may start with. */
3241
absent?: string[]
42+
/** Opened through the home's Open a folder button (world:open-folder) rather than world:map-project. */
43+
openFolder?: boolean
3344
}
3445

3546
const ROWS: Row[] = [
@@ -39,15 +50,20 @@ const ROWS: Row[] = [
3950
{
4051
name: 'vendored',
4152
parts: ['src/main.ts', 'src/util.ts'],
42-
absent: ['node_modules/', '.venv/', 'target/', 'dist/', 'site.min.css']
53+
// src/app.bundle.js is a .js file in a folder the world shows: only its name keeps it out, so this can fail.
54+
absent: ['node_modules/', '.venv/', 'target/', 'dist/', 'src/app.bundle.js']
4355
},
44-
{ name: 'submodule', parts: ['src/app.ts', 'libs/shared/index.ts', 'libs/shared/helper.ts'] }
56+
{ name: 'vendored-untracked', parts: ['src/main.ts', 'src/util.ts'], absent: UNTRACKED_VENDORED },
57+
{ name: 'submodule', parts: ['src/app.ts', 'libs/shared/index.ts', 'libs/shared/helper.ts'] },
58+
{ name: 'nested-repo', parts: ['src/app.ts', 'nested/lib.ts', 'nested/helper.ts'], openFolder: true }
4559
]
4660

4761
test.beforeAll(async () => {
4862
ensureE2eBuild()
4963
base = fs.mkdtempSync(path.join(os.tmpdir(), 'termpolis-world-scenarios-'))
64+
pickFile = path.join(base, 'pick.txt')
5065
const launched = await launchWorldApp({
66+
env: { TERMPOLIS_TEST_WORLD_PICK_FILE: pickFile },
5167
onPage: (p) => {
5268
p.on('console', (m) => { if (m.type() === 'error') consoleErrors.push(m.text()) })
5369
p.on('pageerror', (e) => consoleErrors.push(`pageerror: ${e.message}`))
@@ -64,14 +80,38 @@ test.afterAll(async () => {
6480
})
6581

6682
const pane = () => page.locator('[data-testid="world-view"]')
83+
const home = () => page.getByTestId('world-home')
84+
85+
/** The home's Open a folder button, with the folder dialog answered by `dir`: the real world:open-folder handler. */
86+
async function openFromHome(dir: string): Promise<void> {
87+
fs.writeFileSync(pickFile, dir)
88+
await page.keyboard.press('ControlOrMeta+k')
89+
const input = page.getByPlaceholder('Type a command...')
90+
await expect(input).toBeVisible()
91+
await input.fill('Open Lumopolis')
92+
await page.getByRole('button', { name: /^Open Lumopolis\s*See your project as a 3D world/ }).click()
93+
await expect(pane()).toBeVisible()
94+
// The pane opens on the home, or on the last world; the Worlds list goes home from there.
95+
if (!(await home().isVisible().catch(() => false))) await page.getByRole('combobox', { name: 'Worlds' }).selectOption('__home')
96+
await expect(home()).toBeVisible()
97+
await home().getByRole('button', { name: /^Open a folder/ }).click()
98+
await expect(home()).toHaveCount(0)
99+
}
67100

68-
/** Every building member of the world main stored for `root` (its repository's top when it is in one); null: no world yet. */
101+
/**
102+
* Every building member of the world main stored for `root`; null: no world yet. A folder opened as a folder is stored
103+
* under the folder as given, a project under its repository's top, which git may spell differently (a long name for
104+
* an 8.3 temp path), so both are asked.
105+
*/
69106
async function members(root: string): Promise<string[] | null> {
70107
return page.evaluate(async (r) => {
71108
interface Doc { buildings: { members: string[] }[] }
72109
const api = (window as unknown as { termpolis: { gitFindRoot: (c: string) => Promise<{ success: boolean; data?: string | null }>; worldGet: (c: string) => Promise<{ success: boolean; data?: Doc | null }> } }).termpolis
110+
const own = await api.worldGet(r)
111+
if (own.success && own.data) return own.data.buildings.flatMap((b) => b.members)
73112
const top = await api.gitFindRoot(r)
74-
const res = await api.worldGet(top.success && top.data ? top.data : r)
113+
if (!top.success || !top.data || top.data === r) return null
114+
const res = await api.worldGet(top.data)
75115
return res.success && res.data ? res.data.buildings.flatMap((b) => b.members) : null
76116
}, root)
77117
}
@@ -82,14 +122,18 @@ test.describe.serial('Lumopolis over the scenario fixtures', () => {
82122
test.setTimeout(90_000)
83123
const dir = await makeScenario(row.name, base, row.opts)
84124

85-
// The Open folder path: main maps the folder as a project, then the pane opens on it.
86-
const mapped = await page.evaluate(
87-
(r) => (window as unknown as { termpolis: { worldMapProject: (c: string) => Promise<{ success: boolean; error?: string }> } }).termpolis.worldMapProject(r),
88-
dir
89-
)
90-
expect(mapped.error ?? '', `world:map-project refused ${row.name}`).toBe('')
91-
expect(mapped.success).toBe(true)
92-
await page.evaluate((folder) => window.dispatchEvent(new CustomEvent('termpolis:openWorld', { detail: { folder } })), dir)
125+
if (row.openFolder) {
126+
await openFromHome(dir)
127+
} else {
128+
// The project path: main maps the folder as a project, then the pane opens on it.
129+
const mapped = await page.evaluate(
130+
(r) => (window as unknown as { termpolis: { worldMapProject: (c: string) => Promise<{ success: boolean; error?: string }> } }).termpolis.worldMapProject(r),
131+
dir
132+
)
133+
expect(mapped.error ?? '', `world:map-project refused ${row.name}`).toBe('')
134+
expect(mapped.success).toBe(true)
135+
await page.evaluate((folder) => window.dispatchEvent(new CustomEvent('termpolis:openWorld', { detail: { folder } })), dir)
136+
}
93137
await expectPaneOpens(page)
94138

95139
await expect.poll(() => members(dir), { message: `a world for ${row.name}`, timeout: 60_000 }).not.toBeNull()
@@ -105,6 +149,41 @@ test.describe.serial('Lumopolis over the scenario fixtures', () => {
105149
})
106150
}
107151

152+
for (const withGit of [false, true]) {
153+
test(`huge ${withGit ? 'repository' : 'folder'} (WORLD_SCENARIO_HUGE=1 only): ${withGit ? 'git lists every file' : 'the walk stops at its cap and the pane says so'}; time and memory are printed`, async () => {
154+
test.skip(process.env.WORLD_SCENARIO_HUGE !== '1', 'huge writes 50,000 files; set WORLD_SCENARIO_HUGE=1 to run it')
155+
test.setTimeout(900_000)
156+
const dir = await makeScenario('huge', base, { git: withGit })
157+
const started = Date.now()
158+
const mapped = await page.evaluate(
159+
(r) => (window as unknown as { termpolis: { worldMapProject: (c: string) => Promise<{ success: boolean; error?: string }> } }).termpolis.worldMapProject(r),
160+
dir
161+
)
162+
expect(mapped.error ?? '').toBe('')
163+
await page.evaluate((folder) => window.dispatchEvent(new CustomEvent('termpolis:openWorld', { detail: { folder } })), dir)
164+
await expectPaneOpens(page)
165+
await expect.poll(() => members(dir), { message: 'a world for huge', timeout: 800_000, intervals: [1000] }).not.toBeNull()
166+
const firstMapMs = Date.now() - started
167+
const got = (await members(dir))!
168+
expect(got.length).toBeGreaterThan(0)
169+
if (withGit) {
170+
expect(got.length).toBe(50_000)
171+
await expect(page.locator('[data-testid="world-notice"]', { hasText: 'Showing the first ' })).toHaveCount(0)
172+
} else {
173+
expect(got.length).toBeLessThanOrEqual(20_000)
174+
await expect(page.locator('[data-testid="world-notice"]')).toContainText('Showing the first ', { timeout: 60_000 })
175+
}
176+
const peaks = await app.evaluate(({ app: eapp }) =>
177+
eapp.getAppMetrics().map((m) => ({ process: m.name ? `${m.type}:${m.name}` : m.type, peakMB: Math.round(m.memory.peakWorkingSetSize / 1024) }))
178+
)
179+
const report = { git: withGit, firstMapMs, mapped: got.length, peaks }
180+
console.log(`[huge] ${JSON.stringify(report)}`)
181+
test.info().annotations.push({ type: 'huge', description: JSON.stringify(report) })
182+
await page.locator('[data-testid="world-close"]').click()
183+
await expect(pane()).toHaveCount(0)
184+
})
185+
}
186+
108187
test('the renderer logged no console errors and threw no page errors', async () => {
109188
expect(consoleErrors).toEqual([])
110189
})

‎src/main/codeGraph.ts‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import * as fs from 'fs'
1616
import * as path from 'path'
1717
import { extractFile, extractReferences, languageForFile, type CodeSymbol, type SymbolKind, type FileExtract } from './codeGraphExtract'
1818
import { extractFileTS } from './codeGraphTreeSitter'
19-
import { isIndexableCodeFile, discoverRepoFiles } from './codeIngest'
19+
import { isIndexableCodeFile, discoverRepoFiles, repoListingNotes } from './codeIngest'
2020
import { projectKeyOf } from './projectKey'
2121
// A code-graph JSON is one object (not line-streamable), but reading a file past the V8 string limit
2222
// as 'utf8' fatals uncatchably (see fileLines.ts). Guard the whole-file reads below with this cliff.
@@ -666,8 +666,10 @@ export interface GraphSnapshot {
666666
symbols: SymbolHit[]
667667
callEdges: CodeGraphEdge[]
668668
fileImports: Record<string, string[]>
669-
/** Set when the files come from a folder walk that stopped at its cap: how many files it listed. */
669+
/** Set when the listing stopped at its cap (a folder walk, or nested repositories): how many files are mapped. */
670670
filesCapped?: number
671+
/** Listed names that are not valid UTF-8, so no read finds them: the world counts the source files among them. */
672+
unreadable?: string[]
671673
}
672674

673675
const slashed = (p: string): string => p.replace(/\\/g, '/')
@@ -721,7 +723,13 @@ export function graphSnapshot(projectKey: string, root?: string, opts: GraphSnap
721723
const r = rel(f)
722724
if (r !== null) fileImports[r] = [...imps]
723725
}
724-
if (opts.structureOnly === true) return { root: base, files, symbols: [], callEdges: [], fileImports }
726+
// What the repository's listing could not show (codeIngest.ts discoverRepoFiles), for the world's pane note.
727+
const notes = root ? repoListingNotes(base) : null
728+
const said = {
729+
...(notes?.capped ? { filesCapped: files.length } : {}),
730+
...(notes && notes.unreadable.length > 0 ? { unreadable: [...notes.unreadable] } : {})
731+
}
732+
if (opts.structureOnly === true) return { root: base, files, symbols: [], callEdges: [], fileImports, ...said }
725733
const idMap = new Map<string, string>()
726734
const symbols: SymbolHit[] = []
727735
for (const s of st.symbolsById.values()) {
@@ -739,7 +747,7 @@ export function graphSnapshot(projectKey: string, root?: string, opts: GraphSnap
739747
const to = idMap.get(e.to)
740748
if (from !== undefined && to !== undefined) callEdges.push({ from, to })
741749
}
742-
return { root: base, files, symbols, callEdges, fileImports }
750+
return { root: base, files, symbols, callEdges, fileImports, ...said }
743751
}
744752

745753
/** The projectKey of the graph most recently built/queried (the repo the user is in). */

‎src/main/codeIngest.ts‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import { normalizeNewlines } from './lineEndings'
1414
import { join } from 'path'
1515
import { isIndexableCodeFile } from './codeFileFilter'
1616
import { safeGitAsync } from './gitCommand'
17-
import { lsFilesArgs, lsSubmoduleArgs, mergeListings, parseLsFiles } from './repoFileList'
17+
import { listRepo } from './repoFileList'
1818
import { knownHashes } from './conversationIngest' // one membership-resolution rule for both ingesters
1919

2020
export interface CodeChunk {
@@ -162,27 +162,42 @@ export async function ingestCode(deps: CodeIngestDeps): Promise<CodeIngestStats>
162162
}
163163

164164
// The files a person sees in repoRoot: tracked ones plus untracked ones the ignore files allow (so node_modules and
165-
// dist stay out), and the tracked files of its submodules. Absolute paths. Returns [] if not a git repo / git missing.
165+
// dist stay out), minus untracked dependency folders and virtual environments the ignore files miss (a root-only
166+
// /node_modules rule, a venv with no .gitignore inside: never read, never indexed), the tracked files of its
167+
// submodules, and the files of nested repositories that are not submodules (repoFileList.ts listRepo). Absolute
168+
// paths. Returns [] if not a git repo / git missing.
166169
export async function discoverRepoFiles(repoRoot: string): Promise<string[]> {
167170
if (!repoRoot) return []
168-
const git = (args: string[]): Promise<string> =>
169-
safeGitAsync(args, { cwd: repoRoot, timeout: LIST_TIMEOUT_MS, maxBuffer: 64 * 1024 * 1024 })
170-
let own: string[]
171-
try {
172-
// The proc host runs git (safeGitAsync): the spawn happens off the main thread. This used to start git here with
173-
// child_process.execFile, whose spawn blocks the calling thread (0.1-0.3 s on a busy Windows laptop), on every
174-
// code re-index (a watcher event), and only fell back to the host when that failed. safeGitAsync also resolves the
175-
// binary from the common install locations when a packaged app inherits a PATH without git, which is what stops a
176-
// transient git-off-PATH from silently wiping the code graph. A REAL "not a repo" error still throws → [].
177-
own = parseLsFiles(await git(lsFilesArgs(repoRoot)))
178-
} catch {
179-
return []
180-
}
181-
// Submodules are recursed into (Lumopolis maps their code with the rest). Their files need a second listing, run
182-
// only when the repository declares one; if it fails, the repository's own files still count.
183-
const hasSubmodules = await fsp.access(join(repoRoot, '.gitmodules')).then(() => true, () => false)
184-
const recursed = hasSubmodules ? await git(lsSubmoduleArgs(repoRoot)).then(parseLsFiles, () => []) : []
185-
return mergeListings(own, recursed).map((rel) => join(repoRoot, rel))
171+
// The proc host runs git (safeGitAsync): the spawn happens off the main thread. This used to start git here with
172+
// child_process.execFile, whose spawn blocks the calling thread (0.1-0.3 s on a busy Windows laptop), on every
173+
// code re-index (a watcher event), and only fell back to the host when that failed. safeGitAsync also resolves the
174+
// binary from the common install locations when a packaged app inherits a PATH without git, which is what stops a
175+
// transient git-off-PATH from silently wiping the code graph. A REAL "not a repo" error still lists nothing.
176+
const git = (args: string[]): Promise<string | null> =>
177+
safeGitAsync(args, { cwd: repoRoot, timeout: LIST_TIMEOUT_MS, maxBuffer: 64 * 1024 * 1024 }).then(
178+
(out) => out,
179+
() => null
180+
)
181+
const exists = (p: string): Promise<boolean> => fsp.access(p).then(() => true, () => false)
182+
const listed = await listRepo(repoRoot, { git, exists })
183+
if (listed === null) return []
184+
listingNotes.set(notesKey(repoRoot), { capped: listed.capped, unreadable: listed.unreadable })
185+
return listed.files.map((rel) => join(repoRoot, rel))
186+
}
187+
188+
/** What the last listing of a repository could not show: nested repositories past the cap, names that are not UTF-8. */
189+
export interface RepoListingNotes {
190+
capped: boolean
191+
/** Repository-relative names (at most repoFileList's MAP_FILE_CAP). */
192+
unreadable: string[]
193+
}
194+
195+
const listingNotes = new Map<string, RepoListingNotes>()
196+
const notesKey = (root: string): string => root.replace(/\\/g, '/').replace(/\/+$/, '').toLowerCase()
197+
198+
/** The notes of the last discoverRepoFiles of `root` (Lumopolis says them in its pane); null before any listing. */
199+
export function repoListingNotes(root: string): RepoListingNotes | null {
200+
return listingNotes.get(notesKey(root)) ?? null
186201
}
187202

188203
/** A listing of a large repository with many untracked folders walks them all: more time than git's usual 10 s. */

0 commit comments

Comments
 (0)