diff --git a/docs/advanced/file-cache.md b/docs/advanced/file-cache.md index 549572595..74e31556e 100644 --- a/docs/advanced/file-cache.md +++ b/docs/advanced/file-cache.md @@ -14,6 +14,21 @@ Igir caches the following file operations: The results are stored using the file's absolute path. Igir stores and checks if the file's size or modified timestamp has changed since the cached result was calculated, and if there's a mismatch, will recalculate the file operation. +## Files that Igir writes + +Because results are stored by file path, a file that Igir [copies](../commands.md#copy) or [moves](../commands.md#move) to a new location wouldn't have a cached result at its new path, and would need to be read again on the next run. + +To avoid this, Igir caches what it already knows about every file it writes: + +- When copying or moving files, the output file has the same contents as the input file, so the input file's checksums are cached for the output file's path +- When [writing zip files](../output/writing-archives.md), Igir already knows the checksums of every entry it wrote, so they're cached for the output zip's path + +This means a subsequent run that uses the output directory as an input directory won't need to re-read those files. + +Igir won't cache results for files whose contents it changed while writing them, because the input file's checksums no longer describe the output file. That includes [removing headers](../roms/headers.md), [applying patches](../roms/patching.md), and [restoring padding](../roms/trim-detection.md) to trimmed ROMs. + +Cached results for written files are only as trustworthy as the write that produced them. Use the [`test` command](../commands.md#test) if you want Igir to verify what it wrote before trusting it. + ## File format The cache is a gzipped JSON file. You can explore the contents of it with commands such as: diff --git a/src/cache/fileCache.ts b/src/cache/fileCache.ts index e241c97eb..ae473f4bf 100644 --- a/src/cache/fileCache.ts +++ b/src/cache/fileCache.ts @@ -193,6 +193,49 @@ export default class FileCache { return await File.fileOfObject(filePath, cachedFile); } + /** + * Cache the already-known checksums of a file that was just written, so that subsequent runs + * don't need to read the file again to compute them. + * + * The caller is responsible for ensuring that {@link file} describes the bytes that were + * actually written to {@link filePath}, i.e. that the file wasn't transformed while being + * written. + */ + async setFileChecksums(filePath: string, file: File): Promise { + if ( + file.getCrc32() === undefined && + file.getMd5() === undefined && + file.getSha1() === undefined && + file.getSha256() === undefined + ) { + // We don't know any checksums, there's nothing worth caching + return; + } + + const stats = await FsUtil.stat(filePath); + if (stats.size !== file.getSize()) { + // The file on disk isn't what we expected to write, don't cache incorrect checksums + this.prefixedLogger.trace( + `${filePath}: not caching checksums, real size ${stats.size} !== expected size ${file.getSize()}`, + ); + return; + } + + this.prefixedLogger.trace(`${filePath}: caching checksums of written file`); + await this.cache.set(this.getCacheKey(filePath, undefined, ValueType.FILE_CHECKSUMS), { + fileSize: stats.size, + modifiedTimeSec: stats.mtimeS, + value: { + filePath, + size: stats.size, + crc32: file.getCrc32(), + md5: file.getMd5(), + sha1: file.getSha1(), + sha256: file.getSha256(), + } satisfies FileProps, + }); + } + async getOrComputeArchiveChecksums( archive: T, checksumBitmask: number, @@ -288,6 +331,43 @@ export default class FileCache { ); } + /** + * Cache the already-known entries of an archive that was just written, so that subsequent runs + * don't need to decompress the archive again to compute them. + * + * The caller is responsible for ensuring that {@link entries} describes every entry that was + * actually written to {@link archive}. + */ + async setArchiveChecksums( + archive: T, + entries: ArchiveEntry[], + ): Promise { + if (entries.length === 0) { + // Zero entries are treated as a cache miss, don't bother caching them + return; + } + + const stats = await FsUtil.stat(archive.getFilePath()); + if (stats.size === 0) { + // An empty file can't have entries + return; + } + + this.prefixedLogger.trace(`${archive.getFilePath()}: caching entries of written archive`); + await this.cache.set( + this.getCacheKey( + archive.getFilePath(), + archive.constructor.name, + ValueType.ARCHIVE_CHECKSUMS, + ), + { + fileSize: stats.size, + modifiedTimeSec: stats.mtimeS, + value: entries.map((entry) => entry.toEntryProps()), + }, + ); + } + async getOrComputeFileHeader(file: File): Promise { return await this.getOrComputeAny( file, diff --git a/src/factories/fileFactory.ts b/src/factories/fileFactory.ts index a8f5b3098..05833fcbf 100644 --- a/src/factories/fileFactory.ts +++ b/src/factories/fileFactory.ts @@ -340,6 +340,25 @@ export default class FileFactory { return await this.fileCache.getOrComputeFilePaddings(file, callback); } + /** + * Cache the already-known checksums of a file that was just written to {@link filePath}, so + * that subsequent runs don't need to read the file again to compute them. + */ + async cacheFileChecksums(filePath: string, file: File): Promise { + await this.fileCache.setFileChecksums(filePath, file); + } + + /** + * Cache the already-known entries of an archive that was just written, so that subsequent runs + * don't need to decompress the archive again to compute them. + */ + async cacheArchiveChecksums( + archive: T, + entries: ArchiveEntry[], + ): Promise { + await this.fileCache.setArchiveChecksums(archive, entries); + } + /** * Return the TorrentZip validation result for a zip file, indicating whether its structure * conforms to the TorrentZip specification. diff --git a/src/modules/candidates/candidateWriter.ts b/src/modules/candidates/candidateWriter.ts index 055566f11..ac428a4d0 100644 --- a/src/modules/candidates/candidateWriter.ts +++ b/src/modules/candidates/candidateWriter.ts @@ -158,6 +158,18 @@ export default class CandidateWriter extends Module { } } + /** + * Returns true if the file's contents will be modified while it's being written, meaning the + * input file's checksums won't describe the bytes that end up in the output file. + */ + private static isTransformedWhenWritten(inputRomFile: File): boolean { + return ( + inputRomFile.getFileHeader() !== undefined || + inputRomFile.getPatch() !== undefined || + inputRomFile.getPaddings().length > 0 + ); + } + /** *********************** * @@ -289,6 +301,7 @@ export default class CandidateWriter extends Module { } if (wasWritten) { + await this.cacheWrittenZip(dat, candidate, outputZip, inputToOutputZipEntries); for (const [inputRomFile] of inputToOutputZipEntries) { this.enqueueFileDeletion(candidate, inputRomFile); } @@ -298,6 +311,39 @@ export default class CandidateWriter extends Module { } } + /** + * Cache the entries of a zip that was just written. We already know the checksums of every + * entry we wrote, so subsequent runs don't need to decompress the zip to know them. + */ + private async cacheWrittenZip( + dat: DAT, + candidate: WriteCandidate, + outputZip: Zip, + inputToOutputZipEntries: [File, ArchiveEntry][], + ): Promise { + if ( + inputToOutputZipEntries.some(([inputRomFile]) => + CandidateWriter.isTransformedWhenWritten(inputRomFile), + ) + ) { + // At least one entry's contents were modified while being written, we don't know the + // output's checksums + return; + } + + try { + await this.fileFactory.cacheArchiveChecksums( + outputZip, + inputToOutputZipEntries.map(([, outputEntry]) => outputEntry), + ); + } catch (error) { + // Caching is only an optimization, a failure here shouldn't fail the write + this.prefixedLogger.trace( + `${dat.getName()}: ${candidate.getName()}: ${outputZip.getFilePath()}: failed to cache written zip's entries: ${error}`, + ); + } + } + private async testZipContents( dat: DAT, candidate: WriteCandidate, @@ -667,6 +713,7 @@ export default class CandidateWriter extends Module { } if (written) { + await this.cacheWrittenRaw(dat, candidate, inputRomFile, outputFilePath); this.enqueueFileDeletion(candidate, inputRomFile); } } finally { @@ -674,6 +721,32 @@ export default class CandidateWriter extends Module { } } + /** + * Cache the checksums of a raw file that was just written. Moving and copying files doesn't + * change their contents, so the input file's checksums are also the output file's checksums, + * and subsequent runs don't need to re-read the output file to know them. + */ + private async cacheWrittenRaw( + dat: DAT, + candidate: WriteCandidate, + inputRomFile: File, + outputFilePath: string, + ): Promise { + if (CandidateWriter.isTransformedWhenWritten(inputRomFile)) { + // The file's contents were modified while being written, we don't know the output's checksums + return; + } + + try { + await this.fileFactory.cacheFileChecksums(outputFilePath, inputRomFile); + } catch (error) { + // Caching is only an optimization, a failure here shouldn't fail the write + this.prefixedLogger.trace( + `${dat.getName()}: ${candidate.getName()}: ${outputFilePath}: failed to cache written file's checksums: ${error}`, + ); + } + } + private async moveRawFile( dat: DAT, candidate: WriteCandidate, diff --git a/test/cache/fileCache.test.ts b/test/cache/fileCache.test.ts index 0fef44731..716062712 100644 --- a/test/cache/fileCache.test.ts +++ b/test/cache/fileCache.test.ts @@ -2,10 +2,15 @@ import path from 'node:path'; import FileCache from '../../src/cache/fileCache.js'; import Temp from '../../src/globals/temp.js'; +import ArchiveEntry from '../../src/models/files/archives/archiveEntry.js'; import Zip from '../../src/models/files/archives/zip.js'; +import File from '../../src/models/files/file.js'; import { ChecksumBitmask } from '../../src/models/files/fileChecksums.js'; import FsUtil from '../../src/utils/fsUtil.js'; +const RAW_FILE_PATH = path.join('test', 'fixtures', 'roms', 'raw', 'fizzbuzz.nes'); +const ZIP_FILE_PATH = path.join('test', 'fixtures', 'roms', 'zip', 'foobar.zip'); + describe('loadFile', () => { it('should load after saving', async () => { const tempCache = await FsUtil.mktemp(path.join(Temp.getTempDir(), 'cache')); @@ -13,17 +18,131 @@ describe('loadFile', () => { await fileCache.loadFile(tempCache); // Compute some values - await fileCache.getOrComputeFileChecksums( - path.join('test', 'fixtures', 'roms', 'raw', 'fizzbuzz.nes'), + await fileCache.getOrComputeFileChecksums(RAW_FILE_PATH, ChecksumBitmask.CRC32); + await fileCache.getOrComputeArchiveChecksums(new Zip(ZIP_FILE_PATH), ChecksumBitmask.CRC32); + + await fileCache.save(); + await fileCache.loadFile(tempCache); + }); +}); + +describe('setFileChecksums', () => { + it('should return the cached checksums without reading the file', async () => { + const fileCache = new FileCache(); + const size = await FsUtil.size(RAW_FILE_PATH); + + // Given checksums that don't match the file's real contents + const bogusFile = await File.fileOf( + { filePath: RAW_FILE_PATH, size, crc32: '00000001' }, ChecksumBitmask.CRC32, ); - await fileCache.getOrComputeArchiveChecksums( - new Zip(path.join('test', 'fixtures', 'roms', 'zip', 'foobar.zip')), + await fileCache.setFileChecksums(RAW_FILE_PATH, bogusFile); + + // When + const cachedFile = await fileCache.getOrComputeFileChecksums( + RAW_FILE_PATH, ChecksumBitmask.CRC32, ); - await fileCache.save(); - await fileCache.loadFile(tempCache); + // Then the cached checksums were returned, proving the file wasn't re-read + expect(cachedFile.getCrc32()).toEqual('00000001'); + expect(cachedFile.getSize()).toEqual(size); + }); + + it('should recompute checksums that were cached without every checksum', async () => { + const fileCache = new FileCache(); + const size = await FsUtil.size(RAW_FILE_PATH); + const realFile = await new FileCache().getOrComputeFileChecksums( + RAW_FILE_PATH, + ChecksumBitmask.CRC32 | ChecksumBitmask.SHA1, + ); + + // Given only a CRC32 was cached + const crc32OnlyFile = await File.fileOf( + { filePath: RAW_FILE_PATH, size, crc32: realFile.getCrc32() }, + ChecksumBitmask.CRC32, + ); + await fileCache.setFileChecksums(RAW_FILE_PATH, crc32OnlyFile); + + // When a SHA1 is also needed + const computedFile = await fileCache.getOrComputeFileChecksums( + RAW_FILE_PATH, + ChecksumBitmask.CRC32 | ChecksumBitmask.SHA1, + ); + + // Then the file was re-read + expect(computedFile.getSha1()).toEqual(realFile.getSha1()); + }); + + it("should not cache checksums when the file's size doesn't match", async () => { + const fileCache = new FileCache(); + const size = await FsUtil.size(RAW_FILE_PATH); + const realFile = await new FileCache().getOrComputeFileChecksums( + RAW_FILE_PATH, + ChecksumBitmask.CRC32, + ); + + // Given checksums for a file of a different size + const wrongSizeFile = await File.fileOf( + { filePath: RAW_FILE_PATH, size: size + 1, crc32: '00000001' }, + ChecksumBitmask.CRC32, + ); + await fileCache.setFileChecksums(RAW_FILE_PATH, wrongSizeFile); + + // When + const computedFile = await fileCache.getOrComputeFileChecksums( + RAW_FILE_PATH, + ChecksumBitmask.CRC32, + ); + + // Then the checksums were computed from the real file + expect(computedFile.getCrc32()).toEqual(realFile.getCrc32()); + }); +}); + +describe('setArchiveChecksums', () => { + it('should return the cached entries without reading the archive', async () => { + const fileCache = new FileCache(); + const zip = new Zip(ZIP_FILE_PATH); + + // Given entries that don't match the archive's real contents + const bogusEntry = await ArchiveEntry.entryOf( + { archive: zip, entryPath: 'bogus.rom', size: 7, crc32: '00000002' }, + ChecksumBitmask.CRC32, + ); + await fileCache.setArchiveChecksums(zip, [bogusEntry]); + + // When + const cachedEntries = await fileCache.getOrComputeArchiveChecksums(zip, ChecksumBitmask.CRC32); + + // Then the cached entries were returned, proving the archive wasn't re-read + expect(cachedEntries).toHaveLength(1); + expect(cachedEntries[0].getEntryPath()).toEqual('bogus.rom'); + expect(cachedEntries[0].getCrc32()).toEqual('00000002'); + }); + + it('should not cache zero entries', async () => { + const fileCache = new FileCache(); + const zip = new Zip(ZIP_FILE_PATH); + const realEntries = await new FileCache().getOrComputeArchiveChecksums( + zip, + ChecksumBitmask.CRC32, + ); + expect(realEntries.length).toBeGreaterThan(0); + + // Given + await fileCache.setArchiveChecksums(zip, []); + + // When + const computedEntries = await fileCache.getOrComputeArchiveChecksums( + zip, + ChecksumBitmask.CRC32, + ); + + // Then the entries were read from the real archive + expect(computedEntries.map((entry) => entry.getEntryPath())).toEqual( + realEntries.map((entry) => entry.getEntryPath()), + ); }); }); diff --git a/test/modules/candidates/candidateWriter.test.ts b/test/modules/candidates/candidateWriter.test.ts index a74a6f08f..95a940d70 100644 --- a/test/modules/candidates/candidateWriter.test.ts +++ b/test/modules/candidates/candidateWriter.test.ts @@ -2,6 +2,7 @@ import type { Stats } from 'node:fs'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import zlib from 'node:zlib'; import async from 'async'; @@ -100,6 +101,7 @@ async function candidateWriter( inputGlob: string, patchGlob: string | undefined, outputTemp: string, + writerFileCache: FileCache = new FileCache(), ): Promise { const options = new Options({ ...optionsProps, @@ -185,12 +187,25 @@ async function candidateWriter( return await new CandidateWriter( options, new ProgressBarFake(), - new FileFactory(new FileCache()), + new FileFactory(writerFileCache), writerSemaphore, new FileMoveMutex(), ).write(dat, candidates); } +/** + * Save {@link fileCache} to a temp file and return the keys it persisted. + */ +async function savedCacheKeys(fileCache: FileCache, cacheFilePath: string): Promise { + await fileCache.save(); + if (!(await FsUtil.exists(cacheFilePath))) { + // Nothing was cached, so nothing was saved + return []; + } + const gunzipped = zlib.gunzipSync(await FsUtil.readFile(cacheFilePath)); + return Object.keys(JSON.parse(gunzipped.toString('utf8')) as Record); +} + it('should not do anything if there are no candidates', async () => { await copyFixturesToTemp(async (inputTemp, outputTemp) => { // Given @@ -2140,3 +2155,82 @@ describe('link', () => { }); }); }); + +describe('file cache', () => { + test.each([['copy'], ['move']])( + 'should cache the checksums of raw written files: %s', + async (command) => { + await copyFixturesToTemp(async (inputTemp, outputTemp) => { + // Given + const options = new Options({ commands: [command] }); + const fileCache = new FileCache(); + const cacheFilePath = await FsUtil.mktemp(path.join(Temp.getTempDir(), 'cache')); + await fileCache.loadFile(cacheFilePath); + + // When + await candidateWriter(options, inputTemp, 'raw/*', undefined, outputTemp, fileCache); + + // Then every written file's checksums were cached, so they won't need to be recalculated + const outputFiles = await walkAndStat(outputTemp); + expect(outputFiles.length).toBeGreaterThan(0); + const cacheKeys = await savedCacheKeys(fileCache, cacheFilePath); + for (const [outputFile] of outputFiles) { + const outputFilePath = path.resolve(outputTemp, outputFile); + expect(cacheKeys.some((key) => key.includes(`|${outputFilePath}|`))).toEqual(true); + } + }); + }, + ); + + test.each([['copy'], ['move']])( + 'should cache the entries of written zips: %s', + async (command) => { + await copyFixturesToTemp(async (inputTemp, outputTemp) => { + // Given + const options = new Options({ commands: [command, 'zip'] }); + const fileCache = new FileCache(); + const cacheFilePath = await FsUtil.mktemp(path.join(Temp.getTempDir(), 'cache')); + await fileCache.loadFile(cacheFilePath); + + // When + await candidateWriter(options, inputTemp, 'raw/*', undefined, outputTemp, fileCache); + + // Then every written zip's entries were cached, so they won't need to be re-read + const outputFiles = await walkAndStat(outputTemp); + expect(outputFiles.length).toBeGreaterThan(0); + const cacheKeys = await savedCacheKeys(fileCache, cacheFilePath); + for (const [outputFile] of outputFiles) { + const outputFilePath = path.resolve(outputTemp, outputFile); + expect(cacheKeys.some((key) => key.includes(`|${outputFilePath}|Zip|`))).toEqual(true); + } + }); + }, + ); + + it('should not cache the checksums of files that had their header removed', async () => { + await copyFixturesToTemp(async (inputTemp, outputTemp) => { + // Given + const options = new Options({ commands: ['copy'], removeHeaders: [''] }); + const fileCache = new FileCache(); + const cacheFilePath = await FsUtil.mktemp(path.join(Temp.getTempDir(), 'cache')); + await fileCache.loadFile(cacheFilePath); + + // When + await candidateWriter( + options, + inputTemp, + 'headered/allpads.nes', + undefined, + outputTemp, + fileCache, + ); + + // Then the written file's checksums differ from the input's, so they weren't cached + const outputFiles = await walkAndStat(outputTemp); + expect(outputFiles).toHaveLength(1); + const cacheKeys = await savedCacheKeys(fileCache, cacheFilePath); + const outputFilePath = path.resolve(outputTemp, outputFiles[0][0]); + expect(cacheKeys.some((key) => key.includes(`|${outputFilePath}|`))).toEqual(false); + }); + }); +});