Feature: cache the checksums of files that Igir writes

docs/advanced/file-cache.md

@@ -14,6 +14,21 @@ Igir caches the following file operations:
 
 The results are stored using the file's absolute path. Igir stores and checks if the file's size or modified timestamp has changed since the cached result was calculated, and if there's a mismatch, will recalculate the file operation.
 
+## Files that Igir writes
+
+Because results are stored by file path, a file that Igir [copies](../commands.md#copy) or [moves](../commands.md#move) to a new location wouldn't have a cached result at its new path, and would need to be read again on the next run.
+
+To avoid this, Igir caches what it already knows about every file it writes:
+
+- When copying or moving files, the output file has the same contents as the input file, so the input file's checksums are cached for the output file's path
+- When [writing zip files](../output/writing-archives.md), Igir already knows the checksums of every entry it wrote, so they're cached for the output zip's path
+
+This means a subsequent run that uses the output directory as an input directory won't need to re-read those files.
+
+Igir won't cache results for files whose contents it changed while writing them, because the input file's checksums no longer describe the output file. That includes [removing headers](../roms/headers.md), [applying patches](../roms/patching.md), and [restoring padding](../roms/trim-detection.md) to trimmed ROMs.
+
+Cached results for written files are only as trustworthy as the write that produced them. Use the [`test` command](../commands.md#test) if you want Igir to verify what it wrote before trusting it.
+
 ## File format
 
 The cache is a gzipped JSON file. You can explore the contents of it with commands such as:

src/cache/fileCache.ts

@@ -193,6 +193,49 @@ export default class FileCache {
     return await File.fileOfObject(filePath, cachedFile);
   }
 
+  /**
+   * Cache the already-known checksums of a file that was just written, so that subsequent runs
+   * don't need to read the file again to compute them.
+   *
+   * The caller is responsible for ensuring that {@link file} describes the bytes that were
+   * actually written to {@link filePath}, i.e. that the file wasn't transformed while being
+   * written.
+   */
+  async setFileChecksums(filePath: string, file: File): Promise<void> {
+    if (
+      file.getCrc32() === undefined &&
+      file.getMd5() === undefined &&
+      file.getSha1() === undefined &&
+      file.getSha256() === undefined
+    ) {
+      // We don't know any checksums, there's nothing worth caching
+      return;
+    }
+
+    const stats = await FsUtil.stat(filePath);
+    if (stats.size !== file.getSize()) {
+      // The file on disk isn't what we expected to write, don't cache incorrect checksums
+      this.prefixedLogger.trace(
+        `${filePath}: not caching checksums, real size ${stats.size} !== expected size ${file.getSize()}`,
+      );
+      return;
+    }
+
+    this.prefixedLogger.trace(`${filePath}: caching checksums of written file`);
+    await this.cache.set(this.getCacheKey(filePath, undefined, ValueType.FILE_CHECKSUMS), {
+      fileSize: stats.size,
+      modifiedTimeSec: stats.mtimeS,
+      value: {
+        filePath,
+        size: stats.size,
+        crc32: file.getCrc32(),
+        md5: file.getMd5(),
+        sha1: file.getSha1(),
+        sha256: file.getSha256(),
+      } satisfies FileProps,
+    });
+  }
+
   async getOrComputeArchiveChecksums<T extends Archive>(
     archive: T,
     checksumBitmask: number,
@@ -288,6 +331,43 @@ export default class FileCache {
     );
   }
 
+  /**
+   * Cache the already-known entries of an archive that was just written, so that subsequent runs
+   * don't need to decompress the archive again to compute them.
+   *
+   * The caller is responsible for ensuring that {@link entries} describes every entry that was
+   * actually written to {@link archive}.
+   */
+  async setArchiveChecksums<T extends Archive>(
+    archive: T,
+    entries: ArchiveEntry<T>[],
+  ): Promise<void> {
+    if (entries.length === 0) {
+      // Zero entries are treated as a cache miss, don't bother caching them
+      return;
+    }
+
+    const stats = await FsUtil.stat(archive.getFilePath());
+    if (stats.size === 0) {
+      // An empty file can't have entries
+      return;
+    }
+
+    this.prefixedLogger.trace(`${archive.getFilePath()}: caching entries of written archive`);
+    await this.cache.set(
+      this.getCacheKey(
+        archive.getFilePath(),
+        archive.constructor.name,
+        ValueType.ARCHIVE_CHECKSUMS,
+      ),
+      {
+        fileSize: stats.size,
+        modifiedTimeSec: stats.mtimeS,
+        value: entries.map((entry) => entry.toEntryProps()),
+      },
+    );
+  }
+
   async getOrComputeFileHeader(file: File): Promise<ROMHeader | undefined> {
     return await this.getOrComputeAny(
       file,

src/factories/fileFactory.ts

@@ -340,6 +340,25 @@ export default class FileFactory {
     return await this.fileCache.getOrComputeFilePaddings(file, callback);
   }
 
+  /**
+   * Cache the already-known checksums of a file that was just written to {@link filePath}, so
+   * that subsequent runs don't need to read the file again to compute them.
+   */
+  async cacheFileChecksums(filePath: string, file: File): Promise<void> {
+    await this.fileCache.setFileChecksums(filePath, file);
+  }
+
+  /**
+   * Cache the already-known entries of an archive that was just written, so that subsequent runs
+   * don't need to decompress the archive again to compute them.
+   */
+  async cacheArchiveChecksums<T extends Archive>(
+    archive: T,
+    entries: ArchiveEntry<T>[],
+  ): Promise<void> {
+    await this.fileCache.setArchiveChecksums(archive, entries);
+  }
+
   /**
    * Return the TorrentZip validation result for a zip file, indicating whether its structure
    * conforms to the TorrentZip specification.

src/modules/candidates/candidateWriter.ts

@@ -158,6 +158,18 @@ export default class CandidateWriter extends Module {
     }
   }
 
+  /**
+   * Returns true if the file's contents will be modified while it's being written, meaning the
+   * input file's checksums won't describe the bytes that end up in the output file.
+   */
+  private static isTransformedWhenWritten(inputRomFile: File): boolean {
+    return (
+      inputRomFile.getFileHeader() !== undefined ||
+      inputRomFile.getPatch() !== undefined ||
+      inputRomFile.getPaddings().length > 0
+    );
+  }
+
   /**
    ***********************
    *
@@ -289,6 +301,7 @@ export default class CandidateWriter extends Module {
       }
 
       if (wasWritten) {
+        await this.cacheWrittenZip(dat, candidate, outputZip, inputToOutputZipEntries);
         for (const [inputRomFile] of inputToOutputZipEntries) {
           this.enqueueFileDeletion(candidate, inputRomFile);
         }
@@ -298,6 +311,39 @@ export default class CandidateWriter extends Module {
     }
   }
 
+  /**
+   * Cache the entries of a zip that was just written. We already know the checksums of every
+   * entry we wrote, so subsequent runs don't need to decompress the zip to know them.
+   */
+  private async cacheWrittenZip(
+    dat: DAT,
+    candidate: WriteCandidate,
+    outputZip: Zip,
+    inputToOutputZipEntries: [File, ArchiveEntry<Zip>][],
+  ): Promise<void> {
+    if (
+      inputToOutputZipEntries.some(([inputRomFile]) =>
+        CandidateWriter.isTransformedWhenWritten(inputRomFile),
+      )
+    ) {
+      // At least one entry's contents were modified while being written, we don't know the
+      // output's checksums
+      return;
+    }
+
+    try {
+      await this.fileFactory.cacheArchiveChecksums(
+        outputZip,
+        inputToOutputZipEntries.map(([, outputEntry]) => outputEntry),
+      );
+    } catch (error) {
+      // Caching is only an optimization, a failure here shouldn't fail the write
+      this.prefixedLogger.trace(
+        `${dat.getName()}: ${candidate.getName()}: ${outputZip.getFilePath()}: failed to cache written zip's entries: ${error}`,
+      );
+    }
+  }
+
   private async testZipContents(
     dat: DAT,
     candidate: WriteCandidate,
@@ -667,13 +713,40 @@ export default class CandidateWriter extends Module {
       }
 
       if (written) {
+        await this.cacheWrittenRaw(dat, candidate, inputRomFile, outputFilePath);
         this.enqueueFileDeletion(candidate, inputRomFile);
       }
     } finally {
       childBar.delete();
     }
   }
 
+  /**
+   * Cache the checksums of a raw file that was just written. Moving and copying files doesn't
+   * change their contents, so the input file's checksums are also the output file's checksums,
+   * and subsequent runs don't need to re-read the output file to know them.
+   */
+  private async cacheWrittenRaw(
+    dat: DAT,
+    candidate: WriteCandidate,
+    inputRomFile: File,
+    outputFilePath: string,
+  ): Promise<void> {
+    if (CandidateWriter.isTransformedWhenWritten(inputRomFile)) {
+      // The file's contents were modified while being written, we don't know the output's checksums
+      return;
+    }
+
+    try {
+      await this.fileFactory.cacheFileChecksums(outputFilePath, inputRomFile);
+    } catch (error) {
+      // Caching is only an optimization, a failure here shouldn't fail the write
+      this.prefixedLogger.trace(
+        `${dat.getName()}: ${candidate.getName()}: ${outputFilePath}: failed to cache written file's checksums: ${error}`,
+      );
+    }
+  }
+
   private async moveRawFile(
     dat: DAT,
     candidate: WriteCandidate,