diff --git a/vulnfeeds/cmd/combine-to-osv/main.go b/vulnfeeds/cmd/combine-to-osv/main.go index 23646420f5d..1290134dd0f 100644 --- a/vulnfeeds/cmd/combine-to-osv/main.go +++ b/vulnfeeds/cmd/combine-to-osv/main.go @@ -16,8 +16,8 @@ import ( "cloud.google.com/go/storage" "github.com/google/osv/vulnfeeds/conversion" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/models" - "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/api/iterator" @@ -92,7 +92,7 @@ func main() { vulnerabilities = append(vulnerabilities, v) } - upload.Upload(ctx, "OSV files", *uploadToGCS, *outputBucketName, *overridesBucketName, *numWorkers, *osvOutputPath, vulnerabilities, *syncDeletions) + writer.UploadVulnsToGCS(ctx, "OSV files", *uploadToGCS, *outputBucketName, *overridesBucketName, *numWorkers, *osvOutputPath, vulnerabilities, *syncDeletions) } // extractCVEName extracts the CVE name from a given filename and prefix. diff --git a/vulnfeeds/cmd/converters/alpine/main.go b/vulnfeeds/cmd/converters/alpine/main.go index b0e58654b5c..226285d9193 100644 --- a/vulnfeeds/cmd/converters/alpine/main.go +++ b/vulnfeeds/cmd/converters/alpine/main.go @@ -15,8 +15,8 @@ import ( "strings" "time" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/models" - "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -64,7 +64,7 @@ func main() { } ctx := context.Background() - upload.Upload(ctx, "Alpine CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *alpineOutputPath, vulnerabilities, *syncDeletions) + writer.UploadVulnsToGCS(ctx, "Alpine CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *alpineOutputPath, vulnerabilities, *syncDeletions) logger.Info("Alpine CVE conversion succeeded.") } diff --git a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go index 9d75b7816e3..3e554cf18da 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go @@ -14,8 +14,8 @@ import ( "sync" "time" - "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/conversion/cve5" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -119,8 +119,8 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, outDir string, cnas []string cveID := cve.Metadata.CVEID logger.Info("Processing "+string(cveID), slog.String("cve", string(cveID))) - osvFile, errCVE := conversion.CreateOSVFile(cveID, outDir) - metricsFile, errMetrics := conversion.CreateMetricsFile(cveID, outDir) + osvFile, errCVE := writer.CreateOSVFile(cveID, outDir) + metricsFile, errMetrics := writer.CreateMetricsFile(cveID, outDir) if errCVE != nil || errMetrics != nil { logger.Fatal("File failed to be created for CVE", slog.String("cve", string(cveID))) } diff --git a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go index 3dc45a05c8a..70eaf67fed0 100644 --- a/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go +++ b/vulnfeeds/cmd/converters/cve/cve5/single-converter/main.go @@ -7,8 +7,8 @@ import ( "log/slog" "os" - "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/conversion/cve5" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" ) @@ -46,8 +46,8 @@ func main() { } // create the files - osvFile, errCVE := conversion.CreateOSVFile(cveID, outDir) - metricsFile, errMetrics := conversion.CreateMetricsFile(cveID, outDir) + osvFile, errCVE := writer.CreateOSVFile(cveID, outDir) + metricsFile, errMetrics := writer.CreateMetricsFile(cveID, outDir) if errCVE != nil || errMetrics != nil { logger.Fatal("File failed to be created for CVE", slog.String("cve", string(cveID))) } diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index b1bfdf4c46e..abc8e579a51 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -2,6 +2,7 @@ package main import ( + "context" "encoding/json" "flag" "fmt" @@ -14,11 +15,14 @@ import ( "slices" "sync" + "cloud.google.com/go/storage" c "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/conversion/nvd" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" + "github.com/google/osv/vulnfeeds/vulns" ) var ( @@ -30,6 +34,9 @@ var ( rejectFailed = flag.Bool("reject-failed", false, "If set, OSV records with a failed conversion outcome will not be generated.") outputMetrics = flag.Bool("output-metrics", true, "If true, output the metrics information about the conversion") cpuProfile = flag.String("cpuprofile", "", "Path to write cpu profile to file (default = no output)") + uploadToGCS = flag.Bool("upload-to-gcs", false, "If true, upload to GCS bucket instead of writing to local disk.") + outputBucket = flag.String("output-bucket", "osv-test-cve-osv-conversion", "The GCS bucket to write to.") + gcsPrefix = flag.String("gcs-prefix", "nvd-osv", "The prefix within the GCS bucket.") ) func loadCPEDictionary(productToRepo *c.VPRepoCache, f string) error { @@ -91,12 +98,24 @@ func main() { repoTagsCache := &git.RepoTagsCache{} + var bkt *storage.BucketHandle + ctx := context.Background() + if *uploadToGCS { + client, err := storage.NewClient(ctx) + if err != nil { + logger.Fatal("Failed to create GCS client", slog.Any("err", err)) + } + defer client.Close() + bkt = client.Bucket(*outputBucket) + logger.Info("GCS Client and Bucket initialized", slog.String("bucket", *outputBucket)) + } + jobs := make(chan models.NVDCVE) var wg sync.WaitGroup for range *workers { wg.Add(1) - go worker(&wg, jobs, *outDir, vpRepoCache, repoTagsCache) + go worker(ctx, &wg, jobs, bkt, *outDir, vpRepoCache, repoTagsCache) } for _, cve := range parsed.Vulnerabilities { @@ -122,7 +141,7 @@ func main() { } } -func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache *git.RepoTagsCache) models.ConversionOutcome { +func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache *git.RepoTagsCache) (*vulns.Vulnerability, *models.ConversionMetrics, models.ConversionOutcome) { metrics := &models.ConversionMetrics{ CVEID: cve.ID, CNA: "nvd", @@ -131,24 +150,73 @@ func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache *gi metrics.Repos = repos var outcome models.ConversionOutcome + var vuln *vulns.Vulnerability + var finalMetrics *models.ConversionMetrics switch *outFormat { case "OSV": - outcome = nvd.CVEToOSV(cve, repos, repoTagsCache, *outDir, metrics, *rejectFailed, *outputMetrics) + vuln, finalMetrics, outcome = nvd.CVEToOSV(cve, repos, repoTagsCache, metrics) case "PackageInfo": outcome = nvd.CVEToPackageInfo(cve, repos, repoTagsCache, *outDir, metrics) + finalMetrics = metrics } - return outcome + return vuln, finalMetrics, outcome } -func worker(wg *sync.WaitGroup, jobs <-chan models.NVDCVE, _ string, vpRepoCache *c.VPRepoCache, repoTagsCache *git.RepoTagsCache) { +func worker(ctx context.Context, wg *sync.WaitGroup, jobs <-chan models.NVDCVE, bkt *storage.BucketHandle, outDir string, vpRepoCache *c.VPRepoCache, repoTagsCache *git.RepoTagsCache) { defer wg.Done() for cve := range jobs { - outcome := processCVE(cve, vpRepoCache, repoTagsCache) + vuln, metrics, outcome := processCVE(cve, vpRepoCache, repoTagsCache) + cveID := string(cve.ID) + if outcome == models.Error { + logger.Error("Error generating OSV record", slog.String("cve", cveID), slog.String("outcome", outcome.String())) + return // Don't attempt to output files if there was an error + } + if outcome != models.Successful { - logger.Info("Failed to generate an OSV record", slog.String("cve", string(cve.ID)), slog.String("outcome", outcome.String())) + logger.Info("Failed to generate a successful OSV record", slog.String("cve", cveID), slog.String("outcome", outcome.String())) + if *rejectFailed { + return // Skip outputting OSV file + } + } else { + logger.Info("Generated OSV record for "+cveID, slog.String("cve", cveID)) + } + + if *uploadToGCS && bkt != nil { + if vuln != nil { + if err := writer.UploadVulnToGCS(ctx, bkt, *gcsPrefix, vuln.Vulnerability); err != nil { + logger.Error("Failed to upload vulnerability", slog.String("cve", vuln.Id), slog.Any("err", err)) + } + } + if *outputMetrics && metrics != nil { + if err := writer.UploadMetricsToGCS(ctx, bkt, *gcsPrefix, models.CVEID(cveID), metrics); err != nil { + logger.Error("Failed to upload metrics", slog.String("cve", cveID), slog.Any("err", err)) + } + } } else { - logger.Info("Generated OSV record for "+string(cve.ID), slog.String("cve", string(cve.ID))) + // Local file output + if vuln != nil { + osvFile, err := writer.CreateOSVFile(models.CVEID(vuln.Id), outDir) + if err != nil { + logger.Error("Failed to create OSV file locally", slog.String("cve", vuln.Id), slog.Any("err", err)) + } else { + if err := vuln.ToJSON(osvFile); err != nil { + logger.Error("Failed to write OSV file locally", slog.String("cve", vuln.Id), slog.Any("err", err)) + } + osvFile.Close() + } + } + if *outputMetrics && metrics != nil { + metricsFile, err := writer.CreateMetricsFile(models.CVEID(cveID), outDir) + if err != nil { + logger.Error("Failed to create metrics file locally", slog.String("cve", cveID), slog.Any("err", err)) + } else { + if err := writer.WriteMetricsFile(metrics, metricsFile); err != nil { + logger.Error("Failed to write metrics file locally", slog.String("cve", cveID), slog.Any("err", err)) + } + metricsFile.Close() + } + } } } } diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh index 9a77e11b49c..d8d794fc500 100755 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/run_cve_to_osv_generation.sh @@ -63,21 +63,4 @@ for (( YEAR = $(date +%Y) ; YEAR >= ${FIRST_INSCOPE_YEAR} ; YEAR-- )); do -exec cp '{}' "${WORK_DIR}/nvd2osv/gcs_stage/" \; done -# Copy (and remove any missing) results to GCS bucket, with some sanity -# checking. -objs_present=$(gcloud storage ls "${OSV_OUTPUT_GCS_PATH}" | wc -l) -objs_deleted=$(gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" | wc -l) - -threshold=$(echo "scale=2; ${objs_present} * (${SAFETY_THRESHOLD_PCT:-2} / 100)" | bc) - -# Bash can't deal with floats -if (( $(echo "${objs_deleted} > ${threshold}" | bc -l) )); then - echo "Warning. Unexpectedly high (${objs_deleted}) number of CVE records would be deleted!" >> /dev/stderr - gcloud storage rsync --checksums-only --dry-run --delete-unmatched-destination-objects "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" 2>&1 | grep "Would remove" >> /dev/stderr - # TODO: add back in once nvd-mirror issue fixed: exit 1 -fi - -echo "Copying NVD CVE records successfully converted to GCS bucket" -gcloud storage rsync --quiet --checksums-only "${WORK_DIR}/nvd2osv/gcs_stage" "${OSV_OUTPUT_GCS_PATH}" - echo "Conversion run complete" diff --git a/vulnfeeds/cmd/converters/debian/main.go b/vulnfeeds/cmd/converters/debian/main.go index f1765dbaced..956f35a5745 100644 --- a/vulnfeeds/cmd/converters/debian/main.go +++ b/vulnfeeds/cmd/converters/debian/main.go @@ -14,9 +14,9 @@ import ( "strconv" "strings" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/faulttolerant" "github.com/google/osv/vulnfeeds/models" - "github.com/google/osv/vulnfeeds/upload" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -70,7 +70,7 @@ func main() { } ctx := context.Background() - upload.Upload(ctx, "Debian CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *debianOutputPath, vulnerabilities, *syncDeletions) + writer.UploadVulnsToGCS(ctx, "Debian CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *debianOutputPath, vulnerabilities, *syncDeletions) logger.Info("Debian CVE conversion succeeded.") } diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go index 8609fee8a63..8d230a39257 100644 --- a/vulnfeeds/conversion/common.go +++ b/vulnfeeds/conversion/common.go @@ -130,49 +130,6 @@ func ConductAnalysis(year string, dir string) { } } -// CreateMetricsFile creates the initial file for the metrics record. -func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { - metricsFile := filepath.Join(vulnDir, string(id)+".metrics"+models.Extension) - f, err := os.Create(metricsFile) - if err != nil { - logger.Info("Failed to open for writing "+metricsFile, slog.String("cve", string(id)), slog.String("path", metricsFile), slog.Any("err", err)) - return nil, err - } - - return f, nil -} - -// CreateOSVFile creates the initial file for the OSV record. -func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { - outputFile := filepath.Join(vulnDir, string(id)+models.Extension) - - f, err := os.Create(outputFile) - if err != nil { - logger.Info("Failed to open for writing "+outputFile, slog.String("cve", string(id)), slog.String("path", outputFile), slog.Any("err", err)) - return nil, err - } - - return f, err -} - -func WriteMetricsFile(metrics *models.ConversionMetrics, metricsFile *os.File) error { - marshalledMetrics, err := json.MarshalIndent(&metrics, "", " ") - if err != nil { - logger.Info("Failed to marshal", slog.Any("err", err)) - return err - } - - _, err = metricsFile.Write(marshalledMetrics) - if err != nil { - logger.Warn("Failed to write", slog.String("path", metricsFile.Name()), slog.Any("err", err)) - return fmt.Errorf("failed to write %s: %w", metricsFile.Name(), err) - } - - metricsFile.Close() - - return nil -} - // GitVersionsToCommits examines repos and tries to convert versions to commits by treating them as Git tags. // Returns the resolved ranges, unresolved ranges, and successful repos involved. func GitVersionsToCommits(versionRanges []*osvschema.Range, repos []string, metrics *models.ConversionMetrics, cache *git.RepoTagsCache) ([]*osvschema.Range, []*osvschema.Range, []string) { diff --git a/vulnfeeds/conversion/nvd/converter.go b/vulnfeeds/conversion/nvd/converter.go index 5fd0aa63d3d..5bb4b6f629c 100644 --- a/vulnfeeds/conversion/nvd/converter.go +++ b/vulnfeeds/conversion/nvd/converter.go @@ -12,6 +12,7 @@ import ( "slices" c "github.com/google/osv/vulnfeeds/conversion" + "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" @@ -24,21 +25,17 @@ var ErrNoRanges = errors.New("no ranges") var ErrUnresolvedFix = errors.New("fixes not resolved to commits") -// CVEToOSV Takes an NVD CVE record and outputs an OSV file in the specified directory. -func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, directory string, metrics *models.ConversionMetrics, rejectFailed bool, outputMetrics bool) models.ConversionOutcome { +// CVEToOSV Takes an NVD CVE record and returns an OSV Vulnerability object, ConversionMetrics, and the outcome. +func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, metrics *models.ConversionMetrics) (*vulns.Vulnerability, *models.ConversionMetrics, models.ConversionOutcome) { CPEs := c.CPEs(cve) metrics.CPEs = CPEs // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. - maybeVendorName := "ENOCPE" - maybeProductName := "ENOCPE" if len(CPEs) > 0 { - CPE, err := c.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. - maybeVendorName = CPE.Vendor - maybeProductName = CPE.Product + _, err := c.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. if err != nil { metrics.AddNote("Can't generate an OSV record without valid CPE data") - return models.ConversionUnknown + return nil, metrics, models.ConversionUnknown } } @@ -52,9 +49,7 @@ func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, direc // If there are no repos, there are no commits from the refs either if len(cpeRanges) == 0 && len(repos) == 0 { metrics.SetOutcome(models.NoRepos) - outputFiles(v, directory, maybeVendorName, maybeProductName, metrics, rejectFailed, outputMetrics) - - return models.NoRepos + return v, metrics, models.NoRepos } successfulRepos := make(map[string]bool) @@ -67,15 +62,13 @@ func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, direc affected := MergeRangesAndCreateAffected(resolvedRanges, cpeRanges, nil, nil, metrics) v.Affected = append(v.Affected, affected) // Exit early - outputFiles(v, directory, maybeVendorName, maybeProductName, metrics, rejectFailed, outputMetrics) - - return models.NoRepos + return v, metrics, models.NoRepos } // If we have ranges, try to resolve them r, un, sR := processRanges(cpeRanges, repos, metrics, cache, models.VersionSourceCPE) if metrics.Outcome == models.Error { - return models.Error + return nil, metrics, models.Error } resolvedRanges = append(resolvedRanges, r...) unresolvedRanges = append(unresolvedRanges, un...) @@ -105,7 +98,7 @@ func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, direc } r, un, sR := processRanges(textRanges, repos, metrics, cache, models.VersionSourceDescription) if metrics.Outcome == models.Error { - return models.Error + return nil, metrics, models.Error } resolvedRanges = append(resolvedRanges, r...) unresolvedRanges = append(unresolvedRanges, un...) @@ -115,7 +108,7 @@ func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, direc } if len(resolvedRanges) == 0 && len(commits) == 0 { - metrics.AddNote("No ranges detected for %q", maybeProductName) + metrics.AddNote("No ranges detected") metrics.SetOutcome(models.NoRanges) } @@ -124,13 +117,11 @@ func CVEToOSV(cve models.NVDCVE, repos []string, cache *git.RepoTagsCache, direc affected := MergeRangesAndCreateAffected(resolvedRanges, unresolvedRanges, commits, keys, metrics) v.Affected = append(v.Affected, affected) - if metrics.Outcome == models.Error || (!outputMetrics && rejectFailed && metrics.Outcome != models.Successful) { - return metrics.Outcome + if metrics.Outcome == models.Error { + return nil, metrics, metrics.Outcome } - outputFiles(v, directory, maybeVendorName, maybeProductName, metrics, rejectFailed, outputMetrics) - - return metrics.Outcome + return v, metrics, metrics.Outcome } // CVEToPackageInfo takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. @@ -229,11 +220,11 @@ func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache *git.RepoTagsCach logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) - metricsFile, err := c.CreateMetricsFile(cve.ID, vulnDir) + metricsFile, err := writer.CreateMetricsFile(cve.ID, vulnDir) if err != nil { logger.Warn("Failed to create metrics file", slog.String("path", metricsFile.Name()), slog.Any("err", err)) } - err = c.WriteMetricsFile(metrics, metricsFile) + err = writer.WriteMetricsFile(metrics, metricsFile) if err != nil { logger.Warn("Failed to write metrics file", slog.String("path", metricsFile.Name()), slog.Any("err", err)) } @@ -448,52 +439,6 @@ func convertCommitToEvent(commit models.AffectedCommit) *osvschema.Event { return nil } -// outputFiles writes the OSV vulnerability record and conversion metrics to files in the specified directory. -// It creates the necessary subdirectories based on the vendor and product names and handles whether or not -// the files should be written based on the rejectFailed and outputMetrics flags. -// -// Arguments: -// - v: The OSV Vulnerability object to be written to a file. -// - dir: The base directory where the output files should be created. -// - vendor: The vendor name used to create the subdirectory. -// - product: The product name used to create the subdirectory. -// - metrics: A pointer to ConversionMetrics to be written to a metrics file. -// - rejectFailed: A boolean indicating whether to skip writing the OSV file if the conversion was not successful. -// - outputMetrics: A boolean indicating whether to write the metrics file. -func outputFiles(v *vulns.Vulnerability, dir string, vendor string, product string, metrics *models.ConversionMetrics, rejectFailed bool, outputMetrics bool) { - cveID := v.Id - vulnDir := filepath.Join(dir, vendor, product) - - if err := os.MkdirAll(vulnDir, 0755); err != nil { - logger.Info("Failed to create directory "+vulnDir, slog.String("cve", cveID), slog.String("path", vulnDir), slog.Any("err", err)) - } - - if metrics.Outcome == models.Error { - return - } - - if !rejectFailed || metrics.Outcome == models.Successful { - osvFile, errCVE := c.CreateOSVFile(models.CVEID(cveID), vulnDir) - if errCVE != nil { - logger.Fatal("File failed to be created for CVE", slog.String("cve", cveID)) - } - if err := v.ToJSON(osvFile); err != nil { - logger.Error("Failed to write", slog.Any("err", err)) - } - osvFile.Close() - } - if outputMetrics { - metricsFile, errMetrics := c.CreateMetricsFile(models.CVEID(cveID), vulnDir) - if errMetrics != nil { - logger.Fatal("File failed to be created for CVE", slog.String("cve", cveID)) - } - if err := c.WriteMetricsFile(metrics, metricsFile); err != nil { - logger.Error("Failed to write metrics", slog.Any("err", err)) - } - metricsFile.Close() - } -} - // processRanges attempts to resolve the given ranges to commits and updates the metrics accordingly. func processRanges(ranges []*osvschema.Range, repos []string, metrics *models.ConversionMetrics, cache *git.RepoTagsCache, source models.VersionSource) ([]*osvschema.Range, []*osvschema.Range, []string) { if len(ranges) == 0 { diff --git a/vulnfeeds/conversion/nvd/converter_test.go b/vulnfeeds/conversion/nvd/converter_test.go index 9410cbe0817..bf099b5b1e4 100644 --- a/vulnfeeds/conversion/nvd/converter_test.go +++ b/vulnfeeds/conversion/nvd/converter_test.go @@ -8,12 +8,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport/client" githttp "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/google/go-cmp/cmp" "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" - "github.com/ossf/osv-schema/bindings/go/osvschema" - "google.golang.org/protobuf/encoding/protojson" - "google.golang.org/protobuf/testing/protocmp" ) type roundTripperFunc func(*http.Request) (*http.Response, error) @@ -69,7 +65,7 @@ func TestCVEToOSV_429(t *testing.T) { cache := &git.RepoTagsCache{} outDir := t.TempDir() - outcome := CVEToOSV(cve, []string{"https://github.com/foo/bar"}, cache, outDir, metrics, false, false) + _, _, outcome := CVEToOSV(cve, []string{"https://github.com/foo/bar"}, cache, metrics) // It should fail because of the 429 error causing unresolved fixes if outcome != models.Error { @@ -92,61 +88,3 @@ func TestCVEToOSV_429(t *testing.T) { } } } - -func TestCVEToOSV_ReferencesDeterminism(t *testing.T) { - cve := models.NVDCVE{ - ID: "CVE-2025-12345", - References: []models.Reference{ - {URL: "https://example.com/D"}, - {URL: "https://example.com/A"}, - {URL: "https://example.com/C", Tags: []string{"Patch"}}, - {URL: "https://example.com/C"}, - {URL: "https://example.com/B", Tags: []string{"Issue Tracking"}}, - {URL: "https://example.com/E"}, - }, - Metrics: &models.CVEItemMetrics{}, - } - metrics := &models.ConversionMetrics{} - outDir := t.TempDir() - - var firstResult []*osvschema.Reference - for i := range 10 { - cache := &git.RepoTagsCache{} - CVEToOSV(cve, nil, cache, outDir, metrics, false, false) - - var b []byte - err := filepath.Walk(outDir, func(path string, info os.FileInfo, _ error) error { - if !info.IsDir() && filepath.Ext(path) == ".json" { - var fileErr error - b, fileErr = os.ReadFile(path) - if fileErr != nil { - return fileErr - } - } - - return nil - }) - if err != nil { - t.Fatalf("Failed to walk or read OSV file: %v", err) - } - - if len(b) == 0 { - t.Fatalf("Failed to find OSV file") - } - - var vuln osvschema.Vulnerability - err = protojson.Unmarshal(b, &vuln) - if err != nil { - t.Fatalf("Failed to unmarshal OSV: %v", err) - } - - if i == 0 { - firstResult = vuln.GetReferences() - continue - } - - if diff := cmp.Diff(firstResult, vuln.GetReferences(), protocmp.Transform()); diff != "" { - t.Fatalf("Iteration %d produced different references result:\n%s", i, diff) - } - } -} diff --git a/vulnfeeds/upload/cveworker.go b/vulnfeeds/conversion/writer/writer.go similarity index 70% rename from vulnfeeds/upload/cveworker.go rename to vulnfeeds/conversion/writer/writer.go index 47ef47e13f3..92b56c73de6 100644 --- a/vulnfeeds/upload/cveworker.go +++ b/vulnfeeds/conversion/writer/writer.go @@ -1,26 +1,29 @@ -// Package upload handles allocating workers to intelligently uploading OSV records to a bucket -package upload +// Package writer handles allocating workers to intelligently uploading OSV records to a bucket +package writer import ( "bytes" "context" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "io" "log/slog" "os" "path" + "path/filepath" "sync" "sync/atomic" "time" "cloud.google.com/go/storage" - "google.golang.org/api/iterator" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/google/osv/vulnfeeds/gcs-tools" + "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/google/osv/vulnfeeds/vulns" "github.com/ossf/osv-schema/bindings/go/osvschema" @@ -49,10 +52,10 @@ func writeToDisk(v *osvschema.Vulnerability, preModifiedBuf []byte, outputPrefix return nil } -// uploadToGCS uploads the vulnerability to a GCS bucket. +// uploadIfChanged uploads the vulnerability to a GCS bucket. // It returns an error if the upload failed, or ErrUploadSkipped if the upload // was intentionally avoided (e.g. because the GCS object has a matching hash). -func uploadToGCS(ctx context.Context, v *osvschema.Vulnerability, preModifiedBuf []byte, outBkt *storage.BucketHandle, outputPrefix string) error { +func uploadIfChanged(ctx context.Context, v *osvschema.Vulnerability, preModifiedBuf []byte, outBkt *storage.BucketHandle, outputPrefix string) error { vulnID := v.GetId() filename := vulnID + ".json" @@ -151,7 +154,7 @@ func handleOverride(ctx context.Context, v *osvschema.Vulnerability, overridesBk // For GCS uploads, it calculates a hash of the vulnerability (excluding the modified time) and compares it // with the existing object's hash. The vulnerability is uploaded only if the hashes differ, with the // modified time updated. This prevents updating the modified time for vulnerabilities with no content changes. -func Worker(ctx context.Context, vulnChan <-chan *osvschema.Vulnerability, outBkt, overridesBkt *storage.BucketHandle, outputPrefix string, counter *atomic.Uint64) { +func VulnWorker(ctx context.Context, vulnChan <-chan *osvschema.Vulnerability, outBkt, overridesBkt *storage.BucketHandle, outputPrefix string, counter *atomic.Uint64) { for v := range vulnChan { vulnID := v.GetId() if len(v.GetAffected()) == 0 { @@ -187,7 +190,7 @@ func Worker(ctx context.Context, vulnChan <-chan *osvschema.Vulnerability, outBk writeErr = writeToDisk(vulnToProcess, preModifiedBuf, outputPrefix) } else { // Upload to GCS - writeErr = uploadToGCS(ctx, vulnToProcess, preModifiedBuf, outBkt, outputPrefix) + writeErr = uploadIfChanged(ctx, vulnToProcess, preModifiedBuf, outBkt, outputPrefix) } if writeErr == nil { @@ -203,8 +206,8 @@ func Worker(ctx context.Context, vulnChan <-chan *osvschema.Vulnerability, outBk } } -// Upload delegates workers to upload vulnerabilities to the buckets. -func Upload( +// UploadVulnsToGCS delegates workers to upload vulnerabilities to the buckets. +func UploadVulnsToGCS( ctx context.Context, jobName string, uploadToGCS bool, @@ -238,7 +241,7 @@ func Upload( wg.Add(1) go func() { defer wg.Done() - Worker(ctx, vulnChan, outBkt, overridesBkt, osvOutputPath, &successCount) + VulnWorker(ctx, vulnChan, outBkt, overridesBkt, osvOutputPath, &successCount) }() } @@ -254,7 +257,7 @@ func Upload( func handleDeletion(ctx context.Context, outBkt *storage.BucketHandle, osvOutputPath string, vulnerabilities []*osvschema.Vulnerability) { // Check if any need to be deleted - bucketObjects, err := listBucketObjects(ctx, outBkt, osvOutputPath) + bucketObjects, err := gcs.ListBucketObjects(ctx, outBkt, osvOutputPath) if err != nil { logger.Error("Failed to list bucket objects for deletion check, skipping deletion.", slog.Any("err", err)) return @@ -276,21 +279,79 @@ func handleDeletion(ctx context.Context, outBkt *storage.BucketHandle, osvOutput } } -// listBucketObjects lists the names of all objects in a Google Cloud Storage bucket. -// It does not download the file contents. -func listBucketObjects(ctx context.Context, bucket *storage.BucketHandle, prefix string) ([]string, error) { - it := bucket.Objects(ctx, &storage.Query{Prefix: prefix}) - var filenames []string - for { - attrs, err := it.Next() - if errors.Is(err, iterator.Done) { - break // All objects have been listed. - } - if err != nil { - return nil, fmt.Errorf("bucket.Objects: %w", err) - } - filenames = append(filenames, attrs.Name) +// UploadVulnToGCS marshals a single OSV Vulnerability to JSON and uploads it to GCS. +func UploadVulnToGCS(ctx context.Context, bkt *storage.BucketHandle, prefix string, vuln *osvschema.Vulnerability) error { + if vuln == nil || vuln.GetId() == "" { + return errors.New("invalid vulnerability provided") + } + + data, err := protojson.MarshalOptions{Indent: " "}.Marshal(vuln) + if err != nil { + return fmt.Errorf("failed to marshal vulnerability %s: %w", vuln.GetId(), err) + } + + objectName := filepath.Join(prefix, vuln.GetId()+".json") + reader := bytes.NewReader(data) + + return gcs.UploadToGCS(ctx, bkt, objectName, reader, "application/json") +} + +// UploadMetricsToGCS marshals ConversionMetrics to JSON and uploads it to GCS. +func UploadMetricsToGCS(ctx context.Context, bkt *storage.BucketHandle, prefix string, cveID models.CVEID, metrics *models.ConversionMetrics) error { + if metrics == nil || cveID == "" { + return errors.New("invalid metrics or CVE ID provided") + } + + data, err := json.MarshalIndent(metrics, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal metrics for %s: %w", cveID, err) + } + + objectName := filepath.Join(prefix, string(cveID)+".metrics.json") + reader := bytes.NewReader(data) + + return gcs.UploadToGCS(ctx, bkt, objectName, reader, "application/json") +} + +// CreateMetricsFile creates the initial file for the metrics record. +func CreateMetricsFile(id models.CVEID, vulnDir string) (*os.File, error) { + metricsFile := filepath.Join(vulnDir, string(id)+".metrics"+models.Extension) + f, err := os.Create(metricsFile) + if err != nil { + logger.Info("Failed to open for writing "+metricsFile, slog.String("cve", string(id)), slog.String("path", metricsFile), slog.Any("err", err)) + return nil, err + } + + return f, nil +} + +// CreateOSVFile creates the initial file for the OSV record. +func CreateOSVFile(id models.CVEID, vulnDir string) (*os.File, error) { + outputFile := filepath.Join(vulnDir, string(id)+models.Extension) + + f, err := os.Create(outputFile) + if err != nil { + logger.Info("Failed to open for writing "+outputFile, slog.String("cve", string(id)), slog.String("path", outputFile), slog.Any("err", err)) + return nil, err + } + + return f, err +} + +func WriteMetricsFile(metrics *models.ConversionMetrics, metricsFile *os.File) error { + marshalledMetrics, err := json.MarshalIndent(&metrics, "", " ") + if err != nil { + logger.Info("Failed to marshal", slog.Any("err", err)) + return err } - return filenames, nil + _, err = metricsFile.Write(marshalledMetrics) + if err != nil { + logger.Warn("Failed to write", slog.String("path", metricsFile.Name()), slog.Any("err", err)) + return fmt.Errorf("failed to write %s: %w", metricsFile.Name(), err) + } + + metricsFile.Close() + + return nil } diff --git a/vulnfeeds/upload/cveworker_test.go b/vulnfeeds/conversion/writer/writer_test.go similarity index 96% rename from vulnfeeds/upload/cveworker_test.go rename to vulnfeeds/conversion/writer/writer_test.go index adaa84132f3..3cedf9b6a5b 100644 --- a/vulnfeeds/upload/cveworker_test.go +++ b/vulnfeeds/conversion/writer/writer_test.go @@ -1,4 +1,4 @@ -package upload +package writer import ( "bytes" @@ -59,7 +59,7 @@ func TestUploadToGCS(t *testing.T) { preModifiedBuf := []byte(`{"id":"CVE-2023-1234"}`) t.Run("Upload new object", func(t *testing.T) { - err := uploadToGCS(ctx, v, preModifiedBuf, bkt, "") + err := uploadIfChanged(ctx, v, preModifiedBuf, bkt, "") if err != nil { t.Errorf("Expected uploadToGCS to return nil for new object, got %v", err) } @@ -81,7 +81,7 @@ func TestUploadToGCS(t *testing.T) { t.Run("Skip upload if hash matches", func(t *testing.T) { // Modify the vulnerability to simulate a change in modified time but not content v.Modified = timestamppb.New(time.Now().Add(1 * time.Hour)) - err := uploadToGCS(ctx, v, preModifiedBuf, bkt, "") + err := uploadIfChanged(ctx, v, preModifiedBuf, bkt, "") if !errors.Is(err, ErrUploadSkipped) { t.Errorf("Expected uploadToGCS to return ErrUploadSkipped when hash matches, got %v", err) } @@ -101,7 +101,7 @@ func TestUploadToGCS(t *testing.T) { t.Run("Upload if hash differs", func(t *testing.T) { preModifiedBuf2 := []byte(`{"id":"CVE-2023-1234", "summary": "updated"}`) - err := uploadToGCS(ctx, v, preModifiedBuf2, bkt, "") + err := uploadIfChanged(ctx, v, preModifiedBuf2, bkt, "") if err != nil { t.Errorf("Expected uploadToGCS to return nil when hash differs, got %v", err) } @@ -244,7 +244,7 @@ func TestWorker(t *testing.T) { w.Close() var counter atomic.Uint64 - Worker(ctx, vulnChan, outBkt, overridesBkt, "", &counter) + VulnWorker(ctx, vulnChan, outBkt, overridesBkt, "", &counter) if counter.Load() != 2 { t.Errorf("Expected counter to be 2, got %d", counter.Load()) @@ -299,7 +299,7 @@ func TestUpload(t *testing.T) { }, } - Upload(ctx, "test-job", true, outBucketName, "", 1, "", vulnerabilities, false) + UploadVulnsToGCS(ctx, "test-job", true, outBucketName, "", 1, "", vulnerabilities, false) client := server.Client() bkt := client.Bucket(outBucketName) diff --git a/vulnfeeds/gcs-tools/gcs.go b/vulnfeeds/gcs-tools/gcs.go new file mode 100644 index 00000000000..a73d66a8fe5 --- /dev/null +++ b/vulnfeeds/gcs-tools/gcs.go @@ -0,0 +1,135 @@ +// Package gcs provides utilities for working with Google Cloud Storage. +package gcs + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "cloud.google.com/go/storage" + "golang.org/x/sync/errgroup" + "google.golang.org/api/iterator" +) + +// UploadToGCS uploads data from an io.Reader to a GCS bucket. +func UploadToGCS(ctx context.Context, bkt *storage.BucketHandle, objectName string, data io.Reader, contentType string) error { + obj := bkt.Object(objectName) + wc := obj.NewWriter(ctx) + if contentType != "" { + wc.ContentType = contentType + } + + if _, err := io.Copy(wc, data); err != nil { + if closeErr := wc.Close(); closeErr != nil { + return fmt.Errorf("failed to write to GCS object %q: %w (also failed to close writer: %w)", objectName, err, closeErr) + } + + return fmt.Errorf("failed to write to GCS object %q: %w", objectName, err) + } + + if err := wc.Close(); err != nil { + return fmt.Errorf("failed to close GCS writer for object %q: %w", objectName, err) + } + + return nil +} + +// UploadFile uploads a local file to a GCS bucket. +func UploadFile(ctx context.Context, bkt *storage.BucketHandle, objectName string, filePath string) error { + f, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("os.Open: %w", err) + } + defer f.Close() + + return UploadToGCS(ctx, bkt, objectName, f, "") +} + +// DownloadBucket downloads all objects from a GCS bucket to a local directory. +func DownloadBucket(ctx context.Context, bkt *storage.BucketHandle, prefix string, destDir string) error { + it := bkt.Objects(ctx, &storage.Query{Prefix: prefix}) + + g, ctx := errgroup.WithContext(ctx) + // Limit concurrency to avoid running out of file descriptors or overwhelming the network + g.SetLimit(10) + + for { + if err := ctx.Err(); err != nil { + return err + } + + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + break + } + if err != nil { + return fmt.Errorf("bucket.Objects: %w", err) + } + + // Skip directories + if strings.HasSuffix(attrs.Name, "/") { + continue + } + + destPath := filepath.Join(destDir, attrs.Name) + if !strings.HasPrefix(destPath, filepath.Clean(destDir)+string(os.PathSeparator)) { + return fmt.Errorf("invalid object name %q: path traversal attempt", attrs.Name) + } + + // Capture loop variable for the goroutine + objName := attrs.Name + + g.Go(func() error { + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return fmt.Errorf("os.MkdirAll: %w", err) + } + + f, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("os.Create: %w", err) + } + defer f.Close() + + rc, err := bkt.Object(objName).NewReader(ctx) + if err != nil { + return fmt.Errorf("Object(%q).NewReader: %w", objName, err) + } + defer rc.Close() + + if _, err := io.Copy(f, rc); err != nil { + return fmt.Errorf("io.Copy: %w", err) + } + + return nil + }) + } + + if err := g.Wait(); err != nil { + return err + } + + return nil +} + +// listBucketObjects lists the names of all objects in a Google Cloud Storage bucket. +// It does not download the file contents. +func ListBucketObjects(ctx context.Context, bucket *storage.BucketHandle, prefix string) ([]string, error) { + it := bucket.Objects(ctx, &storage.Query{Prefix: prefix}) + var filenames []string + for { + attrs, err := it.Next() + if errors.Is(err, iterator.Done) { + break // All objects have been listed. + } + if err != nil { + return nil, fmt.Errorf("bucket.Objects: %w", err) + } + filenames = append(filenames, attrs.Name) + } + + return filenames, nil +} diff --git a/vulnfeeds/gcs-tools/gcs_test.go b/vulnfeeds/gcs-tools/gcs_test.go new file mode 100644 index 00000000000..93e36ff5d3d --- /dev/null +++ b/vulnfeeds/gcs-tools/gcs_test.go @@ -0,0 +1,210 @@ +package gcs + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + + "github.com/fsouza/fake-gcs-server/fakestorage" +) + +func TestUploadToGCS(t *testing.T) { + server := fakestorage.NewServer([]fakestorage.Object{}) + t.Cleanup(server.Stop) + + client := server.Client() + bkt := client.Bucket("test-bucket") + if err := bkt.Create(context.Background(), "project", nil); err != nil { + t.Fatalf("failed to create bucket: %v", err) + } + + content := []byte("test content") + err := UploadToGCS(context.Background(), bkt, "test-object.txt", bytes.NewReader(content), "text/plain") + if err != nil { + t.Fatalf("UploadToGCS failed: %v", err) + } + + obj, err := server.GetObject("test-bucket", "test-object.txt") + if err != nil { + t.Fatalf("failed to get object: %v", err) + } + + if !bytes.Equal(obj.Content, content) { + t.Errorf("expected content %q, got %q", content, obj.Content) + } + if obj.ContentType != "text/plain" { + t.Errorf("expected content type %q, got %q", "text/plain", obj.ContentType) + } +} + +func TestUploadFile(t *testing.T) { + server := fakestorage.NewServer([]fakestorage.Object{}) + t.Cleanup(server.Stop) + + client := server.Client() + bkt := client.Bucket("test-bucket") + if err := bkt.Create(context.Background(), "project", nil); err != nil { + t.Fatalf("failed to create bucket: %v", err) + } + + tmpFile, err := os.CreateTemp(t.TempDir(), "test-upload-*.txt") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + + content := []byte("file content") + if _, err := tmpFile.Write(content); err != nil { + t.Fatalf("failed to write to temp file: %v", err) + } + tmpFile.Close() + + err = UploadFile(context.Background(), bkt, "uploaded-file.txt", tmpFile.Name()) + if err != nil { + t.Fatalf("UploadFile failed: %v", err) + } + + obj, err := server.GetObject("test-bucket", "uploaded-file.txt") + if err != nil { + t.Fatalf("failed to get object: %v", err) + } + + if !bytes.Equal(obj.Content, content) { + t.Errorf("expected content %q, got %q", content, obj.Content) + } +} + +func TestDownloadBucket(t *testing.T) { + t.Run("success", func(t *testing.T) { + objects := []fakestorage.Object{ + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: "test-bucket", + Name: "folder/file1.txt", + }, + Content: []byte("content 1"), + }, + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: "test-bucket", + Name: "folder/file2.txt", + }, + Content: []byte("content 2"), + }, + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: "test-bucket", + Name: "folder/subfolder/", // Should be skipped + }, + Content: []byte(""), + }, + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: "test-bucket", + Name: "other-folder/file3.txt", + }, + Content: []byte("content 3"), + }, + } + + server := fakestorage.NewServer(objects) + t.Cleanup(server.Stop) + + client := server.Client() + bkt := client.Bucket("test-bucket") + + tmpDir := t.TempDir() + + err := DownloadBucket(context.Background(), bkt, "folder/", tmpDir) + if err != nil { + t.Fatalf("DownloadBucket failed: %v", err) + } + + // Verify file1.txt + content1, err := os.ReadFile(filepath.Join(tmpDir, "folder/file1.txt")) + if err != nil { + t.Fatalf("failed to read downloaded file1: %v", err) + } + if !bytes.Equal(content1, []byte("content 1")) { + t.Errorf("expected content 1, got %q", content1) + } + + // Verify file2.txt + content2, err := os.ReadFile(filepath.Join(tmpDir, "folder/file2.txt")) + if err != nil { + t.Fatalf("failed to read downloaded file2: %v", err) + } + if !bytes.Equal(content2, []byte("content 2")) { + t.Errorf("expected content 2, got %q", content2) + } + + // Verify file3.txt is NOT downloaded because of the prefix + if _, err := os.Stat(filepath.Join(tmpDir, "other-folder/file3.txt")); !os.IsNotExist(err) { + t.Errorf("expected file3.txt to not exist, but it does") + } + }) + + t.Run("path traversal", func(t *testing.T) { + objects := []fakestorage.Object{ + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: "test-bucket", + Name: "../malicious.txt", + }, + Content: []byte("malicious content"), + }, + } + + server := fakestorage.NewServer(objects) + t.Cleanup(server.Stop) + + client := server.Client() + bkt := client.Bucket("test-bucket") + + tmpDir := t.TempDir() + + err := DownloadBucket(context.Background(), bkt, "", tmpDir) + if err == nil { + t.Fatalf("expected path traversal error, got nil") + } + if err.Error() != "invalid object name \"../malicious.txt\": path traversal attempt" { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("relative dest dir", func(t *testing.T) { + objects := []fakestorage.Object{ + { + ObjectAttrs: fakestorage.ObjectAttrs{ + BucketName: "test-bucket", + Name: "file.txt", + }, + Content: []byte("content"), + }, + } + + server := fakestorage.NewServer(objects) + t.Cleanup(server.Stop) + + client := server.Client() + bkt := client.Bucket("test-bucket") + + // Use a relative directory + destDir := "test-relative-dir" + t.Cleanup(func() { os.RemoveAll(destDir) }) + + err := DownloadBucket(context.Background(), bkt, "", destDir) + if err != nil { + t.Fatalf("DownloadBucket failed with relative dir: %v", err) + } + + content, err := os.ReadFile(filepath.Join(destDir, "file.txt")) + if err != nil { + t.Fatalf("failed to read downloaded file: %v", err) + } + if !bytes.Equal(content, []byte("content")) { + t.Errorf("expected content, got %q", content) + } + }) +}