main.go

// package main contains the conversion logic for turning debian security tracker info to OSV parts
package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"log/slog"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"

	"github.com/google/osv/vulnfeeds/faulttolerant"
	"github.com/google/osv/vulnfeeds/gcs-tools"
	"github.com/google/osv/vulnfeeds/models"
	"github.com/google/osv/vulnfeeds/utility/logger"
	"github.com/google/osv/vulnfeeds/vulns"
	"github.com/ossf/osv-schema/bindings/go/osvschema"
	"google.golang.org/protobuf/types/known/timestamppb"
)

const (
	defaultCvePath           = "cve_jsons"
	debianOutputPathDefault  = "debian-cve-osv"
	debianDistroInfoURL      = "https://debian.pages.debian.net/distro-info-data/debian.csv"
	debianSecurityTrackerURL = "https://security-tracker.debian.org/tracker/data/json"
	outputBucketDefault      = "debian-osv"
)

func main() {
	logger.InitGlobalLogger()
	defer logger.Close()

	debianOutputPath := flag.String("output-path", debianOutputPathDefault, "Path to output OSV files.")
	outputBucketName := flag.String("output-bucket", outputBucketDefault, "The GCS bucket to write to.")
	numWorkers := flag.Int("workers", 64, "Number of workers to process records")
	uploadToGCS := flag.Bool("upload-to-gcs", false, "If true, do not write to GCS bucket and instead write to local disk.")
	syncDeletions := flag.Bool("sync-deletions", false, "If false, do not delete files in bucket that are not local")
	flag.Parse()

	err := os.MkdirAll(*debianOutputPath, 0755)
	if err != nil {
		logger.Fatal("Can't create output path", slog.Any("err", err))
	}

	debianData, err := downloadDebianSecurityTracker()
	if err != nil {
		logger.Fatal("Failed to download/parse Debian Security Tracker json file", slog.Any("err", err))
	}

	debianReleaseMap, err := getDebianReleaseMap()
	if err != nil {
		logger.Fatal("Failed to get Debian distro info data", slog.Any("err", err))
	}

	allCVEs := vulns.LoadAllCVEs(defaultCvePath)
	osvCVEs := generateOSVFromDebianTracker(debianData, debianReleaseMap, allCVEs)

	vulnerabilities := make([]*osvschema.Vulnerability, 0, len(osvCVEs))
	for _, v := range osvCVEs {
		if len(v.Affected) == 0 {
			logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.Id), slog.String("id", v.Id))
			continue
		}
		vulnerabilities = append(vulnerabilities, v.Vulnerability)
	}

	ctx := context.Background()
	gcs.Upload(ctx, "Debian CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *debianOutputPath, vulnerabilities, *syncDeletions)
	logger.Info("Debian CVE conversion succeeded.")
}

// generateOSVFromDebianTracker converts Debian Security Tracker entries to OSV format.
func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[models.CVEID]models.Vulnerability) map[string]*vulns.Vulnerability {
	logger.Info("Converting Debian Security Tracker data to OSV.")
	osvCves := make(map[string]*vulns.Vulnerability)

	// Sorts packages to ensure results remain consistent between runs.
	pkgNames := make([]string, 0, len(debianData))
	for name := range debianData {
		pkgNames = append(pkgNames, name)
	}
	sort.Strings(pkgNames)

	// Sorts releases to ensure pkgInfos remain consistent between runs.
	releaseNames := make([]string, 0, len(debianReleaseMap))
	for k := range debianReleaseMap {
		releaseNames = append(releaseNames, k)
	}

	sort.Slice(releaseNames, func(i, j int) bool {
		vi, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[i]], 64)
		vj, _ := strconv.ParseFloat(debianReleaseMap[releaseNames[j]], 64)

		return vi < vj
	})

	for _, pkgName := range pkgNames {
		pkg := debianData[pkgName]
		for cveID, cveData := range pkg {
			// Debian Security Tracker has some 'TEMP-' Records we don't want to convert
			if !strings.HasPrefix(cveID, "CVE") {
				continue
			}
			v, ok := osvCves[cveID]
			currentNVDCVE := allCVEs[models.CVEID(cveID)]
			if !ok {
				v = &vulns.Vulnerability{
					Vulnerability: &osvschema.Vulnerability{
						Id:        "DEBIAN-" + cveID,
						Upstream:  []string{cveID},
						Published: timestamppb.New(currentNVDCVE.CVE.Published.Time),
						Details:   cveData.Description,
						References: []*osvschema.Reference{
							{
								Type: osvschema.Reference_ADVISORY,
								Url:  "https://security-tracker.debian.org/tracker/" + cveID,
							},
						},
					},
				}
				if currentNVDCVE.CVE.Metrics != nil {
					v.AddSeverity(currentNVDCVE.CVE.Metrics)
				}

				osvCves[cveID] = v
			}

			for _, releaseName := range releaseNames {
				// For reference on urgency levels, see: https://security-team.debian.org/security_tracker.html#severity-levels
				release, ok := cveData.Releases[releaseName]
				if !ok {
					continue
				}
				debianVersion, ok := debianReleaseMap[releaseName]
				if !ok {
					continue
				}

				if release.Status == "resolved" && release.FixedVersion == "0" { // not affected
					continue
				}

				pkgInfo := vulns.PackageInfo{
					PkgName:   pkgName,
					Ecosystem: "Debian:" + debianVersion,
					EcosystemSpecific: map[string]any{
						"urgency": release.Urgency,
					},
				}

				if release.Status == "resolved" {
					pkgInfo.VersionInfo.AffectedVersions = []models.AffectedVersion{{Introduced: "0"}, {Fixed: release.FixedVersion}}
				} else {
					pkgInfo.VersionInfo.AffectedVersions = []models.AffectedVersion{{Introduced: "0"}}
				}

				if len(pkgInfo.VersionInfo.AffectedVersions) > 0 {
					v.AddPkgInfo(pkgInfo)
				}
			}
		}
	}

	return osvCves
}

// getDebianReleaseMap gets the Debian version number, excluding testing and experimental versions.
func getDebianReleaseMap() (map[string]string, error) {
	releaseMap := make(map[string]string)
	res, err := faulttolerant.Get(debianDistroInfoURL)
	if err != nil {
		return releaseMap, err
	}
	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return releaseMap, fmt.Errorf("HTTP request failed: %s", res.Status)
	}

	reader := csv.NewReader(res.Body)
	reader.FieldsPerRecord = -1
	data, err := reader.ReadAll()
	if err != nil {
		return releaseMap, err
	}

	versionIndex := -1
	seriesIndex := -1

	// Get the index number of version and series.
	for i, col := range data[0] {
		switch col {
		case "version":
			versionIndex = i
		case "series":
			seriesIndex = i
		}
	}

	if seriesIndex == -1 || versionIndex == -1 {
		return releaseMap, err
	}

	for _, row := range data[1:] {
		if row[versionIndex] == "" {
			continue
		}
		releaseMap[row[seriesIndex]] = row[versionIndex]
	}

	return releaseMap, err
}

// downloadDebianSecurityTracker download Debian json file
func downloadDebianSecurityTracker() (DebianSecurityTrackerData, error) {
	res, err := faulttolerant.Get(debianSecurityTrackerURL)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	if res.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("HTTP request failed: %s", res.Status)
	}

	var decodedDebianData DebianSecurityTrackerData

	if err := json.NewDecoder(res.Body).Decode(&decodedDebianData); err != nil {
		return nil, err
	}

	logger.Info("Successfully downloaded Debian Security Tracker Data")

	return decodedDebianData, err
}