Skip to content

Commit cc40dc5

Browse files
jess-lowejess-lowe
committed
Add source information to metadata
1 parent 3ed1854 commit cc40dc5

10 files changed

Lines changed: 130 additions & 62 deletions

File tree

‎vulnfeeds/conversion/common.go‎

Lines changed: 49 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -464,70 +464,97 @@ func deduplicateList(list []any) []any {
464464
return unique
465465
}
466466

467-
func ToRangeWithMetadata(r []*osvschema.Range) []models.RangeWithMetadata {
468-
var nr []models.RangeWithMetadata
467+
func ToRangeWithMetadata(r []*osvschema.Range, s models.VersionSource) []models.RangeWithMetadata {
468+
nr := make([]models.RangeWithMetadata, 0, len(r))
469469
for _, rng := range r {
470470
nr = append(nr, models.RangeWithMetadata{
471471
Range: rng,
472+
Metadata: models.Metadata{
473+
Source: s,
474+
},
472475
})
473476
}
474477

475478
return nr
476479
}
477480

478481
func CreateUnresolvedRanges(unresolvedRanges []models.RangeWithMetadata) *structpb.ListValue {
479-
if len(unresolvedRanges) > 0 {
482+
if len(unresolvedRanges) == 0 {
483+
return nil
484+
}
485+
486+
rangesBySource := make(map[string][]models.RangeWithMetadata)
487+
var sources []string
488+
for _, ur := range unresolvedRanges {
489+
sourceStr := string(ur.Metadata.Source)
490+
if _, ok := rangesBySource[sourceStr]; !ok {
491+
sources = append(sources, sourceStr)
492+
}
493+
rangesBySource[sourceStr] = append(rangesBySource[sourceStr], ur)
494+
}
495+
496+
slices.Sort(sources)
497+
498+
listElements := make([]any, 0, len(sources))
499+
500+
for _, source := range sources {
501+
ranges := rangesBySource[source]
480502
cpes := []string{}
481503
unresolvedRangesMap := make(map[string]any)
482504
var events []*osvschema.Event
505+
483506
// Create a range from all those with CPEs
484-
for _, ur := range unresolvedRanges {
507+
for _, ur := range ranges {
485508
if ur.Metadata.CPE != "" {
486509
cpes = append(cpes, ur.Metadata.CPE)
487510
}
488511
urEvents := ur.Range.GetEvents()
489512

490513
for _, e := range urEvents {
491-
if e.Introduced != "0" && e.Introduced != "" {
514+
if e.GetIntroduced() != "0" && e.GetIntroduced() != "" {
492515
events = append(events, e)
493516
continue
494517
}
495-
if e.LastAffected != "" {
518+
if e.GetLastAffected() != "" {
496519
events = append(events, e)
497520
continue
498521
}
499-
if e.Fixed != "" {
522+
if e.GetFixed() != "" {
500523
events = append(events, e)
501524
}
502525
}
503526
}
504527

528+
metadata := make(map[string]any)
505529
if len(cpes) > 1 {
506530
slices.Sort(cpes)
507531
cpes = slices.Compact(cpes)
508-
unresolvedRangesMap["metadata"] = map[string]any{
509-
"cpes": cpes,
510-
}
532+
metadata["cpes"] = cpes
511533
} else if len(cpes) == 1 {
512-
unresolvedRangesMap["metadata"] = map[string]any{
513-
"cpe": cpes[0],
514-
}
534+
metadata["cpe"] = cpes[0]
515535
}
516536

517-
unresolvedRangesMap["versions"] = events
537+
if source != "" {
538+
metadata["source"] = source
539+
}
518540

519-
ds, err := utility.NewStructpbFromMap(map[string]any{
520-
"list": []any{unresolvedRangesMap},
521-
})
522-
if err != nil {
523-
logger.Warn("failed to convert unresolved ranges to structpb", "err", err)
524-
return nil
541+
if len(metadata) > 0 {
542+
unresolvedRangesMap["metadata"] = metadata
525543
}
526544

527-
return ds.GetFields()["list"].GetListValue()
545+
unresolvedRangesMap["versions"] = events
546+
listElements = append(listElements, unresolvedRangesMap)
547+
}
548+
549+
ds, err := utility.NewStructpbFromMap(map[string]any{
550+
"list": listElements,
551+
})
552+
if err != nil {
553+
logger.Warn("failed to convert unresolved ranges to structpb", "err", err)
554+
return nil
528555
}
529556

530-
return nil
557+
return ds.GetFields()["list"].GetListValue()
531558
}
532559

533560
func AddFieldToDatabaseSpecific(ds *structpb.Struct, field string, value any) error {

‎vulnfeeds/conversion/cve5/common.go‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"strconv"
77
"strings"
88

9-
"github.com/google/osv/vulnfeeds/conversion"
9+
c "github.com/google/osv/vulnfeeds/conversion"
1010
"github.com/google/osv/vulnfeeds/models"
1111
"github.com/google/osv/vulnfeeds/vulns"
1212
"github.com/ossf/osv-schema/bindings/go/osvschema"
@@ -53,10 +53,10 @@ func toVersionRangeType(s string) VersionRangeType {
5353

5454
// findCPEVersionRanges extracts version ranges and CPE strings from the CNA's
5555
// CPE applicability statements in a CVE record.
56-
func findCPEVersionRanges(cve models.CVE5) (versionRanges []*osvschema.Range, cpes []string, err error) {
56+
func findCPEVersionRanges(cve models.CVE5) (versionRanges []models.RangeWithMetadata, cpes []string, err error) {
5757
// TODO(jesslowe): Add logic to also extract CPEs from the 'affected' field (e.g., CVE-2025-1110).
58-
for _, c := range cve.Containers.CNA.CPEApplicability {
59-
for _, node := range c.Nodes {
58+
for _, cpe := range cve.Containers.CNA.CPEApplicability {
59+
for _, node := range cpe.Nodes {
6060
if node.Operator != "OR" {
6161
continue
6262
}
@@ -70,11 +70,14 @@ func findCPEVersionRanges(cve models.CVE5) (versionRanges []*osvschema.Range, cp
7070
if match.VersionStartIncluding == "" {
7171
match.VersionStartIncluding = "0"
7272
}
73-
73+
var nr []*osvschema.Range
7474
if match.VersionEndExcluding != "" {
75-
versionRanges = append(versionRanges, conversion.BuildVersionRange(match.VersionStartIncluding, "", match.VersionEndExcluding))
75+
nr = append(nr, c.BuildVersionRange(match.VersionStartIncluding, "", match.VersionEndExcluding))
7676
} else if match.VersionEndIncluding != "" {
77-
versionRanges = append(versionRanges, conversion.BuildVersionRange(match.VersionStartIncluding, match.VersionEndIncluding, ""))
77+
nr = append(nr, c.BuildVersionRange(match.VersionStartIncluding, match.VersionEndIncluding, ""))
78+
}
79+
if nr != nil {
80+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(nr, models.VersionSourceCPE)...)
7881
}
7982
}
8083
}
@@ -98,8 +101,8 @@ func compareSemverLike(a, b string) int {
98101
// We ignore the error, so non-numeric parts default to 0.
99102
numA, _ := strconv.Atoi(partsA[i])
100103
numB, _ := strconv.Atoi(partsB[i])
101-
if c := cmp.Compare(numA, numB); c != 0 {
102-
return c
104+
if v := cmp.Compare(numA, numB); v != 0 {
105+
return v
103106
}
104107
}
105108
// If lengths are the same, they're equal.

‎vulnfeeds/conversion/cve5/default_extractor.go‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ import (
1515
// DefaultVersionExtractor provides the default version extraction logic.
1616
type DefaultVersionExtractor struct{}
1717

18-
func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, metrics *models.ConversionMetrics) []*osvschema.Range {
19-
var ranges []*osvschema.Range
18+
func (d *DefaultVersionExtractor) handleAffected(affected []models.Affected, metrics *models.ConversionMetrics) []models.RangeWithMetadata {
19+
var ranges []models.RangeWithMetadata
2020
for _, cveAff := range affected {
2121
versionRanges, _ := d.FindNormalAffectedRanges(cveAff, metrics)
2222

@@ -56,9 +56,8 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln
5656
return true
5757
}
5858

59-
6059
if len(ranges) != 0 {
61-
if processRanges(c.ToRangeWithMetadata(ranges)) {
60+
if processRanges(ranges) {
6261
gotVersions = true
6362
metrics.SetOutcome(models.Successful)
6463
}
@@ -69,7 +68,7 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln
6968
versionRanges, _ := cpeVersionExtraction(cve, metrics)
7069

7170
if len(versionRanges) != 0 {
72-
if processRanges(c.ToRangeWithMetadata(versionRanges)) {
71+
if processRanges(versionRanges) {
7372
gotVersions = true
7473
}
7574
}
@@ -99,13 +98,13 @@ func (d *DefaultVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vuln
9998
}
10099
}
101100

102-
func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) {
101+
func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]models.RangeWithMetadata, VersionRangeType) {
103102
versionTypesCount := make(map[VersionRangeType]int)
104-
var versionRanges []*osvschema.Range
103+
var versionRanges []models.RangeWithMetadata
105104
for _, vers := range affected.Versions {
106105
ranges, _, shouldContinue := initialNormalExtraction(vers, metrics, versionTypesCount)
107106
if len(ranges) > 0 {
108-
versionRanges = append(versionRanges, ranges...)
107+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(ranges, models.VersionSourceAffected)...)
109108
}
110109

111110
if shouldContinue {
@@ -121,11 +120,16 @@ func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affec
121120
if av.Introduced == "" {
122121
continue
123122
}
123+
124124
if av.Fixed != "" {
125-
versionRanges = append(versionRanges, c.BuildVersionRange(av.Introduced, "", av.Fixed))
125+
vr := []*osvschema.Range{c.BuildVersionRange(av.Introduced, "", av.Fixed)}
126+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(vr, models.VersionSourceAffected)...)
127+
126128
continue
127129
} else if av.LastAffected != "" {
128-
versionRanges = append(versionRanges, c.BuildVersionRange(av.Introduced, av.LastAffected, ""))
130+
vr := []*osvschema.Range{c.BuildVersionRange(av.Introduced, av.LastAffected, "")}
131+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(vr, models.VersionSourceAffected)...)
132+
129133
continue
130134
}
131135
}
@@ -140,7 +144,8 @@ func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affec
140144

141145
// As a fallback, assume a single version means it's the last affected version.
142146
if vulns.CheckQuality(vers.Version).AtLeast(acceptableQuality) {
143-
versionRanges = append(versionRanges, c.BuildVersionRange("0", vers.Version, ""))
147+
vr := []*osvschema.Range{c.BuildVersionRange("0", vers.Version, "")}
148+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(vr, models.VersionSourceAffected)...)
144149
metrics.AddNote("Single version found %v - Assuming introduced = 0 and last affected = %v", vers.Version, vers.Version)
145150
}
146151
}
@@ -156,4 +161,4 @@ func (d *DefaultVersionExtractor) FindNormalAffectedRanges(affected models.Affec
156161
}
157162

158163
return versionRanges, mostFrequentVersionType
159-
}
164+
}

‎vulnfeeds/conversion/cve5/extraction.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@ package cve5
33
import (
44
"github.com/google/osv/vulnfeeds/models"
55
"github.com/google/osv/vulnfeeds/vulns"
6-
"github.com/ossf/osv-schema/bindings/go/osvschema"
76
)
87

98
// VersionExtractor defines the interface for different version extraction strategies.
109
type VersionExtractor interface {
1110
ExtractVersions(cve models.CVE5, v *vulns.Vulnerability, metrics *models.ConversionMetrics, repos []string)
12-
FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType)
11+
FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]models.RangeWithMetadata, VersionRangeType)
1312
}
1413

1514
// GetVersionExtractor returns the appropriate VersionExtractor for a given CNA.

‎vulnfeeds/conversion/cve5/linux_extractor.go‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,9 @@ import (
66
"strconv"
77
"strings"
88

9-
"github.com/google/osv/vulnfeeds/conversion"
9+
c "github.com/google/osv/vulnfeeds/conversion"
1010
"github.com/google/osv/vulnfeeds/models"
11+
"github.com/google/osv/vulnfeeds/utility/logger"
1112
"github.com/google/osv/vulnfeeds/vulns"
1213
"github.com/ossf/osv-schema/bindings/go/osvconstants"
1314
"github.com/ossf/osv-schema/bindings/go/osvschema"
@@ -30,7 +31,11 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected
3031
if cveAff.DefaultStatus == "affected" {
3132
versionRanges, versionType = findInverseAffectedRanges(cveAff, metrics)
3233
} else {
33-
versionRanges, versionType = l.FindNormalAffectedRanges(cveAff, metrics)
34+
var versionRangesWithMetadata []models.RangeWithMetadata
35+
versionRangesWithMetadata, versionType = l.FindNormalAffectedRanges(cveAff, metrics)
36+
for _, r := range versionRangesWithMetadata {
37+
versionRanges = append(versionRanges, r.Range)
38+
}
3439
}
3540
if (versionType == VersionRangeTypeGit && hasGit) || len(versionRanges) == 0 {
3641
continue
@@ -43,7 +48,7 @@ func (l *LinuxVersionExtractor) handleAffected(v *vulns.Vulnerability, affected
4348
}
4449
aff := createLinuxAffected(versionRanges, versionType, cveAff.Repo)
4550
metrics.AddSource(models.VersionSourceAffected)
46-
conversion.AddAffected(v, aff, metrics)
51+
c.AddAffected(v, aff, metrics)
4752
}
4853

4954
return gotVersions
@@ -55,10 +60,16 @@ func (l *LinuxVersionExtractor) ExtractVersions(cve models.CVE5, v *vulns.Vulner
5560

5661
if !gotVersions {
5762
metrics.AddNote("No versions in affected, attempting to extract from CPE")
58-
versionRanges, _ := cpeVersionExtraction(cve, metrics)
59-
63+
versionRanges, err := cpeVersionExtraction(cve, metrics)
64+
if err != nil {
65+
logger.Warn("Error when extracting CPE versions")
66+
}
6067
if len(versionRanges) != 0 {
61-
aff := createLinuxAffected(versionRanges, VersionRangeTypeEcosystem, "")
68+
var ranges []*osvschema.Range
69+
for _, r := range versionRanges {
70+
ranges = append(ranges, r.Range)
71+
}
72+
aff := createLinuxAffected(ranges, VersionRangeTypeEcosystem, "")
6273
v.Affected = append(v.Affected, aff)
6374
}
6475
}
@@ -136,7 +147,7 @@ func findInverseAffectedRanges(cveAff models.Affected, metrics *models.Conversio
136147
// Create ranges by pairing sorted introduced and fixed versions.
137148
for index, f := range fixed {
138149
if index < len(introduced) {
139-
ranges = append(ranges, conversion.BuildVersionRange(introduced[index], "", f))
150+
ranges = append(ranges, c.BuildVersionRange(introduced[index], "", f))
140151
metrics.AddNote("Introduced from version value - %s", introduced[index])
141152
metrics.AddNote("Fixed from version value - %s", f)
142153
}
@@ -150,12 +161,12 @@ func findInverseAffectedRanges(cveAff models.Affected, metrics *models.Conversio
150161
return nil, VersionRangeTypeUnknown
151162
}
152163

153-
func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]*osvschema.Range, VersionRangeType) {
164+
func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affected, metrics *models.ConversionMetrics) ([]models.RangeWithMetadata, VersionRangeType) {
154165
versionTypesCount := make(map[VersionRangeType]int)
155-
var versionRanges []*osvschema.Range
166+
var versionRanges []models.RangeWithMetadata
156167
for _, vers := range affected.Versions {
157168
ranges, currentVersionType, shouldContinue := initialNormalExtraction(vers, metrics, versionTypesCount)
158-
versionRanges = append(versionRanges, ranges...)
169+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(ranges, models.VersionSourceAffected)...)
159170
if shouldContinue {
160171
continue
161172
}
@@ -165,13 +176,16 @@ func (l *LinuxVersionExtractor) FindNormalAffectedRanges(affected models.Affecte
165176
metrics.AddNote("Only version exists")
166177

167178
if currentVersionType == VersionRangeTypeGit {
168-
versionRanges = append(versionRanges, conversion.BuildVersionRange(vers.Version, "", ""))
179+
vr := []*osvschema.Range{c.BuildVersionRange(vers.Version, "", "")}
180+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(vr, models.VersionSourceGit)...)
181+
169182
continue
170183
}
171184

172185
// As a fallback, assume a single version means it's the last affected version.
173186
if vulns.CheckQuality(vers.Version).AtLeast(acceptableQuality) {
174-
versionRanges = append(versionRanges, conversion.BuildVersionRange("0", vers.Version, ""))
187+
vr := []*osvschema.Range{c.BuildVersionRange("0", vers.Version, "")}
188+
versionRanges = append(versionRanges, c.ToRangeWithMetadata(vr, models.VersionSourceAffected)...)
175189
metrics.AddNote("Single version found %v - Assuming introduced = 0 and last affected = %v", vers.Version, vers.Version)
176190
}
177191
}

‎vulnfeeds/conversion/cve5/strategies.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"github.com/ossf/osv-schema/bindings/go/osvschema"
88
)
99

10-
func cpeVersionExtraction(cve models.CVE5, metrics *models.ConversionMetrics) ([]*osvschema.Range, error) {
10+
func cpeVersionExtraction(cve models.CVE5, metrics *models.ConversionMetrics) ([]models.RangeWithMetadata, error) {
1111
cpeRanges, cpeStrings, err := findCPEVersionRanges(cve)
1212
if err == nil && len(cpeRanges) > 0 {
1313
metrics.VersionSources = append(metrics.VersionSources, models.VersionSourceCPE)

‎vulnfeeds/conversion/cve5/version_extraction_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,11 @@ func TestFindNormalAffectedRanges(t *testing.T) {
114114
for _, tt := range tests {
115115
t.Run(tt.name, func(t *testing.T) {
116116
versionExtractor := &DefaultVersionExtractor{}
117-
gotRanges, gotRangeType := versionExtractor.FindNormalAffectedRanges(tt.affected, &models.ConversionMetrics{})
117+
gotRangesWithMeta, gotRangeType := versionExtractor.FindNormalAffectedRanges(tt.affected, &models.ConversionMetrics{})
118+
var gotRanges []*osvschema.Range
119+
for _, r := range gotRangesWithMeta {
120+
gotRanges = append(gotRanges, r.Range)
121+
}
118122
if diff := cmp.Diff(tt.wantRanges, gotRanges, protocmp.Transform()); diff != "" {
119123
t.Errorf("findNormalAffectedRanges() ranges mismatch (-want +got):\n%s", diff)
120124
}

0 commit comments

Comments
 (0)