Skip to content

fix(vulnfeeds): Migrate to latest OSV schema bindings #4241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

fix(vulnfeeds): Migrate to latest OSV schema bindings #4241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1db7a19
migrate
cuixq Oct 27, 2025
14a83bd
lints
cuixq Oct 28, 2025
488679e
more lints
cuixq Oct 28, 2025
cfaa302
even more lints
cuixq Oct 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions vulnfeeds/cmd/alpine/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/google/osv/vulnfeeds/utility/logger"
"github.com/google/osv/vulnfeeds/vulns"
"github.com/ossf/osv-schema/bindings/go/osvschema"
"google.golang.org/protobuf/types/known/timestamppb"
)

const (
Expand Down Expand Up @@ -55,7 +56,7 @@ func main() {
vulnerabilities := make([]*osvschema.Vulnerability, 0, len(osvVulnerabilities))
for _, v := range osvVulnerabilities {
if len(v.Affected) == 0 {
logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.ID), slog.String("id", v.ID))
logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.Id), slog.String("id", v.Id))
continue
}
vulnerabilities = append(vulnerabilities, &v.Vulnerability)
Expand Down Expand Up @@ -171,14 +172,14 @@ func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cv

v := &vulns.Vulnerability{
Vulnerability: osvschema.Vulnerability{
ID: "ALPINE-" + cveID,
Id: "ALPINE-" + cveID,
Upstream: []string{cveID},
Published: published,
Published: timestamppb.New(published),
Details: details,
References: []osvschema.Reference{
References: []*osvschema.Reference{
{
Type: "ADVISORY",
URL: "https://security.alpinelinux.org/vuln/" + cveID,
Type: osvschema.Reference_ADVISORY,
Url: "https://security.alpinelinux.org/vuln/" + cveID,
},
},
},
Expand All @@ -197,7 +198,7 @@ func generateAlpineOSV(allAlpineSecDb map[string][]VersionAndPkg, allCVEs map[cv
}

if len(v.Affected) == 0 {
logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.ID), slog.String("cveID", cveID))
logger.Warn(fmt.Sprintf("Skipping %s as no affected versions found.", v.Id), slog.String("cveID", cveID))
continue
}
if cve.CVE.Metrics != nil {
Expand Down
101 changes: 50 additions & 51 deletions vulnfeeds/cmd/combine-to-osv/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package main

import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
Expand All @@ -21,6 +20,7 @@ import (
"github.com/google/osv/vulnfeeds/utility/logger"
"github.com/ossf/osv-schema/bindings/go/osvschema"
"google.golang.org/api/iterator"
"google.golang.org/protobuf/encoding/protojson"
)

const (
Expand Down Expand Up @@ -86,7 +86,7 @@ func main() {

vulnerabilities := make([]*osvschema.Vulnerability, 0, len(combinedData))
for _, v := range combinedData {
vulnerabilities = append(vulnerabilities, &v)
vulnerabilities = append(vulnerabilities, v)
}

upload.Upload(ctx, "OSV files", *uploadToGCS, *outputBucketName, *overridesBucketName, *numWorkers, *osvOutputPath, vulnerabilities)
Expand Down Expand Up @@ -136,8 +136,8 @@ func listBucketObjects(bucketName string, prefix string) ([]string, error) {
// The function returns a map of CVE IDs to their corresponding Vulnerability objects.
// Files that are not ".json" files, directories, or files ending in ".metrics.json" are skipped.
// The function will log warnings for files that fail to open or decode, and will terminate if it fails to walk the directory.
func loadOSV(osvPath string) map[cves.CVEID]osvschema.Vulnerability {
allVulns := make(map[cves.CVEID]osvschema.Vulnerability)
func loadOSV(osvPath string) map[cves.CVEID]*osvschema.Vulnerability {
allVulns := make(map[cves.CVEID]*osvschema.Vulnerability)
logger.Info("Loading OSV records", slog.String("path", osvPath))
err := filepath.WalkDir(osvPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
Expand All @@ -147,20 +147,19 @@ func loadOSV(osvPath string) map[cves.CVEID]osvschema.Vulnerability {
return nil
}

file, err := os.Open(path)
file, err := os.ReadFile(path)
if err != nil {
logger.Warn("Failed to open OSV JSON file", slog.String("path", path), slog.Any("err", err))
return nil
}

var vuln osvschema.Vulnerability
decodeErr := json.NewDecoder(file).Decode(&vuln)
file.Close()
decodeErr := protojson.Unmarshal(file, &vuln)
if decodeErr != nil {
logger.Error("Failed to decode, skipping", slog.String("file", path), slog.Any("err", decodeErr))
return nil
}
allVulns[cves.CVEID(vuln.ID)] = vuln
allVulns[cves.CVEID(vuln.GetId())] = &vuln

return nil
})
Expand All @@ -173,12 +172,12 @@ func loadOSV(osvPath string) map[cves.CVEID]osvschema.Vulnerability {
}

// combineIntoOSV creates OSV entry by combining loaded CVEs from NVD and PackageInfo information from security advisories.
func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[cves.CVEID]osvschema.Vulnerability, mandatoryCVEIDs []string) map[cves.CVEID]osvschema.Vulnerability {
osvRecords := make(map[cves.CVEID]osvschema.Vulnerability)
func combineIntoOSV(cve5osv map[cves.CVEID]*osvschema.Vulnerability, nvdosv map[cves.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[cves.CVEID]*osvschema.Vulnerability {
osvRecords := make(map[cves.CVEID]*osvschema.Vulnerability)

// Iterate through CVEs from security advisories (cve5) as the base
for cveID, cve5 := range cve5osv {
var baseOSV osvschema.Vulnerability
var baseOSV *osvschema.Vulnerability
nvd, ok := nvdosv[cveID]

if ok {
Expand All @@ -189,7 +188,7 @@ func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[c
baseOSV = cve5
}

if len(baseOSV.Affected) == 0 {
if len(baseOSV.GetAffected()) == 0 {
// check if part exists.
if !slices.Contains(mandatoryCVEIDs, string(cveID)) {
continue
Expand All @@ -200,7 +199,7 @@ func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[c

// Add any remaining CVEs from NVD that were not in the advisory data.
for cveID, nvd := range nvdosv {
if len(nvd.Affected) == 0 {
if len(nvd.GetAffected()) == 0 {
continue
}
osvRecords[cveID] = nvd
Expand All @@ -210,40 +209,40 @@ func combineIntoOSV(cve5osv map[cves.CVEID]osvschema.Vulnerability, nvdosv map[c
}

// combineTwoOSVRecords takes two osv records and combines them into one
func combineTwoOSVRecords(cve5 osvschema.Vulnerability, nvd osvschema.Vulnerability) osvschema.Vulnerability {
func combineTwoOSVRecords(cve5 *osvschema.Vulnerability, nvd *osvschema.Vulnerability) *osvschema.Vulnerability {
baseOSV := cve5
combinedAffected := pickAffectedInformation(cve5.Affected, nvd.Affected)
combinedAffected := pickAffectedInformation(cve5.GetAffected(), nvd.GetAffected())

baseOSV.Affected = combinedAffected
// Merge references, ensuring no duplicates.
refMap := make(map[string]bool)
for _, r := range baseOSV.References {
refMap[r.URL] = true
for _, r := range baseOSV.GetReferences() {
refMap[r.GetUrl()] = true
}
for _, r := range nvd.References {
if !refMap[r.URL] {
for _, r := range nvd.GetReferences() {
if !refMap[r.GetUrl()] {
baseOSV.References = append(baseOSV.References, r)
refMap[r.URL] = true
refMap[r.GetUrl()] = true
}
}

// Merge timestamps: latest modified, earliest published.
cve5Modified := baseOSV.Modified
if nvd.Modified.After(cve5Modified) {
baseOSV.Modified = nvd.Modified
cve5Modified := baseOSV.GetModified()
if nvd.GetModified().AsTime().After(cve5Modified.AsTime()) {
baseOSV.Modified = nvd.GetModified()
}

cve5Published := baseOSV.Published
if nvd.Published.Before(cve5Published) {
baseOSV.Published = nvd.Published
cve5Published := baseOSV.GetPublished()
if nvd.GetPublished().AsTime().Before(cve5Published.AsTime()) {
baseOSV.Published = nvd.GetPublished()
}

// Merge aliases, ensuring no duplicates.
aliasMap := make(map[string]bool)
for _, alias := range baseOSV.Aliases {
for _, alias := range baseOSV.GetAliases() {
aliasMap[alias] = true
}
for _, alias := range nvd.Aliases {
for _, alias := range nvd.GetAliases() {
if !aliasMap[alias] {
baseOSV.Aliases = append(baseOSV.Aliases, alias)
aliasMap[alias] = true
Expand All @@ -258,7 +257,7 @@ func combineTwoOSVRecords(cve5 osvschema.Vulnerability, nvd osvschema.Vulnerabil
// If a match is found, it merges the version range information, preferring the entry
// with more ranges. Unmatched nvdAffected packages are appended.
// It returns a new slice and does not modify cve5Affected in place.
func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []osvschema.Affected) []osvschema.Affected {
func pickAffectedInformation(cve5Affected []*osvschema.Affected, nvdAffected []*osvschema.Affected) []*osvschema.Affected {
if len(nvdAffected) == 0 {
return cve5Affected
}
Expand All @@ -267,31 +266,31 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
return nvdAffected
}

nvdRepoMap := make(map[string][]osvschema.Range)
nvdRepoMap := make(map[string][]*osvschema.Range)
for _, affected := range nvdAffected {
for _, r := range affected.Ranges {
if r.Repo != "" {
repo := strings.ToLower(r.Repo)
for _, r := range affected.GetRanges() {
if r.GetRepo() != "" {
repo := strings.ToLower(r.GetRepo())
nvdRepoMap[repo] = append(nvdRepoMap[repo], r)
}
}
}

cve5RepoMap := make(map[string][]osvschema.Range)
cve5RepoMap := make(map[string][]*osvschema.Range)
for _, affected := range cve5Affected {
for _, r := range affected.Ranges {
if r.Repo != "" {
repo := strings.ToLower(r.Repo)
for _, r := range affected.GetRanges() {
if r.GetRepo() != "" {
repo := strings.ToLower(r.GetRepo())
cve5RepoMap[repo] = append(cve5RepoMap[repo], r)
}
}
}

newRepoAffectedMap := make(map[string]osvschema.Affected)
newRepoAffectedMap := make(map[string]*osvschema.Affected)

for repo, cveRanges := range cve5RepoMap {
if nvdRanges, ok := nvdRepoMap[repo]; ok {
var newAffectedRanges []osvschema.Range
var newAffectedRanges []*osvschema.Range

// Found a match. If NVD has more ranges, use its ranges.
if len(nvdRanges) > len(cveRanges) {
Expand All @@ -300,8 +299,8 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
} else if len(nvdRanges) < len(cveRanges) {
newAffectedRanges = cveRanges
} else if len(cveRanges) == 1 && len(nvdRanges) == 1 {
c5Intro, c5Fixed := getRangeBoundaryVersions(cveRanges[0].Events)
nvdIntro, nvdFixed := getRangeBoundaryVersions(nvdRanges[0].Events)
c5Intro, c5Fixed := getRangeBoundaryVersions(cveRanges[0].GetEvents())
nvdIntro, nvdFixed := getRangeBoundaryVersions(nvdRanges[0].GetEvents())

// Prefer cve5 data, but use nvd data if cve5 data is missing.
if c5Intro == "" {
Expand All @@ -314,30 +313,30 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os
if c5Intro != "" || c5Fixed != "" {
newRange := cves.BuildVersionRange(c5Intro, "", c5Fixed)
newRange.Repo = repo
newRange.Type = osvschema.RangeGit // Preserve the repo
newRange.Type = osvschema.Range_GIT // Preserve the repo
newAffectedRanges = append(newAffectedRanges, newRange)
}
}
// Remove from map so we know which NVD packages are left.
delete(nvdRepoMap, repo)
newRepoAffectedMap[repo] = osvschema.Affected{
newRepoAffectedMap[repo] = &osvschema.Affected{
Ranges: newAffectedRanges,
}
} else {
newRepoAffectedMap[repo] = osvschema.Affected{
newRepoAffectedMap[repo] = &osvschema.Affected{
Ranges: cveRanges,
}
}
}

// Add remaining NVD packages that were not in cve5.
for repo, nvdRange := range nvdRepoMap {
newRepoAffectedMap[repo] = osvschema.Affected{
newRepoAffectedMap[repo] = &osvschema.Affected{
Ranges: nvdRange,
}
}

var combinedAffected []osvschema.Affected //nolint:prealloc
var combinedAffected []*osvschema.Affected //nolint:prealloc
for _, aff := range newRepoAffectedMap {
combinedAffected = append(combinedAffected, aff)
}
Expand All @@ -347,13 +346,13 @@ func pickAffectedInformation(cve5Affected []osvschema.Affected, nvdAffected []os

// getRangeBoundaryVersions extracts the introduced and fixed versions from a slice of OSV events.
// It iterates through the events and returns the last non-empty "introduced" and "fixed" versions found.
func getRangeBoundaryVersions(events []osvschema.Event) (introduced, fixed string) {
func getRangeBoundaryVersions(events []*osvschema.Event) (introduced, fixed string) {
for _, e := range events {
if e.Introduced != "0" && e.Introduced != "" {
introduced = e.Introduced
if e.GetIntroduced() != "0" && e.GetIntroduced() != "" {
introduced = e.GetIntroduced()
}
if e.Fixed != "" {
fixed = e.Fixed
if e.GetFixed() != "" {
fixed = e.GetFixed()
}
}

Expand Down
Loading
Loading