OSV artifact generation for use in vulnerabilities repository (#42203)
**Related issue:** Resolves #41571 **Full Artifacts:** Ubuntu 14.04: 901 KB Ubuntu 16.04: 2.0 MB Ubuntu 18.04: 4.3 MB Ubuntu 20.04: 5.9 MB Ubuntu 22.04: 5.6 MB Ubuntu 24.04: 1.7 MB Ubuntu 24.10: 4.4 KB Ubuntu 25.04: 6.0 KB Ubuntu 25.10: 207 KB **Total Size:** All artifacts (full + deltas): 31 MB (was 54 MB) Full artifacts only: ~20 MB (was ~27 MB) Delta artifacts: ~11 MB (was ~27 MB) ## Testing - [x] Added/updated automated tests - [x] QA'd all new/changed functionality manually <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added a vulnerability data processor that scans OSV JSON inputs, aggregates per-Ubuntu-version artifacts, supports inclusive/exclusive version filters, and can emit optional “today”/“yesterday” delta artifacts. * Added a repository sync-and-change-detection tool that generates de-duplicated lists of CVE-related files changed today and yesterday. * Processor expands certain package names (e.g., emacs) into additional package entries for broader coverage. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Konstantin Sykulev
@@ -0,0 +1,518 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type OSVData struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
ID string `json:"id"`
|
||||
Published string `json:"published"`
|
||||
Modified string `json:"modified"`
|
||||
Details string `json:"details"`
|
||||
Affected []Affected `json:"affected"`
|
||||
Upstream []string `json:"upstream,omitempty"`
|
||||
Related []string `json:"related,omitempty"`
|
||||
}
|
||||
|
||||
type Affected struct {
|
||||
Package Package `json:"package"`
|
||||
Ranges []Range `json:"ranges"`
|
||||
Versions []string `json:"versions,omitempty"`
|
||||
EcosystemSpecific map[string]any `json:"ecosystem_specific,omitempty"`
|
||||
DatabaseSpecific map[string]any `json:"database_specific,omitempty"`
|
||||
}
|
||||
|
||||
type Package struct {
|
||||
Ecosystem string `json:"ecosystem"`
|
||||
Name string `json:"name"`
|
||||
Purl string `json:"purl,omitempty"`
|
||||
}
|
||||
|
||||
type Range struct {
|
||||
Type string `json:"type"`
|
||||
Events []Event `json:"events"`
|
||||
}
|
||||
|
||||
type Event struct {
|
||||
Introduced string `json:"introduced,omitempty"`
|
||||
Fixed string `json:"fixed,omitempty"`
|
||||
}
|
||||
|
||||
type ProcessedVuln struct {
|
||||
CVE string `json:"cve"`
|
||||
Published string `json:"published"`
|
||||
Modified string `json:"modified"`
|
||||
Introduced string `json:"introduced,omitempty"`
|
||||
Fixed string `json:"fixed,omitempty"`
|
||||
Versions []string `json:"versions,omitempty"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
InputDir string
|
||||
OutputDir string
|
||||
Versions string
|
||||
ExcludeVersions string
|
||||
ChangedFilesToday string
|
||||
ChangedFilesYesterday string
|
||||
DateStr string
|
||||
YesterdayStr string
|
||||
GeneratedTimestamp string
|
||||
RunTime time.Time
|
||||
}
|
||||
|
||||
type ArtifactData struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
UbuntuVersion string `json:"ubuntu_version"`
|
||||
Generated string `json:"generated"`
|
||||
TotalCVEs int `json:"total_cves"`
|
||||
TotalPackages int `json:"total_packages"`
|
||||
Vulnerabilities map[string][]ProcessedVuln `json:"vulnerabilities"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
inputDir := flag.String("input", "/tmp/ubuntu-osv", "Input directory with OSV JSON files")
|
||||
outputDir := flag.String("output", "./artifacts", "Output directory for artifacts")
|
||||
versions := flag.String("versions", "", "Comma-separated Ubuntu versions to process (inclusive)")
|
||||
excludeVersions := flag.String("exclude-versions", "", "Comma-separated Ubuntu versions to exclude (ignored if --versions is set)")
|
||||
changedFilesToday := flag.String("changed-files-today", "", "Path to file containing CVE files changed today (generates today's deltas)")
|
||||
changedFilesYesterday := flag.String("changed-files-yesterday", "", "Path to file containing CVE files changed yesterday (generates yesterday's deltas)")
|
||||
flag.Parse()
|
||||
|
||||
runTime := time.Now().UTC()
|
||||
|
||||
cfg := Config{
|
||||
InputDir: *inputDir,
|
||||
OutputDir: *outputDir,
|
||||
Versions: *versions,
|
||||
ExcludeVersions: *excludeVersions,
|
||||
ChangedFilesToday: *changedFilesToday,
|
||||
ChangedFilesYesterday: *changedFilesYesterday,
|
||||
DateStr: runTime.Format("2006-01-02"),
|
||||
YesterdayStr: runTime.AddDate(0, 0, -1).Format("2006-01-02"),
|
||||
GeneratedTimestamp: runTime.Format(time.RFC3339),
|
||||
RunTime: runTime,
|
||||
}
|
||||
|
||||
if err := run(cfg); err != nil {
|
||||
log.Fatalf("Error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg Config) error {
|
||||
if err := os.MkdirAll(cfg.OutputDir, 0o755); err != nil {
|
||||
return fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
|
||||
// Build version filter
|
||||
targetVersions, excludedVersions := buildVersionFilter(cfg.Versions, cfg.ExcludeVersions)
|
||||
switch {
|
||||
case targetVersions != nil:
|
||||
log.Printf("Processing OSV files from %s for versions: %s", cfg.InputDir, cfg.Versions)
|
||||
case excludedVersions != nil:
|
||||
log.Printf("Processing OSV files from %s (auto-detecting, excluding: %s)", cfg.InputDir, cfg.ExcludeVersions)
|
||||
default:
|
||||
log.Printf("Processing OSV files from %s (auto-detecting all versions)", cfg.InputDir)
|
||||
}
|
||||
|
||||
// Load changed CVE files for delta generation
|
||||
var todayCVEFiles, yesterdayCVEFiles map[string]bool
|
||||
generateTodayDeltas := cfg.ChangedFilesToday != ""
|
||||
generateYesterdayDeltas := cfg.ChangedFilesYesterday != ""
|
||||
|
||||
if generateTodayDeltas {
|
||||
log.Printf("Loading today's changed CVE files from %s", cfg.ChangedFilesToday)
|
||||
var err error
|
||||
todayCVEFiles, err = loadChangedFiles(cfg.ChangedFilesToday)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load today's changed files: %w", err)
|
||||
}
|
||||
log.Printf("Found %d CVE files changed today", len(todayCVEFiles))
|
||||
}
|
||||
|
||||
if generateYesterdayDeltas {
|
||||
log.Printf("Loading yesterday's changed CVE files from %s", cfg.ChangedFilesYesterday)
|
||||
var err error
|
||||
yesterdayCVEFiles, err = loadChangedFiles(cfg.ChangedFilesYesterday)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load yesterday's changed files: %w", err)
|
||||
}
|
||||
log.Printf("Found %d CVE files changed yesterday", len(yesterdayCVEFiles))
|
||||
}
|
||||
|
||||
artifacts := make(map[string]*ArtifactData)
|
||||
todayArtifacts := make(map[string]*ArtifactData)
|
||||
yesterdayArtifacts := make(map[string]*ArtifactData)
|
||||
|
||||
filesProcessed := 0
|
||||
filesSkipped := 0
|
||||
|
||||
err := filepath.Walk(cfg.InputDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() || !strings.HasSuffix(path, ".json") {
|
||||
return nil
|
||||
}
|
||||
|
||||
osvData, err := parseOSVFile(path)
|
||||
if err != nil {
|
||||
log.Printf("Failed to parse %s: %v", path, err)
|
||||
filesSkipped++
|
||||
return nil
|
||||
}
|
||||
|
||||
inToday := false
|
||||
inYesterday := false
|
||||
if generateTodayDeltas {
|
||||
inToday = shouldIncludeInDelta(cfg.InputDir, path, todayCVEFiles)
|
||||
}
|
||||
if generateYesterdayDeltas {
|
||||
inYesterday = shouldIncludeInDelta(cfg.InputDir, path, yesterdayCVEFiles)
|
||||
}
|
||||
|
||||
for _, affected := range osvData.Affected {
|
||||
ecosystem := affected.Package.Ecosystem
|
||||
packageName := affected.Package.Name
|
||||
|
||||
ubuntuVer := extractUbuntuVersion(ecosystem)
|
||||
if ubuntuVer == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Filter versions based on flags
|
||||
if targetVersions != nil {
|
||||
// Inclusive mode: only process if in target list
|
||||
if !targetVersions[ubuntuVer] {
|
||||
continue
|
||||
}
|
||||
} else if excludedVersions != nil {
|
||||
// Exclusive mode: skip if in excluded list
|
||||
if excludedVersions[ubuntuVer] {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Otherwise auto-detect all versions (no filtering)
|
||||
|
||||
cveID := extractCVEID(osvData)
|
||||
if cveID == "" {
|
||||
cveID = osvData.ID
|
||||
}
|
||||
|
||||
introduced, fixed := extractVersionRange(affected.Ranges)
|
||||
|
||||
vuln := ProcessedVuln{
|
||||
CVE: cveID,
|
||||
Published: osvData.Published,
|
||||
Modified: osvData.Modified,
|
||||
Introduced: introduced,
|
||||
Fixed: fixed,
|
||||
Versions: affected.Versions,
|
||||
}
|
||||
|
||||
// Apply any transformations/filters to modify the package name or cve
|
||||
packages, modifiedVuln := transformVuln(packageName, cveID, &vuln)
|
||||
if packages == nil {
|
||||
continue
|
||||
}
|
||||
// Use modified vulnerability if provided, otherwise use original
|
||||
vulnToUse := &vuln
|
||||
if modifiedVuln != nil {
|
||||
vulnToUse = modifiedVuln
|
||||
}
|
||||
|
||||
for _, pkg := range packages {
|
||||
if _, exists := artifacts[ubuntuVer]; !exists {
|
||||
artifacts[ubuntuVer] = &ArtifactData{
|
||||
SchemaVersion: "1.0",
|
||||
UbuntuVersion: ubuntuVer,
|
||||
Vulnerabilities: make(map[string][]ProcessedVuln),
|
||||
}
|
||||
}
|
||||
artifacts[ubuntuVer].Vulnerabilities[pkg] = append(artifacts[ubuntuVer].Vulnerabilities[pkg], *vulnToUse)
|
||||
}
|
||||
|
||||
// Add to today's delta artifact if this file was changed today
|
||||
if inToday {
|
||||
for _, pkg := range packages {
|
||||
if _, exists := todayArtifacts[ubuntuVer]; !exists {
|
||||
todayArtifacts[ubuntuVer] = &ArtifactData{
|
||||
SchemaVersion: "1.0",
|
||||
UbuntuVersion: ubuntuVer,
|
||||
Vulnerabilities: make(map[string][]ProcessedVuln),
|
||||
}
|
||||
}
|
||||
todayArtifacts[ubuntuVer].Vulnerabilities[pkg] = append(todayArtifacts[ubuntuVer].Vulnerabilities[pkg], *vulnToUse)
|
||||
}
|
||||
}
|
||||
|
||||
// Add to yesterday's delta artifact if this file was changed yesterday
|
||||
if inYesterday {
|
||||
for _, pkg := range packages {
|
||||
if _, exists := yesterdayArtifacts[ubuntuVer]; !exists {
|
||||
yesterdayArtifacts[ubuntuVer] = &ArtifactData{
|
||||
SchemaVersion: "1.0",
|
||||
UbuntuVersion: ubuntuVer,
|
||||
Vulnerabilities: make(map[string][]ProcessedVuln),
|
||||
}
|
||||
}
|
||||
yesterdayArtifacts[ubuntuVer].Vulnerabilities[pkg] = append(yesterdayArtifacts[ubuntuVer].Vulnerabilities[pkg], *vulnToUse)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
filesProcessed++
|
||||
if filesProcessed%1000 == 0 {
|
||||
log.Printf("Processed %d files...", filesProcessed)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("error walking directory: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Processed %d files, skipped %d files", filesProcessed, filesSkipped)
|
||||
log.Printf("Discovered %d Ubuntu versions", len(artifacts))
|
||||
|
||||
// Write full artifacts
|
||||
for ver, artifact := range artifacts {
|
||||
artifact.Generated = cfg.GeneratedTimestamp
|
||||
artifact.TotalCVEs = countTotalCVEs(artifact)
|
||||
artifact.TotalPackages = len(artifact.Vulnerabilities)
|
||||
|
||||
outputFile := filepath.Join(cfg.OutputDir, fmt.Sprintf("osv-ubuntu-%s-%s.json.gz",
|
||||
strings.ReplaceAll(ver, ".", ""),
|
||||
cfg.DateStr))
|
||||
|
||||
if err := writeArtifact(outputFile, artifact); err != nil {
|
||||
return fmt.Errorf("failed to write artifact for Ubuntu %s: %w", ver, err)
|
||||
}
|
||||
|
||||
log.Printf("Ubuntu %s: %d packages, %d CVEs -> %s",
|
||||
ver, artifact.TotalPackages, artifact.TotalCVEs, outputFile)
|
||||
}
|
||||
|
||||
// Write delta artifacts (if any were generated)
|
||||
if generateTodayDeltas && len(todayArtifacts) > 0 {
|
||||
log.Printf("\nWriting today's delta artifacts (%s)...", cfg.DateStr)
|
||||
for ver, artifact := range todayArtifacts {
|
||||
artifact.Generated = cfg.GeneratedTimestamp
|
||||
artifact.TotalCVEs = countTotalCVEs(artifact)
|
||||
artifact.TotalPackages = len(artifact.Vulnerabilities)
|
||||
|
||||
outputFile := filepath.Join(cfg.OutputDir, fmt.Sprintf("osv-ubuntu-%s-delta-%s.json.gz",
|
||||
strings.ReplaceAll(ver, ".", ""), cfg.DateStr))
|
||||
|
||||
if err := writeArtifact(outputFile, artifact); err != nil {
|
||||
return fmt.Errorf("failed to write today's delta for Ubuntu %s: %w", ver, err)
|
||||
}
|
||||
|
||||
log.Printf("Ubuntu %s (today): %d packages, %d CVEs -> %s",
|
||||
ver, artifact.TotalPackages, artifact.TotalCVEs, outputFile)
|
||||
}
|
||||
}
|
||||
|
||||
if generateYesterdayDeltas && len(yesterdayArtifacts) > 0 {
|
||||
log.Printf("\nWriting yesterday's delta artifacts (%s)...", cfg.YesterdayStr)
|
||||
for ver, artifact := range yesterdayArtifacts {
|
||||
artifact.Generated = cfg.GeneratedTimestamp
|
||||
artifact.TotalCVEs = countTotalCVEs(artifact)
|
||||
artifact.TotalPackages = len(artifact.Vulnerabilities)
|
||||
|
||||
outputFile := filepath.Join(cfg.OutputDir, fmt.Sprintf("osv-ubuntu-%s-delta-%s.json.gz",
|
||||
strings.ReplaceAll(ver, ".", ""), cfg.YesterdayStr))
|
||||
|
||||
if err := writeArtifact(outputFile, artifact); err != nil {
|
||||
return fmt.Errorf("failed to write yesterday's delta for Ubuntu %s: %w", ver, err)
|
||||
}
|
||||
|
||||
log.Printf("Ubuntu %s (yesterday): %d packages, %d CVEs -> %s",
|
||||
ver, artifact.TotalPackages, artifact.TotalCVEs, outputFile)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildVersionFilter(versions, excludeVersions string) (targetVersions, excludedVersions map[string]bool) {
|
||||
if versions != "" {
|
||||
// Inclusive mode: only process specified versions
|
||||
targetVersions = make(map[string]bool)
|
||||
for ver := range strings.SplitSeq(versions, ",") {
|
||||
trimmed := strings.TrimSpace(ver)
|
||||
if trimmed != "" {
|
||||
targetVersions[trimmed] = true
|
||||
}
|
||||
}
|
||||
// If no valid versions were parsed, fall back to auto-detect
|
||||
if len(targetVersions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return targetVersions, nil
|
||||
}
|
||||
|
||||
if excludeVersions != "" {
|
||||
// Exclusive mode: process all except specified versions
|
||||
excludedVersions = make(map[string]bool)
|
||||
for ver := range strings.SplitSeq(excludeVersions, ",") {
|
||||
trimmed := strings.TrimSpace(ver)
|
||||
if trimmed != "" {
|
||||
excludedVersions[trimmed] = true
|
||||
}
|
||||
}
|
||||
// If no valid versions were parsed, fall back to auto-detect
|
||||
if len(excludedVersions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, excludedVersions
|
||||
}
|
||||
|
||||
// Auto-detect all versions
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func shouldIncludeInDelta(inputDir, filePath string, changedFiles map[string]bool) bool {
|
||||
relPath, err := filepath.Rel(inputDir, filePath)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
normalizedRelPath := strings.TrimPrefix(filepath.ToSlash(relPath), "osv/cve/")
|
||||
fullRelPath := "osv/cve/" + normalizedRelPath
|
||||
|
||||
return changedFiles[fullRelPath]
|
||||
}
|
||||
|
||||
func parseOSVFile(path string) (*OSVData, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var osv OSVData
|
||||
if err := json.Unmarshal(data, &osv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &osv, nil
|
||||
}
|
||||
|
||||
func extractUbuntuVersion(ecosystem string) string {
|
||||
// Example: "Ubuntu:24.04:LTS" -> "24.04"
|
||||
// Example: "Ubuntu:Pro:22.04:LTS" -> "22.04"
|
||||
for part := range strings.SplitSeq(ecosystem, ":") {
|
||||
// Look for version pattern like "24.04", "22.04", "20.04"
|
||||
if len(part) == 5 && strings.Contains(part, ".") {
|
||||
return part
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractCVEID(osv *OSVData) string {
|
||||
for _, upstream := range osv.Upstream {
|
||||
if strings.HasPrefix(upstream, "CVE-") {
|
||||
return upstream
|
||||
}
|
||||
}
|
||||
|
||||
if strings.HasPrefix(osv.ID, "CVE-") {
|
||||
return osv.ID
|
||||
}
|
||||
|
||||
if strings.HasPrefix(osv.ID, "UBUNTU-CVE-") {
|
||||
return strings.TrimPrefix(osv.ID, "UBUNTU-")
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractVersionRange(ranges []Range) (introduced string, fixed string) {
|
||||
for _, r := range ranges {
|
||||
if r.Type == "ECOSYSTEM" {
|
||||
for _, event := range r.Events {
|
||||
if event.Introduced != "" && introduced == "" {
|
||||
introduced = event.Introduced
|
||||
}
|
||||
if event.Fixed != "" && fixed == "" {
|
||||
fixed = event.Fixed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func countTotalCVEs(artifact *ArtifactData) int {
|
||||
seen := make(map[string]bool)
|
||||
for _, vulns := range artifact.Vulnerabilities {
|
||||
for _, vuln := range vulns {
|
||||
seen[vuln.CVE] = true
|
||||
}
|
||||
}
|
||||
return len(seen)
|
||||
}
|
||||
|
||||
func writeArtifact(path string, artifact *ArtifactData) (err error) {
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if cerr := file.Close(); err == nil && cerr != nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
gzWriter := gzip.NewWriter(file)
|
||||
defer func() {
|
||||
if cerr := gzWriter.Close(); err == nil && cerr != nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
|
||||
encoder := json.NewEncoder(gzWriter)
|
||||
|
||||
if err = encoder.Encode(artifact); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadChangedFiles(changedFilesPath string) (map[string]bool, error) {
|
||||
file, err := os.Open(changedFilesPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open changed files list: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
changedFiles := make(map[string]bool)
|
||||
scanner := bufio.NewScanner(file)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
changedFiles[line] = true
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error reading changed files: %w", err)
|
||||
}
|
||||
|
||||
return changedFiles, nil
|
||||
}
|
||||
@@ -0,0 +1,860 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExtractUbuntuVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ecosystem string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "Standard Ubuntu LTS",
|
||||
ecosystem: "Ubuntu:24.04:LTS",
|
||||
expected: "24.04",
|
||||
},
|
||||
{
|
||||
name: "Ubuntu Pro",
|
||||
ecosystem: "Ubuntu:Pro:22.04:LTS",
|
||||
expected: "22.04",
|
||||
},
|
||||
{
|
||||
name: "Ubuntu 20.04",
|
||||
ecosystem: "Ubuntu:20.04:LTS",
|
||||
expected: "20.04",
|
||||
},
|
||||
{
|
||||
name: "Ubuntu 18.04",
|
||||
ecosystem: "Ubuntu:18.04:LTS",
|
||||
expected: "18.04",
|
||||
},
|
||||
{
|
||||
name: "No version pattern",
|
||||
ecosystem: "Ubuntu:LTS",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Empty string",
|
||||
ecosystem: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Not Ubuntu",
|
||||
ecosystem: "Debian:12:stable",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Version without LTS",
|
||||
ecosystem: "Ubuntu:24.04",
|
||||
expected: "24.04",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractUbuntuVersion(tt.ecosystem)
|
||||
require.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCVEID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
osv *OSVData
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "CVE in Upstream field",
|
||||
osv: &OSVData{
|
||||
ID: "UBUNTU-CVE-2024-1234",
|
||||
Upstream: []string{"CVE-2024-1234", "https://example.com"},
|
||||
},
|
||||
expected: "CVE-2024-1234",
|
||||
},
|
||||
{
|
||||
name: "CVE as ID",
|
||||
osv: &OSVData{
|
||||
ID: "CVE-2024-5678",
|
||||
Upstream: []string{},
|
||||
},
|
||||
expected: "CVE-2024-5678",
|
||||
},
|
||||
{
|
||||
name: "UBUNTU-CVE prefix",
|
||||
osv: &OSVData{
|
||||
ID: "UBUNTU-CVE-2024-9999",
|
||||
Upstream: []string{},
|
||||
},
|
||||
expected: "CVE-2024-9999",
|
||||
},
|
||||
{
|
||||
name: "No CVE found",
|
||||
osv: &OSVData{
|
||||
ID: "SOME-OTHER-ID",
|
||||
Upstream: []string{"https://example.com"},
|
||||
},
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "Multiple upstreams with CVE first",
|
||||
osv: &OSVData{
|
||||
ID: "UBUNTU-123",
|
||||
Upstream: []string{"CVE-2024-1111", "CVE-2024-2222"},
|
||||
},
|
||||
expected: "CVE-2024-1111",
|
||||
},
|
||||
{
|
||||
name: "Empty upstream, no CVE in ID",
|
||||
osv: &OSVData{
|
||||
ID: "USN-1234-1",
|
||||
Upstream: []string{},
|
||||
},
|
||||
expected: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := extractCVEID(tt.osv)
|
||||
require.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractVersionRange(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ranges []Range
|
||||
expectedIntroduced string
|
||||
expectedFixed string
|
||||
}{
|
||||
{
|
||||
name: "Simple range with introduced and fixed",
|
||||
ranges: []Range{
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Introduced: "1.0.0"},
|
||||
{Fixed: "2.0.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedIntroduced: "1.0.0",
|
||||
expectedFixed: "2.0.0",
|
||||
},
|
||||
{
|
||||
name: "Only introduced version",
|
||||
ranges: []Range{
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Introduced: "1.5.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedIntroduced: "1.5.0",
|
||||
expectedFixed: "",
|
||||
},
|
||||
{
|
||||
name: "Only fixed version",
|
||||
ranges: []Range{
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Fixed: "3.0.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedIntroduced: "",
|
||||
expectedFixed: "3.0.0",
|
||||
},
|
||||
{
|
||||
name: "Multiple ranges, first ECOSYSTEM wins",
|
||||
ranges: []Range{
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Introduced: "1.0.0"},
|
||||
{Fixed: "2.0.0"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Introduced: "3.0.0"},
|
||||
{Fixed: "4.0.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedIntroduced: "1.0.0",
|
||||
expectedFixed: "2.0.0",
|
||||
},
|
||||
{
|
||||
name: "Non-ECOSYSTEM range ignored",
|
||||
ranges: []Range{
|
||||
{
|
||||
Type: "GIT",
|
||||
Events: []Event{
|
||||
{Introduced: "abc123"},
|
||||
{Fixed: "def456"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Introduced: "2.0.0"},
|
||||
{Fixed: "2.5.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedIntroduced: "2.0.0",
|
||||
expectedFixed: "2.5.0",
|
||||
},
|
||||
{
|
||||
name: "Empty ranges",
|
||||
ranges: []Range{},
|
||||
expectedIntroduced: "",
|
||||
expectedFixed: "",
|
||||
},
|
||||
{
|
||||
name: "Multiple events in single range",
|
||||
ranges: []Range{
|
||||
{
|
||||
Type: "ECOSYSTEM",
|
||||
Events: []Event{
|
||||
{Introduced: "0"},
|
||||
{Fixed: "1.2.3"},
|
||||
{Introduced: "2.0.0"}, // This should be ignored (first introduced wins)
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedIntroduced: "0",
|
||||
expectedFixed: "1.2.3",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
introduced, fixed := extractVersionRange(tt.ranges)
|
||||
require.Equal(t, tt.expectedIntroduced, introduced)
|
||||
require.Equal(t, tt.expectedFixed, fixed)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountTotalCVEs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
artifact *ArtifactData
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "Multiple packages with unique CVEs",
|
||||
artifact: &ArtifactData{
|
||||
Vulnerabilities: map[string][]ProcessedVuln{
|
||||
"curl": {
|
||||
{CVE: "CVE-2024-1234"},
|
||||
{CVE: "CVE-2024-5678"},
|
||||
},
|
||||
"openssl": {
|
||||
{CVE: "CVE-2024-9999"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: 3,
|
||||
},
|
||||
{
|
||||
name: "Duplicate CVEs across packages",
|
||||
artifact: &ArtifactData{
|
||||
Vulnerabilities: map[string][]ProcessedVuln{
|
||||
"emacs": {
|
||||
{CVE: "CVE-2024-39331"},
|
||||
},
|
||||
"emacs-common": {
|
||||
{CVE: "CVE-2024-39331"},
|
||||
},
|
||||
"emacs-el": {
|
||||
{CVE: "CVE-2024-39331"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: 1, // Deduplicated
|
||||
},
|
||||
{
|
||||
name: "Mix of unique and duplicate CVEs",
|
||||
artifact: &ArtifactData{
|
||||
Vulnerabilities: map[string][]ProcessedVuln{
|
||||
"package1": {
|
||||
{CVE: "CVE-2024-1111"},
|
||||
{CVE: "CVE-2024-2222"},
|
||||
},
|
||||
"package2": {
|
||||
{CVE: "CVE-2024-1111"}, // Duplicate
|
||||
{CVE: "CVE-2024-3333"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: 3, // CVE-2024-1111, CVE-2024-2222, CVE-2024-3333
|
||||
},
|
||||
{
|
||||
name: "Empty artifact",
|
||||
artifact: &ArtifactData{
|
||||
Vulnerabilities: map[string][]ProcessedVuln{},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "Package with no vulnerabilities",
|
||||
artifact: &ArtifactData{
|
||||
Vulnerabilities: map[string][]ProcessedVuln{
|
||||
"safe-package": {},
|
||||
},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "Single package, single CVE",
|
||||
artifact: &ArtifactData{
|
||||
Vulnerabilities: map[string][]ProcessedVuln{
|
||||
"apache2": {
|
||||
{CVE: "CVE-2024-7777"},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := countTotalCVEs(tt.artifact)
|
||||
require.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildVersionFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
versions string
|
||||
excludeVersions string
|
||||
expectedTargetVersions map[string]bool
|
||||
expectedExcludedVersions map[string]bool
|
||||
}{
|
||||
{
|
||||
name: "Inclusive mode: single version",
|
||||
versions: "20.04",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: multiple versions",
|
||||
versions: "20.04,22.04,24.04",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true, "22.04": true, "24.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: with spaces",
|
||||
versions: "20.04, 22.04, 24.04",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true, "22.04": true, "24.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Exclusive mode: single version",
|
||||
versions: "",
|
||||
excludeVersions: "14.04",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: map[string]bool{"14.04": true},
|
||||
},
|
||||
{
|
||||
name: "Exclusive mode: multiple versions",
|
||||
versions: "",
|
||||
excludeVersions: "14.04,16.04,24.10",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: map[string]bool{"14.04": true, "16.04": true, "24.10": true},
|
||||
},
|
||||
{
|
||||
name: "Exclusive mode: with spaces",
|
||||
versions: "",
|
||||
excludeVersions: "14.04, 16.04, 24.10",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: map[string]bool{"14.04": true, "16.04": true, "24.10": true},
|
||||
},
|
||||
{
|
||||
name: "Auto-detect mode: both empty",
|
||||
versions: "",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive takes precedence: both provided",
|
||||
versions: "20.04,22.04",
|
||||
excludeVersions: "14.04,16.04",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true, "22.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: trailing comma ignored",
|
||||
versions: "20.04,22.04,",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true, "22.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: leading comma ignored",
|
||||
versions: ",20.04,22.04",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true, "22.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: multiple commas ignored",
|
||||
versions: "20.04,,22.04",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: map[string]bool{"20.04": true, "22.04": true},
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Exclusive mode: trailing comma ignored",
|
||||
versions: "",
|
||||
excludeVersions: "14.04,16.04,",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: map[string]bool{"14.04": true, "16.04": true},
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: only empty strings falls back to auto-detect",
|
||||
versions: ",,",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Inclusive mode: only whitespace falls back to auto-detect",
|
||||
versions: " , , ",
|
||||
excludeVersions: "",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
{
|
||||
name: "Exclusive mode: only empty strings falls back to auto-detect",
|
||||
versions: "",
|
||||
excludeVersions: ",,",
|
||||
expectedTargetVersions: nil,
|
||||
expectedExcludedVersions: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
targetVersions, excludedVersions := buildVersionFilter(tt.versions, tt.excludeVersions)
|
||||
require.Equal(t, tt.expectedTargetVersions, targetVersions)
|
||||
require.Equal(t, tt.expectedExcludedVersions, excludedVersions)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldIncludeInDelta(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
inputDir string
|
||||
filePath string
|
||||
changedFiles map[string]bool
|
||||
expectedMatch bool
|
||||
}{
|
||||
{
|
||||
name: "Unix path: file in changed set",
|
||||
inputDir: "/tmp/ubuntu-osv",
|
||||
filePath: "/tmp/ubuntu-osv/osv/cve/CVE-2024-1234.json",
|
||||
changedFiles: map[string]bool{
|
||||
"osv/cve/CVE-2024-1234.json": true,
|
||||
},
|
||||
expectedMatch: true,
|
||||
},
|
||||
{
|
||||
name: "Unix path: file not in changed set",
|
||||
inputDir: "/tmp/ubuntu-osv",
|
||||
filePath: "/tmp/ubuntu-osv/osv/cve/CVE-2024-9999.json",
|
||||
changedFiles: map[string]bool{
|
||||
"osv/cve/CVE-2024-1234.json": true,
|
||||
},
|
||||
expectedMatch: false,
|
||||
},
|
||||
{
|
||||
name: "Nested directory: file in changed set",
|
||||
inputDir: "/data/osv",
|
||||
filePath: "/data/osv/osv/cve/2024/CVE-2024-1111.json",
|
||||
changedFiles: map[string]bool{
|
||||
"osv/cve/2024/CVE-2024-1111.json": true,
|
||||
},
|
||||
expectedMatch: true,
|
||||
},
|
||||
{
|
||||
name: "File already has osv/cve prefix in relative path",
|
||||
inputDir: "/workspace",
|
||||
filePath: "/workspace/osv/cve/CVE-2024-2222.json",
|
||||
changedFiles: map[string]bool{
|
||||
"osv/cve/CVE-2024-2222.json": true,
|
||||
},
|
||||
expectedMatch: true,
|
||||
},
|
||||
{
|
||||
name: "Changed files with leading slash (should not match)",
|
||||
inputDir: "/tmp/ubuntu-osv",
|
||||
filePath: "/tmp/ubuntu-osv/osv/cve/CVE-2024-3333.json",
|
||||
changedFiles: map[string]bool{
|
||||
"/osv/cve/CVE-2024-3333.json": true, // Wrong: has leading slash
|
||||
},
|
||||
expectedMatch: false,
|
||||
},
|
||||
{
|
||||
name: "Changed files without osv/cve prefix (should not match)",
|
||||
inputDir: "/tmp/ubuntu-osv",
|
||||
filePath: "/tmp/ubuntu-osv/osv/cve/CVE-2024-4444.json",
|
||||
changedFiles: map[string]bool{
|
||||
"CVE-2024-4444.json": true, // Wrong: missing osv/cve/ prefix
|
||||
},
|
||||
expectedMatch: false,
|
||||
},
|
||||
{
|
||||
name: "Empty changed files set",
|
||||
inputDir: "/tmp/ubuntu-osv",
|
||||
filePath: "/tmp/ubuntu-osv/osv/cve/CVE-2024-5555.json",
|
||||
changedFiles: map[string]bool{},
|
||||
expectedMatch: false,
|
||||
},
|
||||
{
|
||||
name: "File outside input directory tree (relative path doesn't match)",
|
||||
inputDir: "/tmp/ubuntu-osv/subdir",
|
||||
filePath: "/tmp/other-dir/osv/cve/CVE-2024-6666.json",
|
||||
changedFiles: map[string]bool{
|
||||
"osv/cve/CVE-2024-6666.json": true,
|
||||
},
|
||||
expectedMatch: false, // filepath.Rel will work but path won't match
|
||||
},
|
||||
{
|
||||
name: "Multiple files in changed set, match one",
|
||||
inputDir: "/data/osv",
|
||||
filePath: "/data/osv/osv/cve/CVE-2024-7777.json",
|
||||
changedFiles: map[string]bool{
|
||||
"osv/cve/CVE-2024-1111.json": true,
|
||||
"osv/cve/CVE-2024-7777.json": true,
|
||||
"osv/cve/CVE-2024-9999.json": true,
|
||||
},
|
||||
expectedMatch: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := shouldIncludeInDelta(tt.inputDir, tt.filePath, tt.changedFiles)
|
||||
require.Equal(t, tt.expectedMatch, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRun(t *testing.T) {
|
||||
// Create temporary directories for input and output
|
||||
inputDir := t.TempDir()
|
||||
outputDir := t.TempDir()
|
||||
|
||||
// Create a simple test OSV file
|
||||
testOSVData := `{
|
||||
"schema_version": "1.0",
|
||||
"id": "USN-1234-1",
|
||||
"published": "2024-01-01T00:00:00Z",
|
||||
"modified": "2024-01-02T00:00:00Z",
|
||||
"details": "Test vulnerability",
|
||||
"affected": [{
|
||||
"package": {
|
||||
"ecosystem": "Ubuntu:22.04:LTS",
|
||||
"name": "test-package"
|
||||
},
|
||||
"ranges": [{
|
||||
"type": "ECOSYSTEM",
|
||||
"events": [
|
||||
{"introduced": "0"},
|
||||
{"fixed": "1.2.3"}
|
||||
]
|
||||
}]
|
||||
}],
|
||||
"upstream": ["CVE-2024-1234"]
|
||||
}`
|
||||
|
||||
// Write test file
|
||||
require.NoError(t, os.WriteFile(filepath.Join(inputDir, "CVE-2024-1234.json"), []byte(testOSVData), 0o644))
|
||||
|
||||
// Create config
|
||||
cfg := Config{
|
||||
InputDir: inputDir,
|
||||
OutputDir: outputDir,
|
||||
Versions: "",
|
||||
ExcludeVersions: "",
|
||||
ChangedFilesToday: "",
|
||||
ChangedFilesYesterday: "",
|
||||
DateStr: "2024-01-03",
|
||||
YesterdayStr: "2024-01-02",
|
||||
GeneratedTimestamp: "2024-01-03T00:00:00Z",
|
||||
RunTime: time.Date(2024, 1, 3, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
// Run the function
|
||||
err := run(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify output artifact was created
|
||||
expectedFile := filepath.Join(outputDir, "osv-ubuntu-2204-2024-01-03.json.gz")
|
||||
require.FileExists(t, expectedFile)
|
||||
|
||||
// Verify artifact content (decompress and check)
|
||||
artifact, err := readArtifact(expectedFile)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "1.0", artifact.SchemaVersion)
|
||||
require.Equal(t, "22.04", artifact.UbuntuVersion)
|
||||
require.Equal(t, 1, artifact.TotalCVEs)
|
||||
require.Equal(t, 1, artifact.TotalPackages)
|
||||
require.Contains(t, artifact.Vulnerabilities, "test-package")
|
||||
require.Len(t, artifact.Vulnerabilities["test-package"], 1)
|
||||
require.Equal(t, "CVE-2024-1234", artifact.Vulnerabilities["test-package"][0].CVE)
|
||||
}
|
||||
|
||||
func TestRunWithDeltaGeneration(t *testing.T) {
|
||||
// Create temporary directories
|
||||
inputDir := t.TempDir()
|
||||
outputDir := t.TempDir()
|
||||
changedFilesDir := t.TempDir()
|
||||
|
||||
// Create two test OSV files
|
||||
testOSVData1 := `{
|
||||
"schema_version": "1.0",
|
||||
"id": "USN-1234-1",
|
||||
"published": "2024-01-01T00:00:00Z",
|
||||
"modified": "2024-01-02T00:00:00Z",
|
||||
"affected": [{
|
||||
"package": {
|
||||
"ecosystem": "Ubuntu:22.04:LTS",
|
||||
"name": "changed-today-package"
|
||||
},
|
||||
"ranges": [{
|
||||
"type": "ECOSYSTEM",
|
||||
"events": [{"introduced": "0"}, {"fixed": "1.0"}]
|
||||
}]
|
||||
}],
|
||||
"upstream": ["CVE-2024-1111"]
|
||||
}`
|
||||
|
||||
testOSVData2 := `{
|
||||
"schema_version": "1.0",
|
||||
"id": "USN-5678-1",
|
||||
"published": "2024-01-01T00:00:00Z",
|
||||
"modified": "2024-01-02T00:00:00Z",
|
||||
"affected": [{
|
||||
"package": {
|
||||
"ecosystem": "Ubuntu:22.04:LTS",
|
||||
"name": "changed-yesterday-package"
|
||||
},
|
||||
"ranges": [{
|
||||
"type": "ECOSYSTEM",
|
||||
"events": [{"introduced": "0"}, {"fixed": "2.0"}]
|
||||
}]
|
||||
}],
|
||||
"upstream": ["CVE-2024-2222"]
|
||||
}`
|
||||
|
||||
// Write test files
|
||||
osvCveDir := filepath.Join(inputDir, "osv", "cve")
|
||||
require.NoError(t, os.MkdirAll(osvCveDir, 0o755))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(osvCveDir, "CVE-2024-1111.json"), []byte(testOSVData1), 0o644))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(osvCveDir, "CVE-2024-2222.json"), []byte(testOSVData2), 0o644))
|
||||
|
||||
// Create changed files lists
|
||||
changedTodayFile := filepath.Join(changedFilesDir, "changed_today.txt")
|
||||
changedYesterdayFile := filepath.Join(changedFilesDir, "changed_yesterday.txt")
|
||||
require.NoError(t, os.WriteFile(changedTodayFile, []byte("osv/cve/CVE-2024-1111.json\n"), 0o644))
|
||||
require.NoError(t, os.WriteFile(changedYesterdayFile, []byte("osv/cve/CVE-2024-2222.json\n"), 0o644))
|
||||
|
||||
// Create config with delta generation
|
||||
cfg := Config{
|
||||
InputDir: inputDir,
|
||||
OutputDir: outputDir,
|
||||
Versions: "",
|
||||
ExcludeVersions: "",
|
||||
ChangedFilesToday: changedTodayFile,
|
||||
ChangedFilesYesterday: changedYesterdayFile,
|
||||
DateStr: "2024-01-03",
|
||||
YesterdayStr: "2024-01-02",
|
||||
GeneratedTimestamp: "2024-01-03T00:00:00Z",
|
||||
RunTime: time.Date(2024, 1, 3, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
// Run
|
||||
err := run(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify full artifact
|
||||
fullArtifact, err := readArtifact(filepath.Join(outputDir, "osv-ubuntu-2204-2024-01-03.json.gz"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, fullArtifact.TotalCVEs)
|
||||
require.Equal(t, 2, fullArtifact.TotalPackages)
|
||||
|
||||
// Verify today's delta artifact
|
||||
todayDelta, err := readArtifact(filepath.Join(outputDir, "osv-ubuntu-2204-delta-2024-01-03.json.gz"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, todayDelta.TotalCVEs)
|
||||
require.Equal(t, 1, todayDelta.TotalPackages)
|
||||
require.NotNil(t, todayDelta.Vulnerabilities)
|
||||
require.Contains(t, todayDelta.Vulnerabilities, "changed-today-package")
|
||||
require.NotEmpty(t, todayDelta.Vulnerabilities["changed-today-package"])
|
||||
require.Equal(t, "CVE-2024-1111", todayDelta.Vulnerabilities["changed-today-package"][0].CVE)
|
||||
|
||||
// Verify yesterday's delta artifact
|
||||
yesterdayDelta, err := readArtifact(filepath.Join(outputDir, "osv-ubuntu-2204-delta-2024-01-02.json.gz"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, yesterdayDelta.TotalCVEs)
|
||||
require.Equal(t, 1, yesterdayDelta.TotalPackages)
|
||||
require.NotNil(t, yesterdayDelta.Vulnerabilities)
|
||||
require.Contains(t, yesterdayDelta.Vulnerabilities, "changed-yesterday-package")
|
||||
require.NotEmpty(t, yesterdayDelta.Vulnerabilities["changed-yesterday-package"])
|
||||
require.Equal(t, "CVE-2024-2222", yesterdayDelta.Vulnerabilities["changed-yesterday-package"][0].CVE)
|
||||
}
|
||||
|
||||
func TestRunWithVersionFiltering(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
versions string
|
||||
excludeVersions string
|
||||
expectedVersionCount int
|
||||
expectedVersions []string
|
||||
}{
|
||||
{
|
||||
name: "Inclusive filtering - single version",
|
||||
versions: "22.04",
|
||||
excludeVersions: "",
|
||||
expectedVersionCount: 1,
|
||||
expectedVersions: []string{"22.04"},
|
||||
},
|
||||
{
|
||||
name: "Inclusive filtering - multiple versions",
|
||||
versions: "20.04,22.04",
|
||||
excludeVersions: "",
|
||||
expectedVersionCount: 2,
|
||||
expectedVersions: []string{"20.04", "22.04"},
|
||||
},
|
||||
{
|
||||
name: "Exclusive filtering - exclude one version",
|
||||
versions: "",
|
||||
excludeVersions: "24.04",
|
||||
expectedVersionCount: 2,
|
||||
expectedVersions: []string{"20.04", "22.04"},
|
||||
},
|
||||
{
|
||||
name: "Auto-detect - no filtering",
|
||||
versions: "",
|
||||
excludeVersions: "",
|
||||
expectedVersionCount: 3,
|
||||
expectedVersions: []string{"20.04", "22.04", "24.04"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Create temporary directories
|
||||
inputDir := t.TempDir()
|
||||
outputDir := t.TempDir()
|
||||
|
||||
// Create test OSV files for different Ubuntu versions
|
||||
for _, ver := range []string{"20.04", "22.04", "24.04"} {
|
||||
data := fmt.Sprintf(`{
|
||||
"schema_version": "1.0",
|
||||
"id": "USN-1234-1",
|
||||
"published": "2024-01-01T00:00:00Z",
|
||||
"modified": "2024-01-02T00:00:00Z",
|
||||
"affected": [{
|
||||
"package": {
|
||||
"ecosystem": "Ubuntu:%s:LTS",
|
||||
"name": "test-package-%s"
|
||||
},
|
||||
"ranges": [{
|
||||
"type": "ECOSYSTEM",
|
||||
"events": [{"introduced": "0"}, {"fixed": "1.0"}]
|
||||
}]
|
||||
}],
|
||||
"upstream": ["CVE-2024-%s"]
|
||||
}`, ver, strings.ReplaceAll(ver, ".", ""), strings.ReplaceAll(ver, ".", ""))
|
||||
|
||||
filename := fmt.Sprintf("CVE-2024-%s.json", strings.ReplaceAll(ver, ".", ""))
|
||||
require.NoError(t, os.WriteFile(filepath.Join(inputDir, filename), []byte(data), 0o644))
|
||||
}
|
||||
|
||||
// Create config
|
||||
cfg := Config{
|
||||
InputDir: inputDir,
|
||||
OutputDir: outputDir,
|
||||
Versions: tt.versions,
|
||||
ExcludeVersions: tt.excludeVersions,
|
||||
ChangedFilesToday: "",
|
||||
ChangedFilesYesterday: "",
|
||||
DateStr: "2024-01-03",
|
||||
YesterdayStr: "2024-01-02",
|
||||
GeneratedTimestamp: "2024-01-03T00:00:00Z",
|
||||
RunTime: time.Date(2024, 1, 3, 0, 0, 0, 0, time.UTC),
|
||||
}
|
||||
|
||||
// Run
|
||||
err := run(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Count artifacts created
|
||||
files, err := filepath.Glob(filepath.Join(outputDir, "osv-ubuntu-*.json.gz"))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.expectedVersionCount, len(files))
|
||||
|
||||
// Verify expected versions were generated
|
||||
for _, expectedVer := range tt.expectedVersions {
|
||||
verStr := strings.ReplaceAll(expectedVer, ".", "")
|
||||
expectedFile := filepath.Join(outputDir, fmt.Sprintf("osv-ubuntu-%s-2024-01-03.json.gz", verStr))
|
||||
require.FileExists(t, expectedFile)
|
||||
|
||||
artifact, err := readArtifact(expectedFile)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expectedVer, artifact.UbuntuVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func readArtifact(path string) (*ArtifactData, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
gzReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
var artifact ArtifactData
|
||||
if err := json.NewDecoder(gzReader).Decode(&artifact); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &artifact, nil
|
||||
}
|
||||
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# Sync Canonical OSV repository using shallow clone with rolling window
|
||||
# Usage: ./sync-and-detect-changes.sh
|
||||
#
|
||||
# Outputs:
|
||||
# - Creates/updates ubuntu-security-notices directory (shallow clone)
|
||||
# - changed_files_today.txt and changed_files_yesterday.txt
|
||||
#
|
||||
# Exit codes:
|
||||
# 0: Success
|
||||
# 1: Error occurred
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
REPO_URL="https://github.com/canonical/ubuntu-security-notices.git"
|
||||
REPO_DIR="ubuntu-security-notices"
|
||||
DAYS_TO_KEEP=3 # how much git history to keep
|
||||
|
||||
echo "=== OSV Repository Sync ==="
|
||||
echo ""
|
||||
|
||||
if [ -d "$REPO_DIR/.git" ]; then
|
||||
echo "Repository exists, updating with rolling window..."
|
||||
cd "$REPO_DIR"
|
||||
|
||||
git config core.sparseCheckout true
|
||||
echo "osv/" > .git/info/sparse-checkout
|
||||
|
||||
OLD_SHA=$(git rev-parse HEAD)
|
||||
OLD_COUNT=$(git log --oneline | wc -l | xargs)
|
||||
|
||||
git fetch --update-shallow --shallow-since="${DAYS_TO_KEEP} days ago" origin main
|
||||
|
||||
NEW_SHA=$(git rev-parse origin/main)
|
||||
|
||||
echo ""
|
||||
if [ "$OLD_SHA" = "$NEW_SHA" ]; then
|
||||
echo "No new commits (already at $NEW_SHA)"
|
||||
else
|
||||
echo "Updating: $OLD_SHA -> $NEW_SHA"
|
||||
git reset --hard origin/main
|
||||
fi
|
||||
|
||||
NEW_COUNT=$(git log --oneline | wc -l | xargs)
|
||||
echo "History: $OLD_COUNT commits -> $NEW_COUNT commits"
|
||||
|
||||
cd ..
|
||||
else
|
||||
echo "Cloning repository (shallow since ${DAYS_TO_KEEP} days ago)..."
|
||||
|
||||
mkdir -p "$REPO_DIR"
|
||||
cd "$REPO_DIR"
|
||||
git init --initial-branch=main
|
||||
git remote add origin "$REPO_URL"
|
||||
|
||||
git config core.sparseCheckout true
|
||||
echo "osv/" > .git/info/sparse-checkout
|
||||
|
||||
git fetch --shallow-since="${DAYS_TO_KEEP} days ago" origin main
|
||||
git checkout -b main --track origin/main
|
||||
|
||||
COMMIT_SHA=$(git rev-parse HEAD)
|
||||
COMMIT_COUNT=$(git log --oneline | wc -l | xargs)
|
||||
cd ..
|
||||
|
||||
echo ""
|
||||
echo "Cloned at: $COMMIT_SHA"
|
||||
fi
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
# Get files changed today (since midnight UTC today)
|
||||
TODAY_UTC=$(date -u +%Y-%m-%d)
|
||||
git log --since="${TODAY_UTC}T00:00:00Z" --name-only --pretty="" -- osv/cve \
|
||||
| sed '/^$/d' | sort -u > "../changed_files_today.txt"
|
||||
|
||||
# Get files changed yesterday (from midnight yesterday to midnight today UTC)
|
||||
YESTERDAY_UTC=$(date -u -v-1d +%Y-%m-%d 2>/dev/null || date -u -d "yesterday" +%Y-%m-%d)
|
||||
git log --since="${YESTERDAY_UTC}T00:00:00Z" --until="${TODAY_UTC}T00:00:00Z" --name-only --pretty="" -- osv/cve \
|
||||
| sed '/^$/d' | sort -u > "../changed_files_yesterday.txt"
|
||||
|
||||
TODAY_COUNT=$(wc -l < "../changed_files_today.txt" | xargs)
|
||||
YESTERDAY_COUNT=$(wc -l < "../changed_files_yesterday.txt" | xargs)
|
||||
cd ..
|
||||
|
||||
echo "Today: $TODAY_COUNT CVE files changed"
|
||||
echo "Yesterday: $YESTERDAY_COUNT CVE files changed"
|
||||
|
||||
echo ""
|
||||
echo "Sync Complete"
|
||||
cd "$REPO_DIR"
|
||||
FINAL_SHA=$(git rev-parse HEAD)
|
||||
FINAL_COUNT=$(git log --oneline | wc -l | xargs)
|
||||
cd ..
|
||||
|
||||
du -sh "$REPO_DIR" | awk '{print "Size: " $1}'
|
||||
echo "REPO_SHA=$FINAL_SHA"
|
||||
echo "REPO_COMMITS=$FINAL_COUNT"
|
||||
echo "OSV_DIR=$REPO_DIR/osv/cve"
|
||||
echo "CHANGED_FILES_TODAY=changed_files_today.txt"
|
||||
echo "CHANGED_FILES_YESTERDAY=changed_files_yesterday.txt"
|
||||
echo "TODAY_COUNT=$TODAY_COUNT"
|
||||
echo "YESTERDAY_COUNT=$YESTERDAY_COUNT"
|
||||
|
||||
exit 0
|
||||
@@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
// transformVuln applies transformations and filters to OSV vulnerability data.
|
||||
func transformVuln(packageName, cveID string, vuln *ProcessedVuln) (packages []string, modifiedVuln *ProcessedVuln) {
|
||||
// To completely ignore a CVE definition return nil
|
||||
// if cveID == "CVE-YYYY-XXXXX" {
|
||||
// return nil, nil
|
||||
// }
|
||||
|
||||
// Default: include the original package
|
||||
packages = []string{packageName}
|
||||
|
||||
// Package expansion rules: Add related packages that should also get this CVE
|
||||
|
||||
// Emacs CVEs (CVE-2024-39331, CVE-2024-53920, CVE-2025-1244, etc.)
|
||||
// Emacs vulnerabilities are in the Emacs Lisp runtime/interpreter shared across all packages.
|
||||
if packageName == "emacs" {
|
||||
packages = append(packages, "emacs-common", "emacs-el")
|
||||
}
|
||||
|
||||
// CVE-specific modifications: modify vulnerability details for specific CVEs
|
||||
// if cveID == "CVE-YYYY-XXXXX" {
|
||||
// modified := *vuln // Copy the vulnerability
|
||||
// modified.Fixed = "corrected-version"
|
||||
// return packages, &modified
|
||||
// }
|
||||
|
||||
// If the vulnerability requires no modifications return original
|
||||
return packages, nil
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTransformVuln(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
packageName string
|
||||
cveID string
|
||||
inputVuln ProcessedVuln
|
||||
expectedPackages []string
|
||||
expectModified bool
|
||||
}{
|
||||
{
|
||||
name: "emacs maps to emacs, emacs-common, and emacs-el",
|
||||
packageName: "emacs",
|
||||
cveID: "CVE-2024-39331",
|
||||
inputVuln: ProcessedVuln{
|
||||
CVE: "CVE-2024-39331",
|
||||
Published: "2024-07-01T00:00:00Z",
|
||||
Modified: "2024-07-15T00:00:00Z",
|
||||
Fixed: "1:26.3+1-1ubuntu2.1",
|
||||
Introduced: "0",
|
||||
},
|
||||
expectedPackages: []string{"emacs", "emacs-common", "emacs-el"},
|
||||
expectModified: false,
|
||||
},
|
||||
{
|
||||
name: "curl returns only curl (no transform)",
|
||||
packageName: "curl",
|
||||
cveID: "CVE-2024-1234",
|
||||
inputVuln: ProcessedVuln{
|
||||
CVE: "CVE-2024-1234",
|
||||
Published: "2024-01-01T00:00:00Z",
|
||||
Modified: "2024-01-15T00:00:00Z",
|
||||
},
|
||||
expectedPackages: []string{"curl"},
|
||||
expectModified: false,
|
||||
},
|
||||
{
|
||||
name: "linux returns only linux (no transform)",
|
||||
packageName: "linux",
|
||||
cveID: "CVE-2024-5678",
|
||||
inputVuln: ProcessedVuln{
|
||||
CVE: "CVE-2024-5678",
|
||||
Published: "2024-03-01T00:00:00Z",
|
||||
Modified: "2024-03-15T00:00:00Z",
|
||||
},
|
||||
expectedPackages: []string{"linux"},
|
||||
expectModified: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
packages, modifiedVuln := transformVuln(tt.packageName, tt.cveID, &tt.inputVuln)
|
||||
require.ElementsMatch(t, tt.expectedPackages, packages)
|
||||
|
||||
if tt.expectModified {
|
||||
require.NotNil(t, modifiedVuln, "expected modified vulnerability")
|
||||
} else {
|
||||
require.Nil(t, modifiedVuln, "expected no modification")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user