diff --git a/CHANGELOG.md b/CHANGELOG.md index 3af08f1..771d312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Upcoming changes... +## [0.8.0] - 2026/01/07 +### Added +- Included Exploit Prediction Scoring System (EPSS) to vulnerability response +### Changed +- Refactored OSV use case +- Upgraded `scanoss/papi` to v0.28.0 + ## [0.7.0] - 2025/11/13 ### Changed - Optimized query performance for retrieving vulnerabilities by PURL version using CTE (Common Table Expression) approach in `pkg/models/vulns_purl.go:111` @@ -81,4 +88,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [0.5.0]: https://github.com/scanoss/vulnerabilities/compare/v0.4.0...v0.5.0 [0.6.0]: https://github.com/scanoss/vulnerabilities/compare/v0.5.0...v0.6.0 [0.6.1]: https://github.com/scanoss/vulnerabilities/compare/v0.6.0...v0.6.1 -[0.6.2]: https://github.com/scanoss/vulnerabilities/compare/v0.6.1...v0.6.2 \ No newline at end of file +[0.6.2]: https://github.com/scanoss/vulnerabilities/compare/v0.6.1...v0.6.2 +[0.7.0]: https://github.com/scanoss/vulnerabilities/compare/v0.6.2...v0.7.0 +[0.8.0]: https://github.com/scanoss/vulnerabilities/compare/v0.7.0...v0.8.0 \ No newline at end of file diff --git a/Makefile b/Makefile index e3a1d0b..1a3624b 100644 --- a/Makefile +++ b/Makefile @@ -79,12 +79,6 @@ build_arm: version ## Build an ARM 64 binary go generate ./pkg/cmd/server.go GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -ldflags="-w -s" -o ./target/scanoss-vulnerabilities-api-linux-arm64 ./cmd/server -build_arm: version ## Build an ARM 64 binary - @echo "Building ARM binary $(VERSION)..." - go generate ./pkg/cmd/server.go - GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -ldflags="-w -s" -o ./target/scanoss-vulnerabilities-api-darwin-arm64 ./cmd/server - - package: package_amd ## Build & Package an AMD 64 binary package_amd: version ## Build & Package an AMD 64 binary diff --git a/go.mod b/go.mod index 8d931cb..55bb621 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/pandatix/go-cvss v0.6.2 github.com/scanoss/go-grpc-helper v0.9.0 github.com/scanoss/go-models v0.2.0 - github.com/scanoss/papi v0.17.0 + github.com/scanoss/papi v0.28.0 github.com/scanoss/zap-logging-helper v0.4.0 go.uber.org/zap v1.27.0 google.golang.org/grpc v1.75.0 diff --git a/go.sum b/go.sum index fa6cf91..e19f966 100644 --- a/go.sum +++ b/go.sum @@ -659,8 +659,8 @@ github.com/scanoss/go-purl-helper v0.2.1 h1:jp960a585ycyJSlqZky1NatMJBIQi/JGITDf github.com/scanoss/go-purl-helper v0.2.1/go.mod h1:v20/bKD8G+vGrILdiq6r0hyRD2bO8frCJlu9drEcQ38= github.com/scanoss/ipfilter/v2 v2.0.2 h1:GaB9i8kVJg9JQZm5XGStYkEpiaCVdsrj7ezI2wV/oh8= github.com/scanoss/ipfilter/v2 v2.0.2/go.mod h1:AwrpX4XGbZ7EKISMi1d6E5csBk1nWB8+ugpvXHFcTpA= -github.com/scanoss/papi v0.17.0 h1:YKS6hN1hXbE0yV9LwAridOreNLTPfrwFozUfbrAYv+Y= -github.com/scanoss/papi v0.17.0/go.mod h1:Z4E/4IpwYdzHHRJXTgBCGG1GjksgrFjNW5cvhbKUfeU= +github.com/scanoss/papi v0.28.0 h1:uvevFYoxwzvSH1hvgBoAkScIGTK2U1+rLzHSoJdnARk= +github.com/scanoss/papi v0.28.0/go.mod h1:Z4E/4IpwYdzHHRJXTgBCGG1GjksgrFjNW5cvhbKUfeU= github.com/scanoss/zap-logging-helper v0.4.0 h1:2qTYoaFa9+MlD2/1wmPtiDHfh+42NIEwgKVU3rPpl0Y= github.com/scanoss/zap-logging-helper v0.4.0/go.mod h1:9QuEZcq73g/0Izv1tWeOWukoIK0oTBzM4jSNQ5kRR1w= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= diff --git a/pkg/adapters/vulnerability_support.go b/pkg/adapters/vulnerability_support.go index a3d8139..8c67ede 100644 --- a/pkg/adapters/vulnerability_support.go +++ b/pkg/adapters/vulnerability_support.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + purlhelper "github.com/scanoss/go-purl-helper/pkg" common "github.com/scanoss/papi/api/commonv2" pb "github.com/scanoss/papi/api/vulnerabilitiesv2" zlog "github.com/scanoss/zap-logging-helper/pkg/logger" @@ -33,7 +34,7 @@ func sanitizeComponents(components []dtos.ComponentDTO) ([]dtos.ComponentDTO, [] var sanitized []dtos.ComponentDTO var invalid []dtos.ComponentDTO for _, component := range components { - _, err := utils.PurlFromString(component.Purl) + _, err := purlhelper.PurlFromString(component.Purl) if err != nil { invalid = append(invalid, component) continue diff --git a/pkg/config/server_config.go b/pkg/config/server_config.go index 5839319..15bbcf6 100644 --- a/pkg/config/server_config.go +++ b/pkg/config/server_config.go @@ -76,6 +76,7 @@ type ServerConfig struct { APIBaseURL string `env:"VULN_OSV_API_BASE_URL"` InfoBaseURL string `env:"VULN_OSV_INFO_BASE_URL"` Enabled bool `env:"VULN_OSV_SOURCE_ENABLED"` + APIWorkers int `env:"VULN_OSV_API_WORKERS"` } SCANOSS struct { Enabled bool `env:"VULN_SCANOSS_SOURCE_ENABLED"` @@ -121,6 +122,7 @@ func setServerConfigDefaults(cfg *ServerConfig) { cfg.Source.OSV.APIBaseURL = "https://api.osv.dev/v1" cfg.Source.OSV.InfoBaseURL = "https://osv.dev/vulnerability" cfg.Source.OSV.Enabled = true + cfg.Source.OSV.APIWorkers = 5 cfg.Source.SCANOSS.Enabled = true } diff --git a/pkg/dtos/vulnerability_output.go b/pkg/dtos/vulnerability_output.go index 904212d..1b63658 100644 --- a/pkg/dtos/vulnerability_output.go +++ b/pkg/dtos/vulnerability_output.go @@ -51,6 +51,11 @@ type CVSS struct { CvssSeverity string `json:"cvss_severity"` } +type EPSS struct { + Percentile float32 `json:"percentile"` + Probability float32 `json:"probability"` +} + type VulnerabilitiesOutput struct { ID string `json:"id"` Cve string `json:"cve"` @@ -61,6 +66,7 @@ type VulnerabilitiesOutput struct { Modified u.OnlyDate `json:"modified"` Source string `json:"source"` Cvss []CVSS `json:"cvss"` + Epss EPSS `json:"epss"` } // ParseVulnerabilityOutput converts the input byte array to a VulnerabilityOutput structure. diff --git a/pkg/models/all_urls.go b/pkg/models/all_urls.go deleted file mode 100644 index 95d0aa6..0000000 --- a/pkg/models/all_urls.go +++ /dev/null @@ -1,252 +0,0 @@ -// Package models SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2018-2025 SCANOSS.COM - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package models - -import ( - "context" - "errors" - "fmt" - "sort" - "strings" - - "github.com/Masterminds/semver/v3" - "github.com/jmoiron/sqlx" - "go.uber.org/zap" - - "scanoss.com/vulnerabilities/pkg/utils" -) - -type AllUrlsModel struct { - ctx context.Context - s *zap.SugaredLogger - conn *sqlx.Conn - project *ProjectModel - golangProj *GolangProjects -} - -type AllURL struct { - Component string `db:"component"` - Version string `db:"version"` - SemVer string `db:"semver"` - License string `db:"license"` - LicenseID string `db:"license_id"` - IsSpdx bool `db:"is_spdx"` - PurlName string `db:"purl_name"` - MineID int32 `db:"mine_id"` - URL string `db:"-"` -} - -// SQL Query constants. -const ( - purlSQLQuerySelect = "SELECT component, v.version_name AS version, v.semver AS semver," - licSpdxSQLQuery = " l.license_name AS license, l.spdx_id AS license_id, l.is_spdx AS is_spdx," - verLeftJoinSQL = " LEFT JOIN versions v ON u.version_id = v.id" - licLeftJoinSQL = " LEFT JOIN licenses l ON u.license_id = l.id" - mineLeftJoinSQL = " LEFT JOIN mines m ON u.mine_id = m.id" -) - -// NewAllURLModel creates a new instance of the 'All URL' Model. -func NewAllURLModel(ctx context.Context, s *zap.SugaredLogger, conn *sqlx.Conn, project *ProjectModel, golangProj *GolangProjects) *AllUrlsModel { - return &AllUrlsModel{ctx: ctx, s: s, conn: conn, project: project, golangProj: golangProj} -} - -// GetURLsByPurlString searches for component details of the specified Purl string (and optional requirement). -func (m *AllUrlsModel) GetURLsByPurlString(purlString, purlReq string) (AllURL, error) { - if len(purlString) == 0 { - m.s.Error("Please specify a valid Purl String to query") - return AllURL{}, errors.New("please specify a valid Purl String to query") - } - purl, err := utils.PurlFromString(purlString) - if err != nil { - return AllURL{}, err - } - purlName, err := utils.PurlNameFromString(purlString) // Make sure we just have the bare minimum for a Purl Name - if err != nil { - return AllURL{}, err - } - // TODO check what to do if we get a "file" requirement - if len(purlReq) > 0 && strings.HasPrefix(purlReq, "file:") { // internal dependency requirement. Assume latest - m.s.Debugf("Removing 'local' requirement for purl: %v (req: %v)", purlString, purlReq) - purlReq = "" - } - if len(purl.Version) == 0 && len(purlReq) > 0 { // No version specified, but we might have a specific version in the Requirement - ver := utils.GetVersionFromReq(purlReq) - if len(ver) > 0 { - purl.Version = ver // Switch to exact version search (faster) - purlReq = "" - } - } - if purl.Type == "golang" { - allURL, err := m.golangProj.GetGoLangURLByPurl(purl, purlName, purlReq) // Search a separate table for golang dependencies - // If no golang package/license is found, but it's a GitHub component, search GitHub for it - if err == nil && (len(allURL.Component) == 0 || len(allURL.License) == 0) && strings.HasPrefix(purlString, "pkg:golang/github.com/") { - if len(allURL.Component) == 0 { - m.s.Debugf("Didn't find component in golang projects table for %v. Checking all urls...", purlString) - } else if len(allURL.License) == 0 { - m.s.Debugf("Didn't find license in golang projects table for %v. Checking all urls...", purlString) - } - purlString = utils.ConvertGoPurlStringToGithub(purlString) // Convert to GitHub purl - purl, err = utils.PurlFromString(purlString) - if err != nil { - return AllURL{}, err - } - purlName, err = utils.PurlNameFromString(purlString) // Make sure we just have the bare minimum for a Purl Name - if err != nil { - return AllURL{}, err - } - m.s.Debugf("Now searching All Urls for Purl: %#v, PurlName: %v", purl, purlName) - } else { - return allURL, err - } - } - if len(purl.Version) > 0 { - return m.GetURLsByPurlNameTypeVersion(purlName, purl.Type, purl.Version) - } - return m.GetURLsByPurlNameType(purlName, purl.Type, purlReq) -} - -// GetURLsByPurlNameType searches for component details of the specified Purl Name/Type (and optional requirement). -func (m *AllUrlsModel) GetURLsByPurlNameType(purlName, purlType, purlReq string) (AllURL, error) { - if len(purlName) == 0 { - m.s.Error("Please specify a valid Purl Name to query") - return AllURL{}, errors.New("please specify a valid Purl Name to query") - } - if len(purlType) == 0 { - m.s.Errorf("Please specify a valid Purl Type to query: %v", purlName) - return AllURL{}, errors.New("please specify a valid Purl Type to query") - } - query := purlSQLQuerySelect + licSpdxSQLQuery + " purl_name, mine_id FROM all_urls u" + - mineLeftJoinSQL + licLeftJoinSQL + verLeftJoinSQL + - " WHERE m.purl_type = $1 AND u.purl_name = $2 ORDER BY date DESC" - var allUrls []AllURL - err := m.conn.SelectContext(m.ctx, &allUrls, query, purlType, purlName) - if err != nil { - m.s.Errorf("Failed to query all urls table for %v - %v: %v", purlType, purlName, err) - return AllURL{}, fmt.Errorf("failed to query the all urls table: %v", err) - } - m.s.Debugf("Found %v results for %v, %v.", len(allUrls), purlType, purlName) - // Pick one URL to return (checking for license details also) - return pickOneURL(m.s, m.project, allUrls, purlName, purlType, purlReq) -} - -// GetURLsByPurlNameTypeVersion searches for component details of the specified Purl Name/Type and version. -func (m *AllUrlsModel) GetURLsByPurlNameTypeVersion(purlName, purlType, purlVersion string) (AllURL, error) { - if len(purlName) == 0 { - m.s.Error("Please specify a valid Purl Name to query") - return AllURL{}, errors.New("please specify a valid Purl Name to query") - } - if len(purlType) == 0 { - m.s.Error("Please specify a valid Purl Type to query") - return AllURL{}, errors.New("please specify a valid Purl Type to query") - } - if len(purlVersion) == 0 { - m.s.Error("Please specify a valid Purl Version to query") - return AllURL{}, errors.New("please specify a valid Purl Version to query") - } - query := purlSQLQuerySelect + licSpdxSQLQuery + " purl_name, mine_id FROM all_urls u" + - mineLeftJoinSQL + licLeftJoinSQL + verLeftJoinSQL + - " WHERE m.purl_type = $1 AND u.purl_name = $2 AND v.version_name = $3 ORDER BY date DESC" - var allUrls []AllURL - err := m.conn.SelectContext(m.ctx, &allUrls, query, purlType, purlName, purlVersion) - if err != nil { - m.s.Errorf("Failed to query all urls table for %v - %v: %v", purlType, purlName, err) - return AllURL{}, fmt.Errorf("failed to query the all urls table: %v", err) - } - m.s.Debugf("Found %v results for %v, %v.", len(allUrls), purlType, purlName) - // Pick one URL to return (checking for license details also) - return pickOneURL(m.s, m.project, allUrls, purlName, purlType, "") -} - -func pickOneURL(s *zap.SugaredLogger, projModel *ProjectModel, allUrls []AllURL, purlName, purlType, purlReq string) (AllURL, error) { - if len(allUrls) == 0 { - s.Infof("No component match (in urls) found for %v, %v", purlName, purlType) - return AllURL{}, nil - } - var c *semver.Constraints - var urlMap = make(map[*semver.Version]AllURL) - if len(purlReq) > 0 { - s.Debugf("Building version constraint for %v: %v", purlName, purlReq) - var err error - c, err = semver.NewConstraint(purlReq) - if err != nil { - s.Warnf("Encountered an issue parsing version constraint string '%v' (%v,%v): %v", purlReq, purlName, purlType, err) - } - } - s.Debugf("Checking versions...") - for _, url := range allUrls { - if len(url.SemVer) > 0 || len(url.Version) > 0 { - v, err := semver.NewVersion(url.Version) - if err != nil && len(url.SemVer) > 0 { - s.Debugf("Failed to parse SemVer: '%v'. Trying Version instead: %v (%v)", url.Version, url.SemVer, err) - v, err = semver.NewVersion(url.SemVer) // Semver failed, try the normal version - } - if err != nil { - s.Warnf("Encountered an issue parsing version string '%v' (%v) for %v: %v. Using v0.0.0", url.Version, url.SemVer, url, err) - v, err = semver.NewVersion("v0.0.0") // Semver failed, just use a standard version zero (for now) - } - if err == nil { - if c == nil || c.Check(v) { - _, ok := urlMap[v] - if !ok { - urlMap[v] = url // fits inside the constraint and hasn't already been stored - } - } - } - } else { - s.Infof("Skipping match as it doesn't have a version: %#v", url) - } - } - if len(urlMap) == 0 { // TODO should we return the latest version anyway? - s.Warnf("No component match found for %v, %v after filter %v", purlName, purlType, purlReq) - return AllURL{}, nil - } - var versions = make([]*semver.Version, len(urlMap)) - var vi = 0 - for version := range urlMap { // Save the list of versions so they can be sorted - versions[vi] = version - vi++ - } - sort.Sort(semver.Collection(versions)) - version := versions[len(versions)-1] // Get the latest (acceptable) URL version - s.Debugf("Sorted versions: %v. Highest: %v", versions, version) - - url, ok := urlMap[version] // Retrieve the latest accepted URL version - if !ok { - s.Errorf("Problem retrieving URL data for %v (%v, %v)", version, purlName, purlType) - return AllURL{}, fmt.Errorf("failed to retrieve specific URL version: %v", version) - } - url.URL, _ = utils.ProjectURL(purlName, purlType) - - s.Debugf("Selected version: %#v", url) - if len(url.License) == 0 && projModel != nil { // Check for a project license if we don't have a component one - project, err := projModel.GetProjectByPurlName(purlName, url.MineID) - switch { - case err != nil: - s.Warnf("Problem searching projects table for %v, %v", purlName, purlType) - case len(project.License) > 0: - s.Debugf("Adding project license data to %v from %v", url, project) - url.License = project.License - url.IsSpdx = project.IsSpdx - url.LicenseID = project.LicenseID - case len(project.GitLicense) > 0: - s.Debugf("Adding project git license data to %v from %v", url, project) - url.License = project.GitLicense - url.IsSpdx = project.GitIsSpdx - url.LicenseID = project.GitLicenseID - } - } - return url, nil // Return the best component match -} diff --git a/pkg/models/all_urls_test.go b/pkg/models/all_urls_test.go deleted file mode 100644 index 613241b..0000000 --- a/pkg/models/all_urls_test.go +++ /dev/null @@ -1,422 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2018-2025 SCANOSS.COM - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package models - -import ( - "context" - "fmt" - "testing" - - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" - - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "github.com/jmoiron/sqlx" - "scanoss.com/vulnerabilities/pkg/config" -) - -func TestAllUrlsSearch(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = loadTestSQLDataFiles(db, ctx, conn, []string{ - "../models/tests/all_urls.sql", - "../models/tests/mines.sql", - "../models/tests/projects.sql", - "../models/tests/golang_projects.sql", - "../models/tests/licenses.sql", - "../models/tests/versions.sql", - }) - if err != nil { - t.Fatalf("failed to load SQL test data: %+v", err) - } - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load server config: %+v", err) - } - myConfig.Components.CommitMissing = true - myConfig.Database.Trace = true - allUrlsModel := NewAllURLModel(ctx, zlog.S, conn, NewProjectModel(ctx, zlog.S, conn), - NewGolangProjectModel(ctx, zlog.S, conn, myConfig)) - - allUrls, err := allUrlsModel.GetURLsByPurlNameType("tablestyle", "gem", "") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetUrlsByPurlName() No URLs returned from query") - } - fmt.Printf("All Urls: %#v\n", allUrls) - - allUrls, err = allUrlsModel.GetURLsByPurlNameType("NONEXISTENT", "none", "") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) > 0 { - t.Errorf("all_urls.GetURLsByPurlNameType() URLs found when none should be: %v", allUrlsModel) - } - fmt.Printf("No Urls: %v\n", allUrls) - - _, err = allUrlsModel.GetURLsByPurlNameType("NONEXISTENT", "", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = allUrlsModel.GetURLsByPurlNameType("", "", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = allUrlsModel.GetURLsByPurlString("", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = allUrlsModel.GetURLsByPurlString("rubbish-purl", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - allUrls, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", "") - if err != nil { - t.Errorf("all_urls.GetURLsByPurlString() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlString() No URLs returned from query") - } - fmt.Printf("All Urls: %v\n", allUrls) - - allUrls, err = allUrlsModel.GetURLsByPurlString("pkg:golang/google.golang.org/grpc", "") - if err != nil { - t.Errorf("all_urls.GetURLsByPurlString() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlString() No URLs returned from query") - } - fmt.Printf("Golang URL: %v\n", allUrls) - - fmt.Printf("Searching for pkg:golang/github.com/scanoss/dependencies") - allUrls, err = allUrlsModel.GetURLsByPurlString("pkg:golang/github.com/scanoss/dependencies", "") - if err != nil { - t.Errorf("all_urls.GetURLsByPurlString() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlString() No URLs returned from query") - } - fmt.Printf("Golang URL: %v\n", allUrls) -} - -func TestAllUrlsSearchVersion(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = loadTestSQLDataFiles(db, ctx, conn, []string{ - "../models/tests/all_urls.sql", - "../models/tests/mines.sql", - "../models/tests/projects.sql", - "../models/tests/golang_projects.sql", - "../models/tests/licenses.sql", - "../models/tests/versions.sql", - }) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - s := ctxzap.Extract(ctx).Sugar() - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("an error '%s' was not expected when loading config", err) - } - myConfig.Components.CommitMissing = true - myConfig.Database.Trace = true - allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), - NewGolangProjectModel(ctx, s, conn, myConfig)) - - allUrls, err := allUrlsModel.GetURLsByPurlNameTypeVersion("tablestyle", "gem", "0.0.12") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlNameTypeVersion() No URLs returned from query") - } - fmt.Printf("All Urls Version: %#v\n", allUrls) - - allUrls, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle@0.0.7", "") - if err != nil { - t.Errorf("all_urls.GetURLsByPurlString() error = failed to find purl by version string") - } - fmt.Printf("All Urls Version String: %#v\n", allUrls) - - _, err = allUrlsModel.GetURLsByPurlNameTypeVersion("", "", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlNameTypeVersion() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = allUrlsModel.GetURLsByPurlNameTypeVersion("NONEXISTENT", "", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlNameTypeVersion() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = allUrlsModel.GetURLsByPurlNameTypeVersion("NONEXISTENT", "NONEXISTENT", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlNameTypeVersion() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - - allUrls, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", "22.22.22") // Shouldn't exist - if err != nil { - t.Errorf("all_urls.GetURLsByPurlString() error = failed to find purl by version string") - } else if len(allUrls.PurlName) > 0 { - t.Errorf("all_urls.GetURLsByPurlString() error = Found match, when we shouldn't: %v", allUrls) - } -} - -func TestAllUrlsSearchVersionRequirement(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = loadTestSQLDataFiles(db, ctx, conn, []string{ - "../models/tests/all_urls.sql", - "../models/tests/mines.sql", - "../models/tests/projects.sql", - "../models/tests/golang_projects.sql", - "../models/tests/licenses.sql", - "../models/tests/versions.sql", - }) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - s := ctxzap.Extract(ctx).Sugar() - myConfig, err := config.NewServerConfig(nil) - - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - myConfig.Database.Trace = true - allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), - NewGolangProjectModel(ctx, s, conn, myConfig)) - - allUrls, err := allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", ">0.0.4") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlString() No URLs returned from query") - } - fmt.Printf("All Urls Version: %#v\n", allUrls) - - allUrls, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", "<0.0.4>") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlString() No URLs returned from query") - } -} - -func TestAllUrlsSearchNoProject(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = loadTestSQLDataFiles(db, ctx, conn, []string{ - "../models/tests/all_urls.sql", - "../models/tests/mines.sql", - "../models/tests/projects.sql", - "../models/tests/golang_projects.sql", - "../models/tests/licenses.sql", - "../models/tests/versions.sql", - }) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - s := ctxzap.Extract(ctx).Sugar() - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - myConfig.App.Trace = true - allUrlsModel := NewAllURLModel(ctx, s, conn, nil, NewGolangProjectModel(ctx, s, conn, myConfig)) - - allUrls, err := allUrlsModel.GetURLsByPurlNameType("tablestyle", "gem", "0.0.8") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlNameType() No URLs returned from query") - } - fmt.Printf("All Urls: %#v\n", allUrls) -} - -func TestAllUrlsSearchNoLicense(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = loadTestSQLDataFiles(db, ctx, conn, []string{ - "../models/tests/all_urls.sql", - "../models/tests/mines.sql", - "../models/tests/projects.sql", - "../models/tests/golang_projects.sql", - "../models/tests/licenses.sql", - "../models/tests/versions.sql", - }) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - s := ctxzap.Extract(ctx).Sugar() - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - myConfig.App.Trace = true - allUrlsModel := NewAllURLModel(ctx, s, conn, NewProjectModel(ctx, s, conn), - NewGolangProjectModel(ctx, s, conn, myConfig)) - - allUrls, err := allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle@0.0.8", "") - if err != nil { - t.Errorf("all_urls.GetURLsByPurlString() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlString() No URLs returned from query") - } - fmt.Printf("All (with project) Urls: %#v\n", allUrls) -} - -func TestAllUrlsSearchBadSql(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - myConfig.App.Trace = true - allUrlsModel := NewAllURLModel(ctx, zlog.S, conn, NewProjectModel(ctx, zlog.S, conn), - NewGolangProjectModel(ctx, zlog.S, conn, myConfig)) - _, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle@0.0.8", "") - if err == nil { - t.Errorf("all_urls.GetURLsByPurlString() error = did not get an error: %v", err) - } else { - fmt.Printf("Got expected error = %v\n", err) - } - // Load some tables (leaving out projects) - err = loadTestSQLDataFiles(db, ctx, conn, []string{"./tests/mines.sql", "./tests/all_urls.sql", "./tests/licenses.sql", "./tests/versions.sql"}) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - // allUrls, err := allUrlsModel.GetURLsByPurlNameType("tablestyle", "gem", "") - allUrls, err := allUrlsModel.GetURLsByPurlString("pkg:gem/tablestyle@0.0.8", "") - if err != nil { - t.Errorf("all_urls.GetUrlsByPurlName() error = %v", err) - } - if len(allUrls.PurlName) == 0 { - t.Errorf("all_urls.GetURLsByPurlNameType() No URLs returned from query") - } - fmt.Printf("All Urls: %v\n", allUrls) -} diff --git a/pkg/models/cpe_purl.go b/pkg/models/cpe_purl.go index 46495b1..f4018f4 100644 --- a/pkg/models/cpe_purl.go +++ b/pkg/models/cpe_purl.go @@ -23,6 +23,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/jmoiron/sqlx" + purlhelper "github.com/scanoss/go-purl-helper/pkg" zlog "github.com/scanoss/zap-logging-helper/pkg/logger" "scanoss.com/vulnerabilities/pkg/utils" ) @@ -49,7 +50,7 @@ func (m *CpePurlModel) GetCpeByPurl(purlString, purlReq string) ([]CpePurl, erro zlog.S.Errorf("Please specify a valid Purl String to query") return []CpePurl{}, errors.New("please specify a valid Purl String to query") } - purl, err := utils.PurlFromString(purlString) + purl, err := purlhelper.PurlFromString(purlString) if err != nil { return []CpePurl{}, err } @@ -58,7 +59,7 @@ func (m *CpePurlModel) GetCpeByPurl(purlString, purlReq string) ([]CpePurl, erro purlString = utils.PurlRemoveFromVersionComponent(purlString) // Make sure to get the minimum purl pkg:github... if len(purlVersion) == 0 && len(purlReq) > 0 { // No version specified, but we might have a specific version in the Requirement - ver := utils.GetVersionFromReq(purlReq) + ver := purlhelper.GetVersionFromReq(purlReq) if len(ver) > 0 { purlVersion = ver // Switch to exact version search (faster) purlReq = "" diff --git a/pkg/models/epss.go b/pkg/models/epss.go new file mode 100644 index 0000000..cacc47c --- /dev/null +++ b/pkg/models/epss.go @@ -0,0 +1,55 @@ +package models + +import ( + "context" + + "go.uber.org/zap" + + "scanoss.com/vulnerabilities/pkg/config" + + "github.com/jmoiron/sqlx" + + scdb "github.com/scanoss/go-grpc-helper/pkg/grpc/database" +) + +type EPSSModel struct { + db *sqlx.DB + config *config.ServerConfig + sc *scdb.DBQueryContext + s *zap.SugaredLogger +} + +type EPSS struct { + Cve string `db:"cve"` + Epss float32 `db:"epss"` + Percentile float32 `db:"percentile"` +} + +// NewEPSSModel creates a new instance of the EPSS Model. +func NewEPSSModel(s *zap.SugaredLogger, config *config.ServerConfig, db *sqlx.DB) *EPSSModel { + return &EPSSModel{ + db: db, + config: config, + sc: scdb.NewDBSelectContext(s, db, nil, config.App.Trace), + s: s, + } +} + +// GetEPSSByCVEs List of EPSS by CVEs. +func (m *EPSSModel) GetEPSSByCVEs(ctx context.Context, cves []string) ([]EPSS, error) { + if len(cves) == 0 { + return []EPSS{}, nil + } + var epss []EPSS + query, args, err := sqlx.In("SELECT cve, epss, percentile FROM epss_data WHERE cve IN (?)", cves) + if err != nil { + return nil, err + } + query = m.db.Rebind(query) + err = m.sc.SelectContext(ctx, &epss, query, args...) + if err != nil { + m.s.Errorf("Failed to get EPSS data for CVEs: %v", err) + return nil, err + } + return epss, nil +} diff --git a/pkg/models/epss_test.go b/pkg/models/epss_test.go new file mode 100644 index 0000000..84bc010 --- /dev/null +++ b/pkg/models/epss_test.go @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2018-2025 SCANOSS.COM + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package models + +import ( + "context" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + + "testing" + + myconfig "scanoss.com/vulnerabilities/pkg/config" + + zlog "github.com/scanoss/zap-logging-helper/pkg/logger" + + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" +) + +func TestGetEPSSByCVEs(t *testing.T) { + ctx := context.Background() + err := zlog.NewSugaredDevLogger() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) + } + defer zlog.SyncZap() + + s := ctxzap.Extract(ctx).Sugar() + + db, err := sqlx.Connect("sqlite3", ":memory:") + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + defer CloseDB(db) + + err = loadSQLData(db, nil, nil, "./tests/epss.sql") + if err != nil { + t.Fatalf("failed to load SQL test data: %v", err) + } + + serverConfig, err := myconfig.NewServerConfig(nil) + if err != nil { + t.Fatalf("failed to load Config: %v", err) + } + + epssModel := NewEPSSModel(s, serverConfig, db) + + // Test with multiple CVEs + cves := []string{"CVE-2017-9302", "CVE-2015-0269", "CVE-2018-10083"} + results, err := epssModel.GetEPSSByCVEs(ctx, cves) + if err != nil { + t.Errorf("GetEPSSByCVEs() error = %v", err) + } + if len(results) != 3 { + t.Errorf("GetEPSSByCVEs() expected 3 results, got %d", len(results)) + } +} + +func TestGetEPSSByCVEsEmpty(t *testing.T) { + ctx := context.Background() + err := zlog.NewSugaredDevLogger() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) + } + defer zlog.SyncZap() + s := ctxzap.Extract(ctx).Sugar() + + db, err := sqlx.Connect("sqlite3", ":memory:") + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + defer CloseDB(db) + + serverConfig, err := myconfig.NewServerConfig(nil) + if err != nil { + t.Fatalf("failed to load Config: %v", err) + } + + err = loadSQLData(db, nil, nil, "./tests/epss.sql") + if err != nil { + t.Fatalf("failed to load SQL test data: %v", err) + } + + epssModel := NewEPSSModel(s, serverConfig, db) + + // Test with empty slice + results, err := epssModel.GetEPSSByCVEs(ctx, []string{}) + if err != nil { + t.Errorf("GetEPSSByCVEs() with empty slice error = %v", err) + } + if len(results) != 0 { + t.Errorf("GetEPSSByCVEs() with empty slice expected 0 results, got %d", len(results)) + } +} + +func TestGetEPSSByCVEsNotFound(t *testing.T) { + ctx := context.Background() + err := zlog.NewSugaredDevLogger() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) + } + defer zlog.SyncZap() + s := ctxzap.Extract(ctx).Sugar() + + db, err := sqlx.Connect("sqlite3", ":memory:") + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + defer CloseDB(db) + + serverConfig, err := myconfig.NewServerConfig(nil) + if err != nil { + t.Fatalf("failed to load Config: %v", err) + } + + err = loadSQLData(db, nil, nil, "./tests/epss.sql") + if err != nil { + t.Fatalf("failed to load SQL test data: %v", err) + } + + epssModel := NewEPSSModel(s, serverConfig, db) + + // Test with non-existent CVEs + cves := []string{"CVE-NONEXISTENT-1", "CVE-NONEXISTENT-2"} + results, err := epssModel.GetEPSSByCVEs(ctx, cves) + if err != nil { + t.Errorf("GetEPSSByCVEs() error = %v", err) + } + if len(results) != 0 { + t.Errorf("GetEPSSByCVEs() with non-existent CVEs expected 0 results, got %d", len(results)) + } +} + +func TestGetEPSSByCVEsSingleCVE(t *testing.T) { + ctx := context.Background() + err := zlog.NewSugaredDevLogger() + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) + } + defer zlog.SyncZap() + s := ctxzap.Extract(ctx).Sugar() + + db, err := sqlx.Connect("sqlite3", ":memory:") + if err != nil { + t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) + } + defer CloseDB(db) + + serverConfig, err := myconfig.NewServerConfig(nil) + if err != nil { + t.Fatalf("failed to load Config: %v", err) + } + + err = loadSQLData(db, nil, nil, "./tests/epss.sql") + if err != nil { + t.Fatalf("failed to load SQL test data: %v", err) + } + + epssModel := NewEPSSModel(s, serverConfig, db) + + // Test with single CVE + cves := []string{"CVE-2018-10083"} + results, err := epssModel.GetEPSSByCVEs(ctx, cves) + if err != nil { + t.Errorf("GetEPSSByCVEs() error = %v", err) + } + if len(results) != 1 { + t.Errorf("GetEPSSByCVEs() expected 1 result, got %d", len(results)) + } + if len(results) > 0 && results[0].Cve != "CVE-2018-10083" { + t.Errorf("GetEPSSByCVEs() expected CVE-2018-10083, got %s", results[0].Cve) + } +} diff --git a/pkg/models/golang_projects.go b/pkg/models/golang_projects.go deleted file mode 100644 index 4cce396..0000000 --- a/pkg/models/golang_projects.go +++ /dev/null @@ -1,341 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2018-2025 SCANOSS.COM - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package models - -import ( - "context" - "database/sql" - "errors" - "fmt" - - pkggodevclient "github.com/guseggert/pkggodev-client" - "github.com/jmoiron/sqlx" - "github.com/package-url/packageurl-go" - "go.uber.org/zap" - - myconfig "scanoss.com/vulnerabilities/pkg/config" - "scanoss.com/vulnerabilities/pkg/utils" -) - -type GolangProjects struct { - ctx context.Context - s *zap.SugaredLogger - conn *sqlx.Conn - config *myconfig.ServerConfig - ver *VersionModel - lic *LicenseModel - mine *MineModel - project *ProjectModel // TODO Do we add golang component to the projects table? -} - -// NewGolangProjectModel creates a new instance of Golang Project Model. -func NewGolangProjectModel(ctx context.Context, s *zap.SugaredLogger, conn *sqlx.Conn, config *myconfig.ServerConfig) *GolangProjects { - return &GolangProjects{ctx: ctx, s: s, conn: conn, - ver: NewVersionModel(ctx, conn), - lic: NewLicenseModel(ctx, s, conn), - mine: NewMineModel(ctx, s, conn), - project: NewProjectModel(ctx, s, conn), - config: config, - } -} - -// GetGoLangURLByPurlString searches the Golang Projects for the specified Purl (and requirement). -func (m *GolangProjects) GetGoLangURLByPurlString(purlString, purlReq string) (AllURL, error) { - if len(purlString) == 0 { - m.s.Error("Please specify a valid Purl String to query") - return AllURL{}, errors.New("please specify a valid Purl String to query") - } - purl, err := utils.PurlFromString(purlString) - if err != nil { - return AllURL{}, err - } - purlName, err := utils.PurlNameFromString(purlString) - if err != nil { - return AllURL{}, err - } - if len(purl.Version) == 0 && len(purlReq) > 0 { // No version specified, but we might have a specific version in the Requirement - ver := utils.GetVersionFromReq(purlReq) - if len(ver) > 0 { - purl.Version = ver - purlReq = "" - } - } - return m.GetGoLangURLByPurl(purl, purlName, purlReq) -} - -// GetGoLangURLByPurl searches the Golang Projects for the specified Purl Package (and optional requirement). -func (m *GolangProjects) GetGoLangURLByPurl(purl packageurl.PackageURL, purlName, purlReq string) (AllURL, error) { - if len(purl.Version) > 0 { - return m.GetGolangUrlsByPurlNameTypeVersion(purlName, purl.Type, purl.Version) - } - return m.GetGolangUrlsByPurlNameType(purlName, purl.Type, purlReq) -} - -// GetGolangUrlsByPurlNameType searches Golang Project for the specified Purl by Purl Type (and optional requirement). -func (m *GolangProjects) GetGolangUrlsByPurlNameType(purlName, purlType, purlReq string) (AllURL, error) { - if len(purlName) == 0 { - m.s.Error("Please specify a valid Purl Name to query") - return AllURL{}, errors.New("please specify a valid Purl Name to query") - } - if len(purlType) == 0 { - m.s.Errorf("Please specify a valid Purl Type to query: %v", purlName) - return AllURL{}, errors.New("please specify a valid Purl Type to query") - } - query := "SELECT component, v.version_name AS version, v.semver AS semver," + - " l.license_name AS license, l.spdx_id AS license_id, l.is_spdx AS is_spdx," + - " purl_name, mine_id FROM golang_projects u" + - " LEFT JOIN mines m ON u.mine_id = m.id" + - " LEFT JOIN licenses l ON u.license_id = l.id" + - " LEFT JOIN versions v ON u.version_id = v.id" + - " WHERE m.purl_type = $1 AND u.purl_name = $2 AND is_indexed = True" + - " ORDER BY version_date DESC" - var allURLs []AllURL - err := m.conn.SelectContext(m.ctx, &allURLs, query, purlType, purlName) - if err != nil { - m.s.Errorf("Failed to query golang projects table for %v - %v: %v", purlType, purlName, err) - return AllURL{}, fmt.Errorf("failed to query the golang projects table: %v", err) - } - m.s.Debugf("Found %v results for %v, %v.", len(allURLs), purlType, purlName) - if len(allURLs) == 0 { // Check pkg.go.dev for the latest data - m.s.Debugf("Checking PkgGoDev for live info...") - allURL, err := m.getLatestPkgGoDev(purlName, purlType, "") - if err == nil { - m.s.Debugf("Retrieved golang data from pkg.go.dev: %#v", allURL) - allURLs = append(allURLs, allURL) - } else { - m.s.Infof("Ran into an issue looking up pkg.go.dev for: %v. Ignoring", purlName) - } - } - - // Pick the most appropriate version to return - return pickOneURL(m.s, m.project, allURLs, purlName, purlType, purlReq) -} - -// GetGolangUrlsByPurlNameTypeVersion searches Golang Projects for specified Purl, Type and Version. -func (m *GolangProjects) GetGolangUrlsByPurlNameTypeVersion(purlName, purlType, purlVersion string) (AllURL, error) { - if len(purlName) == 0 { - m.s.Error("Please specify a valid Purl Name to query") - return AllURL{}, errors.New("please specify a valid Purl Name to query") - } - if len(purlType) == 0 { - m.s.Error("Please specify a valid Purl Type to query") - return AllURL{}, errors.New("please specify a valid Purl Type to query") - } - if len(purlVersion) == 0 { - m.s.Error("Please specify a valid Purl Version to query") - return AllURL{}, errors.New("please specify a valid Purl Version to query") - } - query := "SELECT component, v.version_name AS version, v.semver AS semver," + - " l.license_name AS license, l.spdx_id AS license_id, l.is_spdx AS is_spdx," + - " purl_name, mine_id FROM golang_projects u" + - " LEFT JOIN mines m ON u.mine_id = m.id" + - " LEFT JOIN licenses l ON u.license_id = l.id" + - " LEFT JOIN versions v ON u.version_id = v.id" + - " WHERE m.purl_type = $1 AND u.purl_name = $2 AND v.version_name = $3 AND is_indexed = True" + - " ORDER BY version_date DESC" - var allURLs []AllURL - err := m.conn.SelectContext(m.ctx, &allURLs, query, purlType, purlName, purlVersion) - if err != nil { - m.s.Errorf("Failed to query golang projects table for %v - %v: %v", purlType, purlName, err) - return AllURL{}, fmt.Errorf("failed to query the golang projects table: %v", err) - } - m.s.Debugf("Found %v results for %v, %v.", len(allURLs), purlType, purlName) - if len(allURLs) > 0 { // We found an entry. Let's check if it has license data - allURL, err2 := pickOneURL(m.s, m.project, allURLs, purlName, purlType, "") - if len(allURL.License) == 0 { // No license data found. Need to search for live info - m.s.Debugf("Couldn't find license data for component. Need to search live data") - allURLs = allURLs[:0] - } else { - return allURL, err2 // Return the component details - } - } - if len(allURLs) == 0 { // Check pkg.go.dev for the latest data - m.s.Debugf("Checking PkgGoDev for live info...") - allURL, err := m.getLatestPkgGoDev(purlName, purlType, purlVersion) - if err == nil { - m.s.Debugf("Retrieved golang data from pkg.go.dev: %#v", allURL) - allURLs = append(allURLs, allURL) - } else { - m.s.Infof("Ran into an issue looking up pkg.go.dev for: %v - %v. Ignoring", purlName, purlVersion) - } - } - // Pick the most appropriate version to return - return pickOneURL(m.s, m.project, allURLs, purlName, purlType, "") -} - -// savePkg writes the given package details to the Golang Projects table. -// -//goland:noinspection ALL -func (m *GolangProjects) savePkg(allURL AllURL, version Version, license License, comp *pkggodevclient.Package) error { - if len(allURL.PurlName) == 0 { - m.s.Error("Please specify a valid Purl to save") - return errors.New("please specify a valid Purl to save") - } - if allURL.MineID <= 0 { - m.s.Error("Please specify a valid mine id to save") - return errors.New("please specify a valid mine id to save") - } - if version.ID <= 0 || len(version.VersionName) == 0 { - m.s.Error("Please specify a valid version to save") - return errors.New("please specify a valid version to save") - } - if license.ID <= 0 || len(license.LicenseName) == 0 { - m.s.Error("Please specify a valid license to save") - return errors.New("please specify a valid license to save") - } - if comp == nil { - m.s.Error("Please specify a valid component package to save") - return errors.New("please specify a valid component package to save") - } - m.s.Debugf("Attempting to save '%#v' - %#v to the golang_projects table...", allURL, version) - // Search for an existing entry first - var existingPurl string - err := m.conn.QueryRowxContext(m.ctx, - "SELECT purl_name FROM golang_projects"+ - " WHERE purl_name = $1 AND version = $2", - allURL.PurlName, allURL.Version, - ).Scan(&existingPurl) - if err != nil && err != sql.ErrNoRows { - m.s.Warnf("Error: Problem encountered searching golang_projects table for %v: %v", allURL, err) - } - var purlName string - sqlQueryType := "insert" - if len(existingPurl) > 0 { - // update entry - sqlQueryType = "update" - m.s.Debugf("Updating new Golang project: %#v", comp) - //goland:noinspection ALL - err = m.conn.QueryRowxContext(m.ctx, - "UPDATE golang_projects SET component = $1, version = $2, version_id = $3, version_date = $4,"+ - " is_module = $5, is_package = $6, license = $7, license_id = $8, has_valid_go_mod_file = $9,"+ - " has_redistributable_license = $10, has_tagged_version = $11, has_stable_version = $12,"+ - " repository = $13, is_indexed = $14, purl_name = $15, mine_id = $16"+ - " WHERE purl_name = $17 AND version = $18"+ - " RETURNING purl_name", - allURL.Component, allURL.Version, version.ID, comp.Published, - comp.IsModule, comp.IsPackage, license.LicenseName, license.ID, comp.HasValidGoModFile, - comp.HasRedistributableLicense, comp.HasTaggedVersion, comp.HasStableVersion, - comp.Repository, true, allURL.PurlName, allURL.MineID, - allURL.PurlName, allURL.Version, - ).Scan(&purlName) - } else { - m.s.Debugf("Inserting new Golang project: %#v", comp) - // insert new entry - err = m.conn.QueryRowxContext(m.ctx, - "INSERT INTO golang_projects (component, version, version_id, version_date, is_module, is_package,"+ - " license, license_id, has_valid_go_mod_file, has_redistributable_license, has_tagged_version,"+ - " has_stable_version, repository, is_indexed, purl_name, mine_id, index_timestamp)"+ - " VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)"+ - " RETURNING purl_name", - allURL.Component, allURL.Version, version.ID, comp.Published, - comp.IsModule, comp.IsPackage, license.LicenseName, license.ID, comp.HasValidGoModFile, - comp.HasRedistributableLicense, comp.HasTaggedVersion, comp.HasStableVersion, - comp.Repository, true, allURL.PurlName, allURL.MineID, "", - ).Scan(&purlName) - } - if err != nil { - m.s.Errorf("Error: Failed to %v new component into golang_projects table for %v - %#v: %v", sqlQueryType, allURL, comp, err) - return fmt.Errorf("failed to %v new component into golang projects: %v", sqlQueryType, err) - } - m.s.Debugf("Completed %v of %v", sqlQueryType, purlName) - return nil -} - -// getLatestPkgGoDev retrieves the latest information about a Golang Package from https://pkg.go.dev -// If requested (via config), it will commit that data to the Golang Projects table. -func (m *GolangProjects) getLatestPkgGoDev(purlName, purlType, purlVersion string) (AllURL, error) { - allURL, pkg, latest, err := m.queryPkgGoDev(purlName, purlVersion) - if err != nil { - return allURL, err - } - cleansedLicense, err := CleanseLicenseName(allURL.License) - if err != nil { - return allURL, err - } - - m.s.Infof("Getting license name: %s, %+v", cleansedLicense, m.config.Components.CommitMissing) - - license, err := m.lic.GetLicenseByName(cleansedLicense, m.config.Components.CommitMissing) - if err != nil { - m.s.Warnf("No license details in DB for: %v, %+v", cleansedLicense, err) - } - if len(license.LicenseName) == 0 { - m.s.Warnf("No license details in DB for: %v", cleansedLicense) - } else { - allURL.License = license.LicenseName - allURL.LicenseID = license.LicenseID - allURL.IsSpdx = license.IsSpdx - } - version, _ := m.ver.GetVersionByName(allURL.Version, m.config.Components.CommitMissing) - if len(version.VersionName) == 0 { - m.s.Warnf("No version details in DB for: %v", allURL.Version) - } - mineIDs, _ := m.mine.GetMineIdsByPurlType(purlType) - if len(mineIDs) > 0 { - allURL.MineID = mineIDs[0] // Assign the first mine id - } else { - m.s.Warnf("No mine details in DB for purl type: %v", purlType) - } - // Package is not the "latest" version (i.e. queried with a version) and we've been requested to save it - if !latest && m.config.Components.CommitMissing { - _ = m.savePkg(allURL, version, license, pkg) - } - return allURL, nil -} - -// queryPkgGoDev retrieves the latest information about a Golang Package from https://pkg.go.dev -func (m *GolangProjects) queryPkgGoDev(purlName, purlVersion string) (AllURL, *pkggodevclient.Package, bool, error) { - if len(purlName) == 0 { - m.s.Errorf("Please specify a valid Purl Name to query") - return AllURL{}, nil, false, errors.New("please specify a valid Purl Name to query") - } - client := pkggodevclient.New() - - if client == nil { - return AllURL{}, nil, false, errors.New("failed to create pkg.go.dev client") - } - - pkg := purlName - if len(purlVersion) > 0 { - pkg = fmt.Sprintf("%s@%s", purlName, purlVersion) - } - latest := false - m.s.Debugf("Checking pkg.go.dev for the latest info: %v", pkg) - comp, err := client.DescribePackage(pkggodevclient.DescribePackageRequest{Package: pkg}) - if err != nil && len(purlVersion) > 0 { - // We have a version zero search, so look for the latest one - m.s.Debugf("Failed to query pkg.go.dev for %v: %v. Trying without version...", pkg, err) - comp, err = client.DescribePackage(pkggodevclient.DescribePackageRequest{Package: purlName}) - latest = true // Mark that this information is from the latest package and not a specific version - } - if err != nil { - m.s.Warnf("Failed to query pkg.go.dev for %v: %v", pkg, err) - return AllURL{}, nil, latest, fmt.Errorf("failed to query pkg.go.dev: %v", err) - } - var version = comp.Version - if len(purlVersion) > 0 { - version = purlVersion // Force the requested version if specified (the returned value can be concatenated) - } - allURL := AllURL{ - Component: purlName, - Version: version, - License: comp.License, - PurlName: purlName, - URL: fmt.Sprintf("https://%v", comp.Repository), - } - return allURL, comp, latest, nil -} diff --git a/pkg/models/golang_projects_test.go b/pkg/models/golang_projects_test.go deleted file mode 100644 index afd02bf..0000000 --- a/pkg/models/golang_projects_test.go +++ /dev/null @@ -1,469 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2018-2025 SCANOSS.COM - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 2 of the License, or - * (at your option) any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package models - -import ( - "context" - "fmt" - "testing" - - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - pkggodevclient "github.com/guseggert/pkggodev-client" - "github.com/jmoiron/sqlx" - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" - - "scanoss.com/vulnerabilities/pkg/config" -) - -const ScanossPapiURL = "github.com/scanoss/papi" -const VersionV001 = "v0.0.1" -const VersionV002 = "v0.0.2" -const MITLicense = "MIT" - -func TestGolangProjectUrlsSearch(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = LoadTestSQLData(db, ctx, conn) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - myConfig.Database.Trace = true - golangProjModel := NewGolangProjectModel(ctx, zlog.S, conn, myConfig) - - url, err := golangProjModel.GetGolangUrlsByPurlNameType("google.golang.org/grpc", "golang", "") - if err != nil { - t.Errorf("FAILED: golang_projects.GetUrlsByPurlName() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() No URLs returned from query") - } - fmt.Printf("Golang URL: %#v\n", url) - - url, err = golangProjModel.GetGolangUrlsByPurlNameType("NONEXISTENT", "none", "") - if err != nil { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameType() error = %v", err) - } - if len(url.PurlName) > 0 { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameType() URLs found when none should be: %v", golangProjModel) - } - fmt.Printf("No Urls: %v\n", url) - - _, err = golangProjModel.GetGolangUrlsByPurlNameType("NONEXISTENT", "", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameType() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = golangProjModel.GetGolangUrlsByPurlNameType("", "", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = golangProjModel.GetGoLangURLByPurlString("", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetURLsByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = golangProjModel.GetGoLangURLByPurlString("rubbish-purl", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", "") - if err != nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() No URLs returned from query") - } - fmt.Printf("Golang URL: %v\n", url) -} - -func TestGolangProjectsSearchVersion(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = LoadTestSQLData(db, ctx, conn) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("FAILED: failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - myConfig.Database.Trace = true - golangProjModel := NewGolangProjectModel(ctx, zlog.S, conn, myConfig) - - url, err := golangProjModel.GetGolangUrlsByPurlNameTypeVersion("google.golang.org/grpc", "golang", "1.19.0") - if err != nil { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameTypeVersion() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameTypeVersion() No URLs returned from query") - } - fmt.Printf("Golang URL Version: %#v\n", url) - - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc@v1.19.0", "") - if err != nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = failed to find purl by version string") - } - fmt.Printf("Golang URL Version: %#v\n", url) - - _, err = golangProjModel.GetGolangUrlsByPurlNameTypeVersion("", "", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameTypeVersion() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - - _, err = golangProjModel.GetGolangUrlsByPurlNameTypeVersion("NONEXISTENT", "", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameTypeVersion() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - - _, err = golangProjModel.GetGolangUrlsByPurlNameTypeVersion("NONEXISTENT", "NONEXISTENT", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGolangUrlsByPurlNameTypeVersion() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", "22.22.22") // Shouldn't exist - if err != nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = failed to find purl by version string") - } - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", "=v1.19.0") - if err != nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() No URLs returned from query") - } - fmt.Printf("Golang URL: %v\n", url) - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", "==v1.19.0") - if err != nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() No URLs returned from query") - } - fmt.Printf("Golang URL: %v\n", url) - - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc@1.7.0", "") // Should be missing license - if err != nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = %v", err) - } - if len(url.License) == 0 { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() No URL License returned from query") - } - fmt.Printf("Golang URL: %v\n", url) -} - -func TestGolangProjectsSearchVersionRequirement(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - err = LoadTestSQLData(db, ctx, conn) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - golangProjModel := NewGolangProjectModel(ctx, zlog.S, conn, myConfig) - - url, err := golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", ">0.0.4") - if err != nil { - t.Errorf("FAILED: golang_projects.GetUrlsByPurlName() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetUrlsByPurlName() No URLs returned from query") - } - fmt.Printf("Golang URL Version: %#v\n", url) - - url, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", "v0.0.0-201910101010-s3333") - if err != nil { - t.Errorf("FAILED: golang_projects.GetUrlsByPurlName() error = %v", err) - } - if len(url.PurlName) == 0 { - t.Errorf("FAILED: golang_projects.GetUrlsByPurlName() No URLs returned from query") - } - fmt.Printf("Golang URL Version: %#v\n", url) -} - -func TestGolangPkgGoDev(t *testing.T) { - // Setup test environment and models - golangProjModel, cleanup := setupTestEnvironment(t) - defer cleanup() - - // Run subtests - t.Run("QueryPkgGoDev", testQueryPkgGoDev(golangProjModel)) - t.Run("GetLatestPkgGoDev", testGetLatestPkgGoDev(golangProjModel)) - t.Run("SavePkg", testSavePkg(golangProjModel)) -} - -// setupTestEnvironment creates the test environment and returns cleanup function. -func setupTestEnvironment(t *testing.T) (*GolangProjects, func()) { - t.Helper() - ctx := context.Background() - - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - - conn, err := db.Connx(ctx) - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - - err = LoadTestSQLData(db, ctx, conn) - if err != nil { - t.Fatalf("failed to load SQL test data: %v", err) - } - - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - - golangProjModel := NewGolangProjectModel(ctx, zlog.S, conn, myConfig) - - cleanup := func() { - CloseConn(conn) - CloseDB(db) - zlog.SyncZap() - } - - return golangProjModel, cleanup -} - -func testQueryPkgGoDev(model *GolangProjects) func(t *testing.T) { - return func(t *testing.T) { - _, _, _, err := model.queryPkgGoDev("", "") - if err == nil { - t.Error("FAILED: golang_projects.queryPkgGoDev() error = did not get an error") - } - } -} - -func testGetLatestPkgGoDev(model *GolangProjects) func(t *testing.T) { - return func(t *testing.T) { - t.Run("GRPC Package", func(t *testing.T) { - url, err := model.getLatestPkgGoDev("google.golang.org/grpc", "golang", "v0.0.0-201910101010-s3333") - if err != nil { - t.Errorf("error = %v", err) - } - if len(url.PurlName) == 0 { - t.Error("No URLs returned from query") - } - }) - - t.Run("Scanoss Package", func(t *testing.T) { - url, err := model.getLatestPkgGoDev(ScanossPapiURL, "golang", "v0.0.3") - if err != nil { - t.Errorf("error = %v", err) - } - if len(url.PurlName) == 0 { - t.Error("No URLs returned from query") - } - }) - } -} - -func testSavePkg(model *GolangProjects) func(t *testing.T) { - return func(t *testing.T) { - t.Run("Empty URL", func(t *testing.T) { - var allURL AllURL - var license License - var version Version - - err := model.savePkg(allURL, version, license, nil) - if err == nil { - t.Error("expected error with empty URL") - } - }) - - t.Run("Missing MineID", func(t *testing.T) { - allURL := AllURL{PurlName: ScanossPapiURL} - var license License - var version Version - - err := model.savePkg(allURL, version, license, nil) - if err == nil { - t.Error("expected error with missing MineID") - } - }) - - t.Run("Valid Package", func(t *testing.T) { - allURL, version, license, comp := createValidTestData() - err := model.savePkg(allURL, version, license, &comp) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - }) - - t.Run("Updated Version", func(t *testing.T) { - allURL, version, license, comp := createValidTestData() - allURL.Version = VersionV002 - version.VersionName = VersionV002 - comp.Version = VersionV002 - - err := model.savePkg(allURL, version, license, &comp) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - }) - } -} - -func createValidTestData() (AllURL, Version, License, pkggodevclient.Package) { - allURL := AllURL{ - PurlName: ScanossPapiURL, - MineID: 45, - Version: VersionV001, - } - - version := Version{ - VersionName: VersionV001, - ID: 5958021, - } - - license := License{ - LicenseName: MITLicense, - ID: 5614, - } - - comp := pkggodevclient.Package{ - Package: ScanossPapiURL, - IsPackage: true, - IsModule: true, - Version: VersionV001, - License: MITLicense, - HasRedistributableLicense: true, - HasStableVersion: true, - HasTaggedVersion: true, - HasValidGoModFile: true, - Repository: ScanossPapiURL, - } - - return allURL, version, license, comp -} - -func TestGolangProjectsSearchBadSql(t *testing.T) { - ctx := context.Background() - err := zlog.NewSugaredDevLogger() - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) - } - defer zlog.SyncZap() - db, err := sqlx.Connect("sqlite3", ":memory:") - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseDB(db) - conn, err := db.Connx(ctx) // Get a connection from the pool - if err != nil { - t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) - } - defer CloseConn(conn) - s := ctxzap.Extract(ctx).Sugar() - myConfig, err := config.NewServerConfig(nil) - if err != nil { - t.Fatalf("failed to load Config: %v", err) - } - myConfig.Components.CommitMissing = true - golangProjModel := NewGolangProjectModel(ctx, s, conn, myConfig) - - _, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = golangProjModel.GetGoLangURLByPurlString("pkg:golang/google.golang.org/grpc@1.19.0", "") - if err == nil { - t.Errorf("FAILED: golang_projects.GetGoLangURLByPurlString() error = did not get an error") - } else { - fmt.Printf("Got expected error = %v\n", err) - } - _, err = golangProjModel.getLatestPkgGoDev("github.com/scanoss/does-not-exist", "golang", "v0.0.99") - if err == nil { - t.Errorf("FAILED: golang_projects.getLatestPkgGoDev() error = did not get an error: %v", err) - } else { - fmt.Printf("Got expected error = %v\n", err) - } -} diff --git a/pkg/models/tests/epss.sql b/pkg/models/tests/epss.sql new file mode 100644 index 0000000..3a251c2 --- /dev/null +++ b/pkg/models/tests/epss.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS epss_data; +CREATE TABLE "epss_data" ( + "cve" TEXT NOT NULL UNIQUE, + "epss" REAL NOT NULL, + "percentile" REAL NOT NULL, + PRIMARY KEY ("cve") +); +INSERT INTO epss_data (cve, epss, percentile) VALUES ('CVE-2017-9302', 0.00143, 0.5124); +INSERT INTO epss_data (cve, epss, percentile) VALUES ('CVE-2015-0269', 0.00285, 0.6832); +INSERT INTO epss_data (cve, epss, percentile) VALUES ('CVE-2018-10083', 0.00891, 0.8215); \ No newline at end of file diff --git a/pkg/models/vulns_purl.go b/pkg/models/vulns_purl.go index 7d9c998..e13515b 100644 --- a/pkg/models/vulns_purl.go +++ b/pkg/models/vulns_purl.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + purlhelper "github.com/scanoss/go-purl-helper/pkg" zlog "github.com/scanoss/zap-logging-helper/pkg/logger" "github.com/jmoiron/sqlx" @@ -59,7 +60,7 @@ func (m *VulnsForPurlModel) GetVulnsByPurl(purl string, version string) ([]Vulns } // used to valid the PURL - _, err := utils.PurlFromString(purl) + _, err := purlhelper.PurlFromString(purl) if err != nil { return []VulnsForPurl{}, err } diff --git a/pkg/service/vulnerability_service.go b/pkg/service/vulnerability_service.go index 44fe65d..53d9ffe 100644 --- a/pkg/service/vulnerability_service.go +++ b/pkg/service/vulnerability_service.go @@ -53,6 +53,7 @@ func (d vulnerabilityServer) Echo(ctx context.Context, request *common.EchoReque // //nolint:staticcheck // SA1019: keeping deprecated pb.VulnerabilityRequest and pb.VulnerabilityResponse for backward compatibility func (d vulnerabilityServer) GetVulnerabilities(ctx context.Context, request *pb.VulnerabilityRequest) (*pb.VulnerabilityResponse, error) { + s := ctxzap.Extract(ctx).Sugar() // Sanitize request components, invalidComponents, err := adapters.FromVulnerabilityRequestToComponentDTO(request) // Convert to internal DTO for processing if err != nil { @@ -67,7 +68,7 @@ func (d vulnerabilityServer) GetVulnerabilities(ctx context.Context, request *pb return &pb.VulnerabilityResponse{Status: &statusResp}, errors.New("no request data supplied") } - vulUseCase := usecase.NewVulnerabilityUseCase(ctx, d.db, d.config) + vulUseCase := usecase.NewVulnerabilityUseCase(s, d.config, d.db) vulnerabilities, err := vulUseCase.Execute(ctx, components) if err != nil { zlog.S.Errorf("Failed to get Vulnerabilities: %v", err) @@ -191,7 +192,7 @@ func (d vulnerabilityServer) GetComponentVulnerabilities(ctx context.Context, re errors.New("problem parsing Vulnerability input data") } - vulUseCase := usecase.NewVulnerabilityUseCase(ctx, d.db, d.config) + vulUseCase := usecase.NewVulnerabilityUseCase(s, d.config, d.db) vulnerabilities, err := vulUseCase.Execute(ctx, []dtos.ComponentDTO{componentDTO}) if err != nil { @@ -222,7 +223,7 @@ func (d vulnerabilityServer) GetComponentsVulnerabilities(ctx context.Context, r errors.New("problem parsing Vulnerability input data") } - vulUseCase := usecase.NewVulnerabilityUseCase(ctx, d.db, d.config) + vulUseCase := usecase.NewVulnerabilityUseCase(s, d.config, d.db) vulnerabilities, err := vulUseCase.Execute(ctx, componentDTOs) if err != nil { diff --git a/pkg/usecase/OSV_use_case.go b/pkg/usecase/OSV_use_case.go index aa9e5db..4cc90a7 100644 --- a/pkg/usecase/OSV_use_case.go +++ b/pkg/usecase/OSV_use_case.go @@ -20,11 +20,13 @@ import ( "bytes" "context" "encoding/json" - "io" "net/http" - "sync" "time" + "go.uber.org/zap" + + "scanoss.com/vulnerabilities/pkg/config" + "scanoss.com/vulnerabilities/pkg/dtos" "scanoss.com/vulnerabilities/pkg/utils" @@ -45,18 +47,20 @@ type OSVRequest struct { type OSVUseCase struct { OSVAPIBaseURL string OSVInfoBaseURL string - semaphore chan struct{} // Used to limit concurrent requests - client *http.Client // Single shared + client *http.Client // Single shared + MaxAPIWorkers int + s *zap.SugaredLogger } -func NewOSVUseCase(osvAPIBaseURL string, osvInfoBaseURL string) *OSVUseCase { +func NewOSVUseCase(s *zap.SugaredLogger, config *config.ServerConfig) *OSVUseCase { return &OSVUseCase{ - OSVAPIBaseURL: osvAPIBaseURL, - OSVInfoBaseURL: osvInfoBaseURL, - semaphore: make(chan struct{}, 4), + OSVAPIBaseURL: config.Source.OSV.APIBaseURL, + OSVInfoBaseURL: config.Source.OSV.InfoBaseURL, client: &http.Client{ - Timeout: 10 * time.Second, + Timeout: 15 * time.Second, }, + MaxAPIWorkers: config.Source.OSV.APIWorkers, + s: s, } } @@ -83,90 +87,104 @@ func (us OSVUseCase) Execute(dto []dtos.ComponentDTO) dtos.VulnerabilityOutput { } func (us OSVUseCase) processRequests(requests []OSVRequest) dtos.VulnerabilityOutput { - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(context.Background()) - results := make(chan dtos.VulnerabilityComponentOutput, len(requests)) + numJobs := len(requests) + jobs := make(chan OSVRequest, numJobs) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() - for _, request := range requests { - wg.Add(1) - go func(req OSVRequest) { - defer wg.Done() - // Try to acquire semaphore - us.semaphore <- struct{}{} // Will block if 4 requests are already running - defer func() { <-us.semaphore }() // Release when done - - select { - case <-ctx.Done(): - return - default: - r, _ := us.processRequest(req) - results <- r - } - }(request) + results := make(chan dtos.VulnerabilityComponentOutput, numJobs) + workers := min(us.MaxAPIWorkers, numJobs) + for i := 0; i < workers; i++ { + go us.processRequest(ctx, jobs, results) } - wg.Wait() - close(results) + + for _, r := range requests { + jobs <- r + } + close(jobs) // Collect all results into a slice var response = dtos.VulnerabilityOutput{ Components: []dtos.VulnerabilityComponentOutput{}, } - for result := range results { + for i := 0; i < numJobs; i++ { + result := <-results response.Components = append(response.Components, result) } - return response } -func (us OSVUseCase) processRequest(osvRequest OSVRequest) (dtos.VulnerabilityComponentOutput, error) { - out, err := json.Marshal(osvRequest) - if err != nil { - zlog.S.Errorf("Failed to marshal request: %s", err) - return dtos.VulnerabilityComponentOutput{}, err - } - - req, err := http.NewRequest(http.MethodPost, us.OSVAPIBaseURL+"/query", bytes.NewBuffer(out)) - if err != nil { - zlog.S.Errorf("Failed to create HTTP request: %s", err) - return dtos.VulnerabilityComponentOutput{}, err - } - req.Header.Set("Content-Type", "application/json") - - // Use a shared HTTP client to avoid creating a new one every call - resp, err := us.client.Do(req) - if err != nil { - zlog.S.Errorf("HTTP request failed: %s", err) - return dtos.VulnerabilityComponentOutput{}, err - } - - defer func(Body io.ReadCloser) { - err = Body.Close() - if err != nil { - zlog.S.Errorf("Failed to close HTTP response body: %s", err) +// processRequest is a worker function that processes OSV vulnerability requests concurrently. +// It reads requests from the jobs channel, queries the OSV API for each request, and sends +// the results to the results channel. The worker terminates when the jobs channel is closed +// or when the context is cancelled. +func (us OSVUseCase) processRequest(ctx context.Context, jobs chan OSVRequest, results chan dtos.VulnerabilityComponentOutput) { + for { + select { + case j, ok := <-jobs: + if !ok { + return // Channel closed, stop worker + } + response := dtos.VulnerabilityComponentOutput{ + Purl: j.Package.Purl, + Requirement: j.Requirement, + Version: j.Version, + } + out, err := json.Marshal(struct { + Version string `json:"version,omitempty"` + Package OSVPackageRequest `json:"package"` + }{ + Version: j.Version, + Package: j.Package, + }) + if err != nil { + us.s.Errorf("Failed to marshal request: %s", err) + results <- response + continue + } + req, err := http.NewRequest(http.MethodPost, us.OSVAPIBaseURL+"/query", bytes.NewBuffer(out)) + if err != nil { + us.s.Errorf("Failed to create HTTP request: %s", err) + results <- response + continue + } + req.Header.Set("Content-Type", "application/json") + + // Use a shared HTTP client to avoid creating a new one every call + resp, err := us.client.Do(req) + if err != nil { + us.s.Errorf("HTTP request failed: %s", err) + results <- response + continue + } + // Check for non-200 HTTP responses + if resp.StatusCode != http.StatusOK { + us.s.Errorf("Unexpected HTTP status: %d", resp.StatusCode) + err = resp.Body.Close() + if err != nil { + us.s.Errorf("Failed to close response body: %s", err) + } + results <- response + continue + } + var OSVResponse dtos.OSVResponseDTO + err = json.NewDecoder(resp.Body).Decode(&OSVResponse) + if err != nil { + us.s.Errorf("Failed to decode response: %s", err) + results <- response + continue + } + err = resp.Body.Close() + if err != nil { + us.s.Errorf("Failed to close response body: %s", err) + } + response.Vulnerabilities = us.mapOSVVulnerabilities(OSVResponse.Vulns) + results <- response + case <-ctx.Done(): + // Cancellation signal received: stop working and return immediately + us.s.Debugf("Worker: Cancellation signal received, stopping.") + return } - }(resp.Body) - - // Check for non-200 HTTP responses - if resp.StatusCode != http.StatusOK { - zlog.S.Errorf("Unexpected HTTP status: %d", resp.StatusCode) - return dtos.VulnerabilityComponentOutput{}, err - } - - var OSVResponse dtos.OSVResponseDTO - err = json.NewDecoder(resp.Body).Decode(&OSVResponse) - if err != nil { - // Handle error - zlog.S.Errorf("Failed to decode response: %s", err) - return dtos.VulnerabilityComponentOutput{}, err - } - - response := dtos.VulnerabilityComponentOutput{ - Purl: osvRequest.Package.Purl, - Requirement: osvRequest.Requirement, - Version: osvRequest.Version, - Vulnerabilities: us.mapOSVVulnerabilities(OSVResponse.Vulns), } - return response, nil } // mapOSVVulnerabilities converts OSV vulnerabilities to the required DTO structure. diff --git a/pkg/usecase/OSV_use_case_test.go b/pkg/usecase/OSV_use_case_test.go index 5ef9e94..fde298d 100644 --- a/pkg/usecase/OSV_use_case_test.go +++ b/pkg/usecase/OSV_use_case_test.go @@ -17,10 +17,13 @@ package usecase import ( + "context" "testing" - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + zlog "github.com/scanoss/zap-logging-helper/pkg/logger" + "scanoss.com/vulnerabilities/pkg/config" "scanoss.com/vulnerabilities/pkg/dtos" ) @@ -30,6 +33,14 @@ func TestOSVUseCase(t *testing.T) { t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) } defer zlog.SyncZap() + ctx := ctxzap.ToContext(context.Background(), zlog.L) + s := ctxzap.Extract(ctx).Sugar() + + serverConfig, err := config.NewServerConfig(nil) + if err != nil { + t.Fatalf("failed to load Config: %v", err) + } + testCases := []struct { name string input []dtos.ComponentDTO @@ -47,9 +58,7 @@ func TestOSVUseCase(t *testing.T) { }, }, } - OSVBaseURL := "https://api.osv.dev/v1" - OSVInfoBaseURL := "https://test.osv.dev/vulnerability" - OSVUseCase := NewOSVUseCase(OSVBaseURL, OSVInfoBaseURL) + OSVUseCase := NewOSVUseCase(s, serverConfig) for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := OSVUseCase.Execute(tc.input) diff --git a/pkg/usecase/vulnerability_use_case.go b/pkg/usecase/vulnerability_use_case.go index b7d729e..e06cf73 100644 --- a/pkg/usecase/vulnerability_use_case.go +++ b/pkg/usecase/vulnerability_use_case.go @@ -23,9 +23,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/scanoss/go-models/pkg/scanoss" "github.com/scanoss/go-models/pkg/types" - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" - - myconfig "scanoss.com/vulnerabilities/pkg/config" + "go.uber.org/zap" + "scanoss.com/vulnerabilities/pkg/config" "scanoss.com/vulnerabilities/pkg/dtos" "scanoss.com/vulnerabilities/pkg/helpers" "scanoss.com/vulnerabilities/pkg/models" @@ -37,26 +36,27 @@ type IVulnerabilityUseCase interface { type VulnerabilityUseCase struct { db *sqlx.DB - config *myconfig.ServerConfig + config *config.ServerConfig + s *zap.SugaredLogger } -func NewVulnerabilityUseCase(ctx context.Context, db *sqlx.DB, config *myconfig.ServerConfig) *VulnerabilityUseCase { +func NewVulnerabilityUseCase(s *zap.SugaredLogger, config *config.ServerConfig, db *sqlx.DB) *VulnerabilityUseCase { return &VulnerabilityUseCase{ db: db, config: config, + s: s, } } func (us VulnerabilityUseCase) Execute(ctx context.Context, components []dtos.ComponentDTO) (dtos.VulnerabilityOutput, error) { - zlog.S.Infof("Processing Vulnerabilities request: %v", components) + us.s.Debugf("Processing Vulnerabilities request: %v", components) if len(components) == 0 { return dtos.VulnerabilityOutput{}, errors.New("no request data supplied") } - - zlog.S.Infof("Getting DB Connection from pool: %v", ctx) + us.s.Debugf("Getting DB Connection from pool: %v", ctx) conn, err := us.db.Connx(ctx) // Get a connection from the pool if err != nil { - zlog.S.Errorf("Failed to get a database connection from the pool: %v", err) + us.s.Errorf("Failed to get a database connection from the pool: %v", err) return dtos.VulnerabilityOutput{}, errors.New("problem getting database pool connection") } defer models.CloseConn(conn) @@ -70,7 +70,7 @@ func (us VulnerabilityUseCase) Execute(ctx context.Context, components []dtos.Co Requirement: c.Requirement, }) if err != nil { - zlog.S.Warnf("Failed to get component: %s, %s", c.Purl, c.Requirement) + us.s.Warnf("Failed to get component: %s, %s", c.Purl, c.Requirement) continue } if component.Version != "" { @@ -81,8 +81,8 @@ func (us VulnerabilityUseCase) Execute(ctx context.Context, components []dtos.Co // Gets OSV vulnerabilities only if enabled var osvVulnerabilities = dtos.VulnerabilityOutput{} if us.config.Source.OSV.Enabled { - zlog.S.Infof("vulnerabilities: OSV enabled") - osvUseCase := NewOSVUseCase(us.config.Source.OSV.APIBaseURL, us.config.Source.OSV.InfoBaseURL) + us.s.Debugf("vulnerabilities: OSV enabled") + osvUseCase := NewOSVUseCase(us.s, us.config) osvVulnerabilities = osvUseCase.Execute(components) } // ************* OSV Use case end *************** / @@ -92,14 +92,49 @@ func (us VulnerabilityUseCase) Execute(ctx context.Context, components []dtos.Co localVulUc := NewLocalVulnerabilitiesUseCase(ctx, conn, us.config) localVulnerabilities, err = localVulUc.GetVulnerabilities(components) if err != nil { - zlog.S.Errorf("Failed to get Vulnerabilities: %v", err) + us.s.Errorf("Failed to get Vulnerabilities: %v", err) return dtos.VulnerabilityOutput{}, errors.New("problems encountered extracting vulnerability data") } } // Merge OSV and local vulnerabilities in one response. Avoids duplicated vulnerabilities := helpers.MergeOSVAndLocalVulnerabilities(localVulnerabilities, osvVulnerabilities) + // Add EPSS data + us.enrichWithEPSS(ctx, &vulnerabilities) return vulnerabilities, nil } +// enrichWithEPSS fetches EPSS data for all CVEs and adds it to the vulnerabilities. +func (us VulnerabilityUseCase) enrichWithEPSS(ctx context.Context, vulnerabilities *dtos.VulnerabilityOutput) { + var cves []string + for _, c := range vulnerabilities.Components { + for _, v := range c.Vulnerabilities { + if v.Cve != "" { + cves = append(cves, v.Cve) + } + } + } + if len(cves) == 0 { + return + } + EPSSModel := models.NewEPSSModel(us.s, us.config, us.db) + epss, err := EPSSModel.GetEPSSByCVEs(ctx, cves) + if err != nil { + us.s.Errorf("Failed to get EPSS data for CVEs: %v", err) + return + } + epssMap := make(map[string]models.EPSS, len(epss)) + for _, e := range epss { + epssMap[e.Cve] = e + } + for i := range vulnerabilities.Components { + for j := range vulnerabilities.Components[i].Vulnerabilities { + if val, ok := epssMap[vulnerabilities.Components[i].Vulnerabilities[j].Cve]; ok { + vulnerabilities.Components[i].Vulnerabilities[j].Epss.Probability = val.Epss + vulnerabilities.Components[i].Vulnerabilities[j].Epss.Percentile = val.Percentile + } + } + } +} + // Optional: Type assertion to ensure VulnerabilityUseCase implements IVulnerabilityUseCase. var _ IVulnerabilityUseCase = (*VulnerabilityUseCase)(nil) diff --git a/pkg/usecase/vulnerability_use_case_test.go b/pkg/usecase/vulnerability_use_case_test.go index 0f8c0b8..db3d59b 100644 --- a/pkg/usecase/vulnerability_use_case_test.go +++ b/pkg/usecase/vulnerability_use_case_test.go @@ -21,6 +21,8 @@ import ( "fmt" "testing" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "scanoss.com/vulnerabilities/pkg/dtos" myconfig "scanoss.com/vulnerabilities/pkg/config" @@ -32,12 +34,13 @@ import ( ) func TestVulnerabilityUseCase(t *testing.T) { - ctx := context.Background() err := zlog.NewSugaredDevLogger() if err != nil { t.Fatalf("an error '%s' was not expected when opening a sugared logger", err) } defer zlog.SyncZap() + ctx := ctxzap.ToContext(context.Background(), zlog.L) + s := ctxzap.Extract(ctx).Sugar() db, err := sqlx.Connect("sqlite3", "file::memory:?cache=shared&uri=true") if err != nil { t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) @@ -57,7 +60,7 @@ func TestVulnerabilityUseCase(t *testing.T) { t.Fatalf("an error '%s' was not expected when loading server config", err) } serverConfig.Source.OSV.Enabled = false - vulnUseCase := NewVulnerabilityUseCase(ctx, db, serverConfig) + vulnUseCase := NewVulnerabilityUseCase(s, serverConfig, db) tests := []struct { name string diff --git a/pkg/utils/purl.go b/pkg/utils/purl.go index 85f4051..41fe685 100644 --- a/pkg/utils/purl.go +++ b/pkg/utils/purl.go @@ -17,128 +17,14 @@ package utils import ( - "errors" - "fmt" "regexp" - "strings" - - zlog "github.com/scanoss/zap-logging-helper/pkg/logger" - - "github.com/package-url/packageurl-go" ) -var pkgRegex = regexp.MustCompile(`^pkg:(?P\w+)/(?P.+)$`) // regex to parse purl name from purl string -var typeRegex = regexp.MustCompile(`^(npm|nuget)$`) // regex to parse purl types that should not be lower cased -var vRegex = regexp.MustCompile(`^(=|==|)(?P\w+\S+)$`) // regex to extract version from requirement field +// regex to extract version from requirement field. var fromVRegex = regexp.MustCompile(`(@|\?|#).+`) -// PurlFromString takes an input Purl string and returns a decomposed structure of all the elements. -func PurlFromString(purlString string) (packageurl.PackageURL, error) { - if len(purlString) == 0 { - return packageurl.PackageURL{}, errors.New("no Purl string specified to parse") - } - purl, err := packageurl.FromString(purlString) - if err != nil { - return packageurl.PackageURL{}, err - } - return purl, nil -} - -// PurlNameFromString take an input Purl string and returns the Purl Name only. -func PurlNameFromString(purlString string) (string, error) { - if len(purlString) == 0 { - return "", fmt.Errorf("no purl string supplied to parse") - } - matches := pkgRegex.FindStringSubmatch(purlString) - if len(matches) > 0 { - ti := pkgRegex.SubexpIndex("type") - ni := pkgRegex.SubexpIndex("name") - if ni >= 0 { - // Remove any version@/subpath?/qualifiers# info from the PURL - pn := strings.Split(strings.Split(strings.Split(matches[ni], "@")[0], "?")[0], "#")[0] - // Lowercase the purl name if it's not on the exclusion list (defined in the regex) - if ti >= 0 && !typeRegex.MatchString(matches[ti]) { - pn = strings.ToLower(pn) - } - return pn, nil - } - } - return "", fmt.Errorf("no purl name found in '%v'", purlString) -} - // PurlRemoveFromVersionComponent From a purlString removes everything that proceds @. // See purl specs scheme:type/namespace/name@version?qualifiers#subpath. func PurlRemoveFromVersionComponent(purlString string) string { return fromVRegex.ReplaceAllString(purlString, "") } - -// ConvertPurlString takes an input PURL and checks to see if anything needs to be modified before search the KB. -func ConvertPurlString(purlString string) string { - // Replace Golang GitHub package reference with just GitHub - if len(purlString) > 0 && strings.HasPrefix(purlString, "pkg:golang/github.com/") { - s := strings.ReplaceAll(purlString, "pkg:golang/github.com/", "pkg:github/") - p := strings.Split(s, "/") - if len(p) >= 3 { - return fmt.Sprintf("%s/%s/%s", p[0], p[1], p[2]) // Only return the GitHub part of the url - } - return s - } - return purlString -} - -// GetVersionFromReq parses a requirement string looking for an exact version specifier. -func GetVersionFromReq(purlReq string) string { - matches := vRegex.FindStringSubmatch(purlReq) - if len(matches) > 0 { - ni := vRegex.SubexpIndex("name") - if ni >= 0 { - zlog.S.Debugf("Changing requirement %v to Version %v", purlReq, matches[ni]) - return matches[ni] - } - } - return "" -} - -// ProjectURL returns a browsable URL for the given purl type and name. -func ProjectURL(purlName, purlType string) (string, error) { - if len(purlName) == 0 { - return "", fmt.Errorf("no purl name supplied") - } - if len(purlType) == 0 { - return "", fmt.Errorf("no purl type supplied") - } - switch purlType { - case "github": - return fmt.Sprintf("https://github.com/%v", purlName), nil - case "npm": - return fmt.Sprintf("https://www.npmjs.com/package/%v", purlName), nil - case "maven": - return fmt.Sprintf("https://mvnrepository.com/artifact/%v", purlName), nil - case "gem": - return fmt.Sprintf("https://rubygems.org/gems/%v", purlName), nil - case "pypi": - return fmt.Sprintf("https://pypi.org/project/%v", purlName), nil - case "golang": - return fmt.Sprintf("https://pkg.go.dev/%v", purlName), nil - } - return "", fmt.Errorf("no url prefix found for '%v': %v", purlType, purlName) -} - -// ConvertGoPurlStringToGithub takes an input PURL string and converts it to its GitHub equivalent if possible. -func ConvertGoPurlStringToGithub(purlString string) string { - // Replace Golang GitHub package reference with just GitHub - if len(purlString) > 0 && strings.HasPrefix(purlString, "pkg:golang/github.com/") { - s := strings.ReplaceAll(purlString, "pkg:golang/github.com/", "pkg:github/") - p := strings.Split(s, "/") - if len(p) >= 3 { - return fmt.Sprintf("%s/%s/%s", p[0], p[1], p[2]) // Only return the GitHub part of the url - } - return s - } - return purlString -} - -func StripVersionFromPurl(input string) string { - purl := strings.Split(input, "@") - return purl[0] -} diff --git a/pkg/utils/purl_test.go b/pkg/utils/purl_test.go index f1922dc..77c6bef 100644 --- a/pkg/utils/purl_test.go +++ b/pkg/utils/purl_test.go @@ -19,114 +19,8 @@ package utils import ( "reflect" "testing" - - "github.com/package-url/packageurl-go" ) -// Help with test details can be found here: https://go.dev/doc/code - -func TestPurlFromString(t *testing.T) { - w, _ := packageurl.FromString("pkg:maven/io.prestosql/presto-main@v1.0") - w2, _ := packageurl.FromString("pkg:npm/%40babel/core") - tests := []struct { - name string - input string - want packageurl.PackageURL - wantErr bool - }{ - { - name: "Purl from String", - input: "pkg:maven/io.prestosql/presto-main@v1.0", - want: w, - }, - { - name: "Purl from String", - input: "pkg:npm/%40babel/core", - want: w2, - }, - { - name: "Empty String", - input: "", - want: w, - wantErr: true, - }, - { - name: "Rubbish String", - input: "rubbish.string", - want: w, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := PurlFromString(tt.input) - if (err != nil) != tt.wantErr { - t.Errorf("utils.PurlFromString() error = %v, wantErr %v", err, tt.wantErr) - return - } - t.Logf("Got: %v: '%v' '%v' '%v' '%v' '%v' '%v'", got, got.Type, got.Namespace, got.Name, got.Version, got.Qualifiers, got.Subpath) - t.Logf("Exp: %v: '%v' '%v' '%v' '%v' '%v' '%v'", tt.want, tt.want.Type, tt.want.Namespace, tt.want.Name, tt.want.Version, tt.want.Qualifiers, tt.want.Subpath) - if err == nil && !reflect.DeepEqual(got, tt.want) { - t.Errorf("utils.PurlFromString() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPurlNameFromString(t *testing.T) { - tests := []struct { - name string - input string - want string - wantErr bool - }{ - { - name: "Maven", - input: "pkg:maven/io.prestosql/Presto-main@v1.0", - want: "io.prestosql/presto-main", - }, - { - name: "NPM1", - input: "pkg:npm/%40babel/Core", - want: "%40babel/Core", - }, - { - name: "NPM2", - input: "pkg:npm/%40babel/Core@7.0.0", - want: "%40babel/Core", - }, - { - name: "NPM3", - input: "pkg:npm/Core@0.0.1", - want: "Core", - }, - { - name: "Empty String", - input: "", - want: "", - wantErr: true, - }, - { - name: "Rubbish String", - input: "rubbish.string", - want: "", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := PurlNameFromString(tt.input) - if (err != nil) != tt.wantErr { - t.Errorf("utils.PurlFromString() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err == nil && !reflect.DeepEqual(got, tt.want) { - t.Errorf("utils.PurlFromString() = %v, want %v", got, tt.want) - } - }) - } -} - func TestPurlRemoveFromVersionComponent(t *testing.T) { tests := []struct { name string @@ -168,120 +62,3 @@ func TestPurlRemoveFromVersionComponent(t *testing.T) { }) } } - -func TestConvertPurlString(t *testing.T) { - tests := []struct { - name string - input string - want string - }{ - { - name: "Maven", - input: "pkg:maven/io.prestosql/presto-main@v1.0", - want: "pkg:maven/io.prestosql/presto-main@v1.0", - }, - { - name: "Golang1", - input: "pkg:golang/github.com/scanoss/papi", - want: "pkg:github/scanoss/papi", - }, - { - name: "Golang2", - input: "pkg:golang/github.com/scanoss/papi/v2", - want: "pkg:github/scanoss/papi", - }, - { - name: "Empty String", - input: "", - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := ConvertPurlString(tt.input) - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("utils.PurlFromString() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestPurlUrl(t *testing.T) { - tests := []struct { - name string - pname string - ptype string - want string - wantErr bool - }{ - { - name: "GitHub", - pname: "scanoss/scanoss.py", - ptype: "github", - want: "https://github.com/scanoss/scanoss.py", - }, - { - name: "Maven", - pname: "io.prestosql/presto-main", - ptype: "maven", - want: "https://mvnrepository.com/artifact/io.prestosql/presto-main", - }, - { - name: "NPM", - pname: "%40babel/core", - ptype: "npm", - want: "https://www.npmjs.com/package/%40babel/core", - }, - { - name: "PyPI", - pname: "scanoss", - ptype: "pypi", - want: "https://pypi.org/project/scanoss", - }, - { - name: "Gem", - pname: "tablestyle", - ptype: "gem", - want: "https://rubygems.org/gems/tablestyle", - }, - { - name: "Golang", - pname: "github.com/scanoss/papi", - ptype: "golang", - want: "https://pkg.go.dev/github.com/scanoss/papi", - }, - { - name: "Empty String1", - pname: "", - ptype: "gem", - want: "", - wantErr: true, - }, - { - name: "Empty String2", - pname: "io.prestosql/presto-main", - ptype: "", - want: "", - wantErr: true, - }, - { - name: "Rubbish String", - pname: "rubbish.string", - ptype: "rubbish.string", - want: "", - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ProjectURL(tt.pname, tt.ptype) - if (err != nil) != tt.wantErr { - t.Errorf("utils.ProjectURL() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err == nil && !reflect.DeepEqual(got, tt.want) { - t.Errorf("utils.ProjectURL() = %v, want %v", got, tt.want) - } - }) - } -}