Skip to content

chore:SP-3888 Implement multithreading support for the local vulnerab… #30

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
agustingroh merged 1 commit into from
Jan 7, 2026
Merged

chore:SP-3888 Implement multithreading support for the local vulnerab… #30

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [0.8.0] - 2026/01/07
### Added
- Included Exploit Prediction Scoring System (EPSS) to vulnerability response
- Added configurable worker pool for local vulnerability processing (`VULN_SCANOSS_WORKERS`)
### Changed
- Refactored OSV use case
- Refactored local vulnerability use case with multithreading support and context cancellation handling
- Upgraded `scanoss/papi` to v0.28.0

## [0.7.0] - 2025/11/13
Expand Down
4 changes: 3 additions & 1 deletion pkg/config/server_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ type ServerConfig struct {
APIWorkers int `env:"VULN_OSV_API_WORKERS"`
}
SCANOSS struct {
Enabled bool `env:"VULN_SCANOSS_SOURCE_ENABLED"`
Enabled bool `env:"VULN_SCANOSS_SOURCE_ENABLED"`
MaxWorkers int `env:"VULN_SCANOSS_WORKERS"`
}
}
}
Expand Down Expand Up @@ -124,6 +125,7 @@ func setServerConfigDefaults(cfg *ServerConfig) {
cfg.Source.OSV.Enabled = true
cfg.Source.OSV.APIWorkers = 5
cfg.Source.SCANOSS.Enabled = true
cfg.Source.SCANOSS.MaxWorkers = 5
}

func IsValidConfig(cfg *ServerConfig) error {
Expand Down
19 changes: 9 additions & 10 deletions pkg/models/versions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ import (
)

type VersionModel struct {
ctx context.Context
conn *sqlx.Conn
db *sqlx.DB
}

type Version struct {
Expand All @@ -48,18 +47,18 @@ type PurlVersion struct {
// TODO add cache for versions already searched for?

// NewVersionModel creates a new instance of the Version Model.
func NewVersionModel(ctx context.Context, conn *sqlx.Conn) *VersionModel {
return &VersionModel{ctx: ctx, conn: conn}
func NewVersionModel(db *sqlx.DB) *VersionModel {
return &VersionModel{db: db}
}

// GetVersionByName gets the given version from the versions table.
func (m *VersionModel) GetVersionByName(name string, create bool) (Version, error) {
func (m *VersionModel) GetVersionByName(ctx context.Context, name string, create bool) (Version, error) {
if len(name) == 0 {
zlog.S.Error("Please specify a valid Version Name to query")
return Version{}, errors.New("please specify a valid Version Name to query")
}
var version Version
err := m.conn.QueryRowxContext(m.ctx,
err := m.db.QueryRowxContext(ctx,
"SELECT id, version_name, semver FROM versions"+
" WHERE version_name = $1",
name).StructScan(&version)
Expand All @@ -68,28 +67,28 @@ func (m *VersionModel) GetVersionByName(name string, create bool) (Version, erro
return Version{}, fmt.Errorf("failed to query the versions table: %v", err)
}
if create && len(version.VersionName) == 0 { // No version found and requested to create an entry
return m.saveVersion(name)
return m.saveVersion(ctx, name)
}

return version, nil
}

// saveVersion writes the given version name to the versions table.
func (m *VersionModel) saveVersion(name string) (Version, error) {
func (m *VersionModel) saveVersion(ctx context.Context, name string) (Version, error) {
if len(name) == 0 {
zlog.S.Error("Please specify a valid version Name to save")
return Version{}, errors.New("please specify a valid Version Name to save")
}
zlog.S.Debugf("Attempting to save '%v' to the versions table...", name)
var version Version
err := m.conn.QueryRowxContext(m.ctx,
err := m.db.QueryRowxContext(ctx,
"INSERT INTO versions (version_name, semver) VALUES($1, $2)"+
" RETURNING id, version_name, semver",
name, "", false, false,
).StructScan(&version)
Comment on lines +84 to 88
Copy link

@coderabbitai coderabbitai bot Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# First, read the actual file content around the specified lines
cat -n pkg/models/versions.go | sed -n '80,95p'

Repository: scanoss/vulnerabilities

Length of output: 795

🏁 Script executed:

#!/bin/bash
# Search for other INSERT statements into versions table to understand schema
rg -n "INSERT INTO versions" --type=go -B 2 -A 3

Repository: scanoss/vulnerabilities

Length of output: 434

🏁 Script executed:

#!/bin/bash
# Search for versions table schema definition (likely in migrations or schema files)
fd -type f \( -name "*.sql" -o -name "*migration*" -o -name "*schema*" \) | head -20

Repository: scanoss/vulnerabilities

Length of output: 237

🏁 Script executed:

#!/bin/bash
# Search for Version struct definition
rg -n "type Version struct" --type=go -A 10

Repository: scanoss/vulnerabilities

Length of output: 575

🏁 Script executed:

#!/bin/bash
# Search for other methods that interact with versions table
rg -n "versions" pkg/models/versions.go | head -20

Repository: scanoss/vulnerabilities

Length of output: 783

Fix parameter count mismatch in SQL statement.

The INSERT statement has 2 placeholders ($1, $2) but the call passes 4 arguments. The SQL will reject the extra false, false parameters.

Fix 
 	err := m.db.QueryRowxContext(ctx,
 		"INSERT INTO versions (version_name, semver) VALUES($1, $2)"+
 			" RETURNING id, version_name, semver",
-		name, "", false, false,
+		name, "",
 	).StructScan(&version)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
err := m.db.QueryRowxContext(ctx,
"INSERT INTO versions (version_name, semver) VALUES($1, $2)"+
" RETURNING id, version_name, semver",
name, "", false, false,
).StructScan(&version)
err := m.db.QueryRowxContext(ctx,
"INSERT INTO versions (version_name, semver) VALUES($1, $2)"+
" RETURNING id, version_name, semver",
name, "",
).StructScan(&version)
🤖 Prompt for AI Agents 
In @pkg/models/versions.go around lines 84 - 88, The INSERT call in
QueryRowxContext inside pkg/models/versions.go has a parameter count mismatch:
the SQL uses two placeholders ($1, $2) but the call passes four arguments (name,
"", false, false); update the call in the function that builds the version (the
QueryRowxContext(...).StructScan(&version) block) to either remove the extra
false, false arguments so only name and semver ("") are passed, or if additional
columns were intended, update the SQL to include matching placeholders and
column names; ensure the arguments and the INSERT column list/placeholders
align.

if err != nil {
zlog.S.Errorf("Error: Failed to insert new version name into versions table for %v: %v", name, err)
return m.GetVersionByName(name, false) // Search one more time for it, just in case someone else added it
return m.GetVersionByName(ctx, name, false) // Search one more time for it, just in case someone else added it
}
return version, nil
}
25 changes: 10 additions & 15 deletions pkg/models/versions_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,14 @@ func TestVersionsSearch(t *testing.T) {
t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
}
defer CloseDB(db)
conn, err := db.Connx(ctx) // Get a connection from the pool
if err != nil {
t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
}
defer CloseConn(conn)
err = loadTestSQLDataFiles(db, ctx, conn, []string{"../models/tests/versions.sql"})
err = loadTestSQLDataFiles(db, ctx, nil, []string{"../models/tests/versions.sql"})
if err != nil {
t.Fatalf("failed to load SQL test data: %v", err)
}
versionModel := NewVersionModel(ctx, conn)
versionModel := NewVersionModel(db)
var name = "1.0.0"
fmt.Printf("Searching for version: %v\n", name)
version, err := versionModel.GetVersionByName(name, false)
version, err := versionModel.GetVersionByName(ctx, name, false)
if err != nil {
t.Errorf("versions.GetVersionByName() error = %v", err)
}
Expand All @@ -61,15 +56,15 @@ func TestVersionsSearch(t *testing.T) {

name = ""
fmt.Printf("Searching for license: %v\n", name)
_, err = versionModel.GetVersionByName(name, false)
_, err = versionModel.GetVersionByName(ctx, name, false)
if err == nil {
t.Errorf("versions.GetVersionByName() error = did not get an error")
} else {
fmt.Printf("Got expected error = %v\n", err)
}
name = ""
fmt.Printf("Saving for license: %v\n", name)
_, err = versionModel.saveVersion(name)
_, err = versionModel.saveVersion(ctx, name)
if err == nil {
t.Errorf("versions.saveVersion() error = did not get an error")
} else {
Expand All @@ -78,7 +73,7 @@ func TestVersionsSearch(t *testing.T) {

name = "22.22.22"
fmt.Printf("Searching for version: %v\n", name)
version, err = versionModel.GetVersionByName(name, true)
version, err = versionModel.GetVersionByName(ctx, name, true)
if err != nil {
t.Errorf("versions.GetVersionByName() error = %v", err)
}
Expand All @@ -89,7 +84,7 @@ func TestVersionsSearch(t *testing.T) {

name = "22.22.22"
fmt.Printf("Searching for version: %v\n", name)
version, err = versionModel.saveVersion(name)
version, err = versionModel.saveVersion(ctx, name)
if err != nil {
t.Errorf("versions.GetVersionByName() error = %v", err)
}
Expand All @@ -116,14 +111,14 @@ func TestVersionsSearchBadSql(t *testing.T) {
t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
}
defer CloseConn(conn)
versionModel := NewVersionModel(ctx, conn)
_, err = versionModel.GetVersionByName("rubbish", false)
versionModel := NewVersionModel(db)
_, err = versionModel.GetVersionByName(ctx, "rubbish", false)
if err == nil {
t.Errorf("versions.GetVersionByName() error = did not get an error")
} else {
fmt.Printf("Got expected error = %v\n", err)
}
_, err = versionModel.saveVersion("rubbish")
_, err = versionModel.saveVersion(ctx, "rubbish")
if err == nil {
t.Errorf("versions.saveVersion() error = did not get an error")
} else {
Expand Down
21 changes: 10 additions & 11 deletions pkg/models/vulns_purl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ import (
)

type VulnsForPurlModel struct {
ctx context.Context
conn *sqlx.Conn
db *sqlx.DB
}

type VulnsForPurl struct {
Expand All @@ -48,12 +47,12 @@ type OnlyPurl struct {
}

// NewVulnsForPurlModel creates a new instance of the CPE Purl Model.
func NewVulnsForPurlModel(ctx context.Context, conn *sqlx.Conn) *VulnsForPurlModel {
return &VulnsForPurlModel{ctx: ctx, conn: conn}
func NewVulnsForPurlModel(db *sqlx.DB) *VulnsForPurlModel {
return &VulnsForPurlModel{db: db}
}

// GetVulnsByPurl gets vulnerabilities by purl.
func (m *VulnsForPurlModel) GetVulnsByPurl(purl string, version string) ([]VulnsForPurl, error) {
func (m *VulnsForPurlModel) GetVulnsByPurl(ctx context.Context, purl string, version string) ([]VulnsForPurl, error) {
if len(purl) == 0 {
zlog.S.Errorf("Please specify a valid Purl String to query")
return []VulnsForPurl{}, errors.New("please specify a valid Purl String to query")
Expand All @@ -68,21 +67,21 @@ func (m *VulnsForPurlModel) GetVulnsByPurl(purl string, version string) ([]Vulns
purlName := utils.PurlRemoveFromVersionComponent(purl) // Remove everything after the component name

if len(version) > 0 {
return m.GetVulnsByPurlVersion(purlName, version)
return m.GetVulnsByPurlVersion(ctx, purlName, version)
}
return m.GetVulnsByPurlName(purlName)
return m.GetVulnsByPurlName(ctx, purlName)
}

// GetVulnsByPurlName searches for component details of the specified Purl Name/Type (and optional requirement).
func (m *VulnsForPurlModel) GetVulnsByPurlName(purlName string) ([]VulnsForPurl, error) {
func (m *VulnsForPurlModel) GetVulnsByPurlName(ctx context.Context, purlName string) ([]VulnsForPurl, error) {
if len(purlName) == 0 {
zlog.S.Errorf("Please specify a valid Purl Name to query")
return []VulnsForPurl{}, errors.New("please specify a valid Purl Name to query")
}

var vulns []VulnsForPurl
purlName = strings.TrimSpace(purlName)
err := m.conn.SelectContext(m.ctx, &vulns,
err := m.db.SelectContext(ctx, &vulns,
"SELECT c2.cve, c2.severity, c2.published, c2.modified, c2.summary "+
"FROM short_cpe_purl scp "+
"INNER JOIN cpes c ON scp.cpe_id = c.id "+
Expand All @@ -101,7 +100,7 @@ func (m *VulnsForPurlModel) GetVulnsByPurlName(purlName string) ([]VulnsForPurl,
return vulns, nil
}

func (m *VulnsForPurlModel) GetVulnsByPurlVersion(purlName string, purlVersion string) ([]VulnsForPurl, error) {
func (m *VulnsForPurlModel) GetVulnsByPurlVersion(ctx context.Context, purlName string, purlVersion string) ([]VulnsForPurl, error) {
if len(purlName) == 0 {
zlog.S.Errorf("Please specify a valid Purl Name to query")
return []VulnsForPurl{}, errors.New("please specify a valid Purl Name to query")
Expand Down Expand Up @@ -142,7 +141,7 @@ func (m *VulnsForPurlModel) GetVulnsByPurlVersion(purlName string, purlVersion s
WHERE c2.match_criteria_ids && mc.criteria_ids
ORDER BY c2.cve, c2.severity, c2.published, c2.modified, c2.summary;`

err := m.conn.SelectContext(m.ctx, &vulns, query, purlName, purlVersion)
err := m.db.SelectContext(ctx, &vulns, query, purlName, purlVersion)

if err != nil {
zlog.S.Errorf("Failed to query short_cpe for %s: %v", purlName, err)
Expand Down
28 changes: 8 additions & 20 deletions pkg/models/vulns_purl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestGetVulnsByPurl(t *testing.T) {
t.Fatalf("failed to load SQL test data: %v", err)
}

cpeModel := NewVulnsForPurlModel(ctx, conn)
cpeModel := NewVulnsForPurlModel(db)

type inputGetVulnsForPurl struct {
purl string
Expand All @@ -75,7 +75,7 @@ func TestGetVulnsByPurl(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := cpeModel.GetVulnsByPurl(tt.input.purl, tt.input.requirement)
got, err := cpeModel.GetVulnsByPurl(ctx, tt.input.purl, tt.input.requirement)
if (err != nil) != tt.wantErr {
t.Errorf("cpeModel.GetCpeByPurl() error = %v, wantErr %v", err, tt.wantErr)
return
Expand All @@ -102,26 +102,14 @@ func TestGetVulnsByPurlName(t *testing.T) {
}
db.SetMaxOpenConns(1)
defer CloseDB(db)

conn, err := db.Connx(ctx) // Get a connection from the pool
if err != nil {
t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
}
defer CloseConn(conn)
err = LoadTestSQLData(db, ctx, conn)
err = LoadTestSQLData(db, ctx, nil)
if err != nil {
t.Fatalf("failed to load SQL test data: %v", err)
}

cpeModel := NewVulnsForPurlModel(ctx, conn)

_, err = cpeModel.GetVulnsByPurlName("")
if err == nil {
t.Errorf("Error was expected because purl is empty in cpeModel.GetVulnsByPurlName()")
}
cpeModel := NewVulnsForPurlModel(db)

CloseConn(conn)
_, err = cpeModel.GetVulnsByPurlName("pkg:github/hapijs/call")
_, err = cpeModel.GetVulnsByPurlName(ctx, "")
if err == nil {
t.Errorf("Error was expected because purl is empty in cpeModel.GetVulnsByPurlName()")
}
Expand Down Expand Up @@ -151,15 +139,15 @@ func TestGetVulnsByPurlVersion(t *testing.T) {
t.Fatalf("failed to load SQL test data: %v", err)
}

cpeModel := NewVulnsForPurlModel(ctx, conn)
cpeModel := NewVulnsForPurlModel(db)

_, err = cpeModel.GetVulnsByPurlVersion("", "")
_, err = cpeModel.GetVulnsByPurlVersion(ctx, "", "")
if err == nil {
t.Errorf("Error was expected because purl is empty in cpeModel.GetVulnsByPurlVersion()")
}

CloseConn(conn)
_, err = cpeModel.GetVulnsByPurlVersion("pkg:github/hapijs/call", "1.0.0")
_, err = cpeModel.GetVulnsByPurlVersion(ctx, "pkg:github/hapijs/call", "1.0.0")
if err == nil {
t.Errorf("Error was expected because purl is empty in cpeModel.GetVulnsByPurlVersion()")
}
Comment on lines 149 to 153
Copy link

@coderabbitai coderabbitai bot Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Intentional connection close tests error handling.

The test intentionally closes the connection at line 149 before calling GetVulnsByPurlVersion at line 150 to verify error handling. However, since the model now uses the DB pool (m.db) rather than a specific connection, closing a single connection from the pool won't necessarily cause the subsequent call to fail as intended.

The test logic may need adjustment since:

  • The model uses m.db (the connection pool), not a specific connection
  • Closing one connection from the pool doesn't prevent other calls from getting a new connection
  • This test might not actually verify what it intends to verify

Consider either:

  1. Closing the entire DB pool with CloseDB(db) if testing DB closure scenarios, or
  2. Removing this specific test case if it's no longer relevant with the pool-based approach
🤖 Prompt for AI Agents 
In @pkg/models/vulns_purl_test.go around lines 149 - 153, The test currently
closes a single connection via CloseConn(conn) but the model method
GetVulnsByPurlVersion uses the DB pool (m.db), so closing one connection won't
reliably cause an error; either change the test to close the entire pool using
CloseDB(db) before calling GetVulnsByPurlVersion to assert error behavior when
the DB is closed, or remove the test case if simulating a closed DB is no longer
relevant with pool-based access—update references to CloseConn(conn) accordingly
and keep the assertion that expects an error only when you explicitly close the
pool.

Expand Down
8 changes: 4 additions & 4 deletions pkg/usecase/OSV_use_case.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,15 +81,15 @@ func (us OSVUseCase) getOSVRequestsFromDTO(dto []dtos.ComponentDTO) []OSVRequest
return osvRequests
}

func (us OSVUseCase) Execute(dto []dtos.ComponentDTO) dtos.VulnerabilityOutput {
func (us OSVUseCase) Execute(ctx context.Context, dto []dtos.ComponentDTO) dtos.VulnerabilityOutput {
osvRequests := us.getOSVRequestsFromDTO(dto)
return us.processRequests(osvRequests)
return us.processRequests(ctx, osvRequests)
}

func (us OSVUseCase) processRequests(requests []OSVRequest) dtos.VulnerabilityOutput {
func (us OSVUseCase) processRequests(ctx context.Context, requests []OSVRequest) dtos.VulnerabilityOutput {
numJobs := len(requests)
jobs := make(chan OSVRequest, numJobs)
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
results := make(chan dtos.VulnerabilityComponentOutput, numJobs)
workers := min(us.MaxAPIWorkers, numJobs)
Expand Down
2 changes: 1 addition & 1 deletion pkg/usecase/OSV_use_case_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func TestOSVUseCase(t *testing.T) {
OSVUseCase := NewOSVUseCase(s, serverConfig)
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
r := OSVUseCase.Execute(tc.input)
r := OSVUseCase.Execute(ctx, tc.input)
if len(r.Components) == 0 {
t.Errorf("Expected Purls to have elements, got empty slice")
}
Expand Down
Loading
Loading