From 76a7927aa35251522230897971b6f03ca902f808 Mon Sep 17 00:00:00 2001 From: Giuseppe Guerra Date: Fri, 7 Nov 2025 13:58:09 +0100 Subject: [PATCH] feat: add -parallel flag for running analyzers in parallel --- pkg/analysis/analysis.go | 21 +++- .../passes/archivename/archivename.go | 4 +- .../passes/backendbinary/backendbinary.go | 4 +- .../passes/backenddebug/backenddebug.go | 4 +- .../binarypermissions/binarypermissions.go | 4 +- .../passes/brokenlinks/brokenlinks.go | 4 +- pkg/analysis/passes/changelog/changelog.go | 2 +- .../circulardependencies.go | 2 +- pkg/analysis/passes/coderules/coderules.go | 2 +- .../passes/discoverability/discoverability.go | 2 +- pkg/analysis/passes/gomanifest/gomanifest.go | 6 +- pkg/analysis/passes/gosec/gosec.go | 2 +- .../passes/includesnested/includesnested.go | 2 +- pkg/analysis/passes/jargon/jargon.go | 2 +- .../passes/jssourcemap/jssourcemap.go | 4 +- .../passes/legacybuilder/legacybuilder.go | 4 +- .../passes/legacyplatform/legacyplatform.go | 4 +- pkg/analysis/passes/license/license.go | 2 +- pkg/analysis/passes/llmreview/llmreview.go | 2 +- pkg/analysis/passes/logos/logos.go | 2 +- pkg/analysis/passes/manifest/manifest.go | 4 +- pkg/analysis/passes/metadata/metadata.go | 2 +- .../passes/metadatapaths/metadatapaths.go | 4 +- .../passes/metadatavalid/metadatavalid.go | 4 +- pkg/analysis/passes/modulejs/modulejs.go | 2 +- .../passes/nestedmetadata/nestedmetadata.go | 2 +- pkg/analysis/passes/org/org.go | 2 +- pkg/analysis/passes/osvscanner/osvscanner.go | 4 +- .../passes/packagejson/packagejson.go | 4 +- pkg/analysis/passes/pluginname/pluginname.go | 4 +- pkg/analysis/passes/published/published.go | 2 +- pkg/analysis/passes/readme/readme.go | 2 +- .../passes/restrictivedep/restrictivedep.go | 2 +- pkg/analysis/passes/safelinks/safelinks.go | 2 +- .../passes/screenshots/screenshots.go | 6 +- pkg/analysis/passes/sdkusage/sdkusage.go | 4 +- pkg/analysis/passes/signature/signature.go | 6 +- .../passes/sponsorshiplink/sponsorshiplink.go | 2 +- .../passes/templatereadme/templatereadme.go | 2 +- .../passes/trackingscripts/trackingscripts.go | 2 +- pkg/analysis/passes/typesuffix/typesuffix.go | 2 +- pkg/analysis/passes/unsafesvg/unsafesvg.go | 2 +- pkg/analysis/passes/version/version.go | 4 +- pkg/analysis/passes/virusscan/virusscan.go | 4 +- pkg/cmd/plugincheck2/main.go | 9 ++ pkg/runner/resultsbroker.go | 39 ++++++ pkg/runner/runner.go | 119 +++++++++++++++++- 47 files changed, 249 insertions(+), 71 deletions(-) create mode 100644 pkg/runner/resultsbroker.go diff --git a/pkg/analysis/analysis.go b/pkg/analysis/analysis.go index b859d596..87f1d26c 100644 --- a/pkg/analysis/analysis.go +++ b/pkg/analysis/analysis.go @@ -2,6 +2,7 @@ package analysis import ( "fmt" + "sync" "github.com/grafana/plugin-validator/pkg/logme" ) @@ -20,8 +21,22 @@ type Pass struct { AnalyzerName string RootDir string CheckParams CheckParams - ResultOf map[*Analyzer]interface{} - Report func(string, Diagnostic) + // ResultOf is a map[*Analyzer]any that holds the results of all analyzers and is safe for concurrent use. + // The convenience method GetResult can be used to retrieve typed results, or the map can be accessed directly. + ResultOf sync.Map + Report func(string, Diagnostic) +} + +func GetResult[T any](pass *Pass, key *Analyzer) (T, bool) { + anyV, ok := pass.ResultOf.Load(key) + if anyV == nil { + // Special case for nil result. + // Return the zero value of T, otherwise the type assertion below will panic + // due to nil interface{} not being assignable to T. + var zero T + return zero, ok + } + return anyV.(T), ok } type CheckParams struct { @@ -39,6 +54,8 @@ type CheckParams struct { ArchiveCalculatedMD5 string // ArchiveCalculatedSHA1 contains the sha1 checksum calculated from the archive ArchiveCalculatedSHA1 string + + Parallel bool } func (p *Pass) ReportResult(analysisName string, rule *Rule, message string, detail string) { diff --git a/pkg/analysis/passes/archivename/archivename.go b/pkg/analysis/passes/archivename/archivename.go index 9718b87c..e8a38482 100644 --- a/pkg/analysis/passes/archivename/archivename.go +++ b/pkg/analysis/passes/archivename/archivename.go @@ -26,12 +26,12 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/backendbinary/backendbinary.go b/pkg/analysis/passes/backendbinary/backendbinary.go index f0d8628c..73023554 100644 --- a/pkg/analysis/passes/backendbinary/backendbinary.go +++ b/pkg/analysis/passes/backendbinary/backendbinary.go @@ -43,12 +43,12 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/backenddebug/backenddebug.go b/pkg/analysis/passes/backenddebug/backenddebug.go index 5ee98eb7..18404e11 100644 --- a/pkg/analysis/passes/backenddebug/backenddebug.go +++ b/pkg/analysis/passes/backenddebug/backenddebug.go @@ -36,12 +36,12 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/binarypermissions/binarypermissions.go b/pkg/analysis/passes/binarypermissions/binarypermissions.go index 9074f31b..36f03997 100644 --- a/pkg/analysis/passes/binarypermissions/binarypermissions.go +++ b/pkg/analysis/passes/binarypermissions/binarypermissions.go @@ -45,12 +45,12 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/brokenlinks/brokenlinks.go b/pkg/analysis/passes/brokenlinks/brokenlinks.go index 31c2e9e6..adffd239 100644 --- a/pkg/analysis/passes/brokenlinks/brokenlinks.go +++ b/pkg/analysis/passes/brokenlinks/brokenlinks.go @@ -48,12 +48,12 @@ type contextURL struct { } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } - readmeResult, ok := pass.ResultOf[readme.Analyzer].([]byte) + readmeResult, ok := analysis.GetResult[[]byte](pass, readme.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/changelog/changelog.go b/pkg/analysis/passes/changelog/changelog.go index 450d2e2c..66dedb88 100644 --- a/pkg/analysis/passes/changelog/changelog.go +++ b/pkg/analysis/passes/changelog/changelog.go @@ -25,7 +25,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/circulardependencies/circulardependencies.go b/pkg/analysis/passes/circulardependencies/circulardependencies.go index 1c61c647..51164748 100644 --- a/pkg/analysis/passes/circulardependencies/circulardependencies.go +++ b/pkg/analysis/passes/circulardependencies/circulardependencies.go @@ -36,7 +36,7 @@ func run(pass *analysis.Pass) (interface{}, error) { ctx, canc := context.WithTimeout(context.Background(), time.Minute*1) defer canc() - rawMetadataMaps, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + rawMetadataMaps, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/coderules/coderules.go b/pkg/analysis/passes/coderules/coderules.go index f46caa4a..8ce8b7a4 100644 --- a/pkg/analysis/passes/coderules/coderules.go +++ b/pkg/analysis/passes/coderules/coderules.go @@ -49,7 +49,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (any, error) { - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok { // no source code for the validator return nil, nil diff --git a/pkg/analysis/passes/discoverability/discoverability.go b/pkg/analysis/passes/discoverability/discoverability.go index 9b636307..3ca5b011 100644 --- a/pkg/analysis/passes/discoverability/discoverability.go +++ b/pkg/analysis/passes/discoverability/discoverability.go @@ -25,7 +25,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/gomanifest/gomanifest.go b/pkg/analysis/passes/gomanifest/gomanifest.go index b1f76c5e..146da0f0 100644 --- a/pkg/analysis/passes/gomanifest/gomanifest.go +++ b/pkg/analysis/passes/gomanifest/gomanifest.go @@ -44,7 +44,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } @@ -57,11 +57,11 @@ func run(pass *analysis.Pass) (interface{}, error) { return nil, nil } - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, errors.New("archive dir not found") } - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok { // no source code found so we can't check the manifest return nil, nil diff --git a/pkg/analysis/passes/gosec/gosec.go b/pkg/analysis/passes/gosec/gosec.go index 3d3b9b89..d0f5cc09 100644 --- a/pkg/analysis/passes/gosec/gosec.go +++ b/pkg/analysis/passes/gosec/gosec.go @@ -34,7 +34,7 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { // only run if sourcecode.Analyzer succeeded - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/includesnested/includesnested.go b/pkg/analysis/passes/includesnested/includesnested.go index ab84a8f1..22b49f29 100644 --- a/pkg/analysis/passes/includesnested/includesnested.go +++ b/pkg/analysis/passes/includesnested/includesnested.go @@ -45,7 +45,7 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/jargon/jargon.go b/pkg/analysis/passes/jargon/jargon.go index 9e688bbd..56223337 100644 --- a/pkg/analysis/passes/jargon/jargon.go +++ b/pkg/analysis/passes/jargon/jargon.go @@ -30,7 +30,7 @@ func run(pass *analysis.Pass) (interface{}, error) { "nodejs", } - readmeContent, ok := pass.ResultOf[readme.Analyzer].([]byte) + readmeContent, ok := analysis.GetResult[[]byte](pass, readme.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/jssourcemap/jssourcemap.go b/pkg/analysis/passes/jssourcemap/jssourcemap.go index 20b234ff..b6fae910 100644 --- a/pkg/analysis/passes/jssourcemap/jssourcemap.go +++ b/pkg/analysis/passes/jssourcemap/jssourcemap.go @@ -36,12 +36,12 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok || sourceCodeDir == "" { return nil, nil } - archiveFilesPath, ok := pass.ResultOf[archive.Analyzer].(string) + archiveFilesPath, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok || archiveFilesPath == "" { return nil, nil } diff --git a/pkg/analysis/passes/legacybuilder/legacybuilder.go b/pkg/analysis/passes/legacybuilder/legacybuilder.go index beb95000..39a2b6dd 100644 --- a/pkg/analysis/passes/legacybuilder/legacybuilder.go +++ b/pkg/analysis/passes/legacybuilder/legacybuilder.go @@ -26,14 +26,14 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - publishedStatus, ok := pass.ResultOf[published.Analyzer].(*published.PluginStatus) + publishedStatus, ok := analysis.GetResult[*published.PluginStatus](pass, published.Analyzer) // we don't fail published plugins for using toolkit (yet) if ok && publishedStatus.Status != "unknown" { legacyBuilder.Severity = analysis.Warning } - parsedJsonContent, ok := pass.ResultOf[packagejson.Analyzer].(packagejson.PackageJson) + parsedJsonContent, ok := analysis.GetResult[*packagejson.PackageJson](pass, packagejson.Analyzer) if !ok { return nil, nil diff --git a/pkg/analysis/passes/legacyplatform/legacyplatform.go b/pkg/analysis/passes/legacyplatform/legacyplatform.go index 65cce04d..0792806f 100644 --- a/pkg/analysis/passes/legacyplatform/legacyplatform.go +++ b/pkg/analysis/passes/legacyplatform/legacyplatform.go @@ -111,7 +111,7 @@ func fetchDetectors() ([]detector, error) { func run(pass *analysis.Pass) (interface{}, error) { - status, ok := pass.ResultOf[published.Analyzer].(*published.PluginStatus) + status, ok := analysis.GetResult[*published.PluginStatus](pass, published.Analyzer) if !ok { return nil, nil @@ -122,7 +122,7 @@ func run(pass *analysis.Pass) (interface{}, error) { legacyPlatform.Severity = analysis.Warning } - moduleJsMap, ok := pass.ResultOf[modulejs.Analyzer].(map[string][]byte) + moduleJsMap, ok := analysis.GetResult[map[string][]byte](pass, modulejs.Analyzer) if !ok || len(moduleJsMap) == 0 { return nil, nil } diff --git a/pkg/analysis/passes/license/license.go b/pkg/analysis/passes/license/license.go index 04c16f51..60dca394 100644 --- a/pkg/analysis/passes/license/license.go +++ b/pkg/analysis/passes/license/license.go @@ -53,7 +53,7 @@ var validLicensesRegex = []*regexp.Regexp{ const minRequiredConfidenceLevel float32 = 0.9 func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/llmreview/llmreview.go b/pkg/analysis/passes/llmreview/llmreview.go index 6064577c..0355352c 100644 --- a/pkg/analysis/passes/llmreview/llmreview.go +++ b/pkg/analysis/passes/llmreview/llmreview.go @@ -76,7 +76,7 @@ func run(pass *analysis.Pass) (any, error) { var err error // only run if sourcecode.Analyzer succeeded - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/logos/logos.go b/pkg/analysis/passes/logos/logos.go index 1c863b8c..eb5a2cb7 100644 --- a/pkg/analysis/passes/logos/logos.go +++ b/pkg/analysis/passes/logos/logos.go @@ -25,7 +25,7 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/manifest/manifest.go b/pkg/analysis/passes/manifest/manifest.go index 651d0e17..1b9c0a38 100644 --- a/pkg/analysis/passes/manifest/manifest.go +++ b/pkg/analysis/passes/manifest/manifest.go @@ -50,12 +50,12 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } - publishStatus, ok := pass.ResultOf[published.Analyzer].(*published.PluginStatus) + publishStatus, ok := analysis.GetResult[*published.PluginStatus](pass, published.Analyzer) isPublished := false if ok && publishStatus.Status != "unknown" { isPublished = true diff --git a/pkg/analysis/passes/metadata/metadata.go b/pkg/analysis/passes/metadata/metadata.go index cc50652c..cac424a7 100644 --- a/pkg/analysis/passes/metadata/metadata.go +++ b/pkg/analysis/passes/metadata/metadata.go @@ -25,7 +25,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil diff --git a/pkg/analysis/passes/metadatapaths/metadatapaths.go b/pkg/analysis/passes/metadatapaths/metadatapaths.go index 73f97473..6f6ad088 100644 --- a/pkg/analysis/passes/metadatapaths/metadatapaths.go +++ b/pkg/analysis/passes/metadatapaths/metadatapaths.go @@ -54,12 +54,12 @@ type CheckPath struct { func checkMetadataPaths(pass *analysis.Pass) (interface{}, error) { var paths []CheckPath - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/metadatavalid/metadatavalid.go b/pkg/analysis/passes/metadatavalid/metadatavalid.go index 9a741580..ee2855f6 100644 --- a/pkg/analysis/passes/metadatavalid/metadatavalid.go +++ b/pkg/analysis/passes/metadatavalid/metadatavalid.go @@ -33,7 +33,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - schema, ok := pass.ResultOf[metadataschema.Analyzer].([]byte) + schema, ok := analysis.GetResult[[]byte](pass, metadataschema.Analyzer) if !ok { return nil, nil } @@ -55,7 +55,7 @@ func run(pass *analysis.Pass) (interface{}, error) { return nil, err } - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/modulejs/modulejs.go b/pkg/analysis/passes/modulejs/modulejs.go index 1b9b41cf..bdd6e867 100644 --- a/pkg/analysis/passes/modulejs/modulejs.go +++ b/pkg/analysis/passes/modulejs/modulejs.go @@ -25,7 +25,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (any, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok || archiveDir == "" { // this should never happen return nil, nil diff --git a/pkg/analysis/passes/nestedmetadata/nestedmetadata.go b/pkg/analysis/passes/nestedmetadata/nestedmetadata.go index 91936864..25680071 100644 --- a/pkg/analysis/passes/nestedmetadata/nestedmetadata.go +++ b/pkg/analysis/passes/nestedmetadata/nestedmetadata.go @@ -34,7 +34,7 @@ var MainPluginJson = "plugin.json" type Metadatamap map[string]metadata.Metadata func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil diff --git a/pkg/analysis/passes/org/org.go b/pkg/analysis/passes/org/org.go index cd38879b..b3ddbbe6 100644 --- a/pkg/analysis/passes/org/org.go +++ b/pkg/analysis/passes/org/org.go @@ -30,7 +30,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/osvscanner/osvscanner.go b/pkg/analysis/passes/osvscanner/osvscanner.go index 027c3708..1fa33766 100644 --- a/pkg/analysis/passes/osvscanner/osvscanner.go +++ b/pkg/analysis/passes/osvscanner/osvscanner.go @@ -78,11 +78,11 @@ func run(pass *analysis.Pass) (interface{}, error) { return nil, nil } - archiveFilesPath, ok := pass.ResultOf[archive.Analyzer].(string) + archiveFilesPath, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok || archiveFilesPath == "" { return nil, nil } - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok || sourceCodeDir == "" { return nil, nil } diff --git a/pkg/analysis/passes/packagejson/packagejson.go b/pkg/analysis/passes/packagejson/packagejson.go index 70777a6a..f0841f13 100644 --- a/pkg/analysis/passes/packagejson/packagejson.go +++ b/pkg/analysis/passes/packagejson/packagejson.go @@ -31,14 +31,14 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) // we don't fail published plugins for using toolkit (yet) if !ok || sourceCodeDir == "" { return nil, nil } - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/pluginname/pluginname.go b/pkg/analysis/passes/pluginname/pluginname.go index a7ca99ea..bf24a516 100644 --- a/pkg/analysis/passes/pluginname/pluginname.go +++ b/pkg/analysis/passes/pluginname/pluginname.go @@ -26,12 +26,12 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } - publishStatus, ok := pass.ResultOf[published.Analyzer].(*published.PluginStatus) + publishStatus, ok := analysis.GetResult[*published.PluginStatus](pass, published.Analyzer) // we don't check published plugins for naming conventions if ok && publishStatus.Status != "unknown" { diff --git a/pkg/analysis/passes/published/published.go b/pkg/analysis/passes/published/published.go index fda8c72d..d96db3fc 100644 --- a/pkg/analysis/passes/published/published.go +++ b/pkg/analysis/passes/published/published.go @@ -30,7 +30,7 @@ type PluginStatus struct { } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/readme/readme.go b/pkg/analysis/passes/readme/readme.go index 66429da8..b04dcdac 100644 --- a/pkg/analysis/passes/readme/readme.go +++ b/pkg/analysis/passes/readme/readme.go @@ -27,7 +27,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/restrictivedep/restrictivedep.go b/pkg/analysis/passes/restrictivedep/restrictivedep.go index 8260f10a..8e1fa9ec 100644 --- a/pkg/analysis/passes/restrictivedep/restrictivedep.go +++ b/pkg/analysis/passes/restrictivedep/restrictivedep.go @@ -33,7 +33,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - md, ok := pass.ResultOf[metadata.Analyzer].([]byte) + md, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/safelinks/safelinks.go b/pkg/analysis/passes/safelinks/safelinks.go index 618945f4..3fd34e4c 100644 --- a/pkg/analysis/passes/safelinks/safelinks.go +++ b/pkg/analysis/passes/safelinks/safelinks.go @@ -72,7 +72,7 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { webriskApiKey := os.Getenv("WEBRISK_API_KEY") - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/screenshots/screenshots.go b/pkg/analysis/passes/screenshots/screenshots.go index f519f042..e4a1e2ad 100644 --- a/pkg/analysis/passes/screenshots/screenshots.go +++ b/pkg/analysis/passes/screenshots/screenshots.go @@ -36,17 +36,17 @@ var svgImage = "image/svg+xml" var acceptedImageTypes = []string{"image/jpeg", "image/png", "image/gif", svgImage} func checkScreenshots(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } // Ensure metadatavalid.Analyzer ran (it returns nil but we need it as dependency) - _, ok = pass.ResultOf[metadatavalid.Analyzer] + _, ok = pass.ResultOf.Load(metadatavalid.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/sdkusage/sdkusage.go b/pkg/analysis/passes/sdkusage/sdkusage.go index 77f54b90..c48792c2 100644 --- a/pkg/analysis/passes/sdkusage/sdkusage.go +++ b/pkg/analysis/passes/sdkusage/sdkusage.go @@ -52,13 +52,13 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok { // no source code found so we can't go.mod return nil, nil } - metadatamap, ok := pass.ResultOf[nestedmetadata.Analyzer].(nestedmetadata.Metadatamap) + metadatamap, ok := analysis.GetResult[nestedmetadata.Metadatamap](pass, nestedmetadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/signature/signature.go b/pkg/analysis/passes/signature/signature.go index 942d1112..caa856e7 100644 --- a/pkg/analysis/passes/signature/signature.go +++ b/pkg/analysis/passes/signature/signature.go @@ -47,17 +47,17 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } - md, ok := pass.ResultOf[metadata.Analyzer].([]byte) + md, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } - mf, ok := pass.ResultOf[manifest.Analyzer].([]byte) + mf, ok := analysis.GetResult[[]byte](pass, manifest.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/sponsorshiplink/sponsorshiplink.go b/pkg/analysis/passes/sponsorshiplink/sponsorshiplink.go index a176d014..30c31d5d 100644 --- a/pkg/analysis/passes/sponsorshiplink/sponsorshiplink.go +++ b/pkg/analysis/passes/sponsorshiplink/sponsorshiplink.go @@ -26,7 +26,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/templatereadme/templatereadme.go b/pkg/analysis/passes/templatereadme/templatereadme.go index 5659352f..e55e5871 100644 --- a/pkg/analysis/passes/templatereadme/templatereadme.go +++ b/pkg/analysis/passes/templatereadme/templatereadme.go @@ -23,7 +23,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - readmeResult, ok := pass.ResultOf[readme.Analyzer].([]byte) + readmeResult, ok := analysis.GetResult[[]byte](pass, readme.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/trackingscripts/trackingscripts.go b/pkg/analysis/passes/trackingscripts/trackingscripts.go index 84b30cd6..e7adb143 100644 --- a/pkg/analysis/passes/trackingscripts/trackingscripts.go +++ b/pkg/analysis/passes/trackingscripts/trackingscripts.go @@ -30,7 +30,7 @@ var Analyzer = &analysis.Analyzer{ func run(pass *analysis.Pass) (interface{}, error) { - moduleJsMap, ok := pass.ResultOf[modulejs.Analyzer].(map[string][]byte) + moduleJsMap, ok := analysis.GetResult[map[string][]byte](pass, modulejs.Analyzer) if !ok || len(moduleJsMap) == 0 { return nil, nil } diff --git a/pkg/analysis/passes/typesuffix/typesuffix.go b/pkg/analysis/passes/typesuffix/typesuffix.go index b135018b..2e6a070e 100644 --- a/pkg/analysis/passes/typesuffix/typesuffix.go +++ b/pkg/analysis/passes/typesuffix/typesuffix.go @@ -24,7 +24,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/unsafesvg/unsafesvg.go b/pkg/analysis/passes/unsafesvg/unsafesvg.go index 469bea5c..06d76ba3 100644 --- a/pkg/analysis/passes/unsafesvg/unsafesvg.go +++ b/pkg/analysis/passes/unsafesvg/unsafesvg.go @@ -27,7 +27,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil } diff --git a/pkg/analysis/passes/version/version.go b/pkg/analysis/passes/version/version.go index 383129c8..c256afc8 100644 --- a/pkg/analysis/passes/version/version.go +++ b/pkg/analysis/passes/version/version.go @@ -27,7 +27,7 @@ var Analyzer = &analysis.Analyzer{ } func run(pass *analysis.Pass) (interface{}, error) { - metadataBody, ok := pass.ResultOf[metadata.Analyzer].([]byte) + metadataBody, ok := analysis.GetResult[[]byte](pass, metadata.Analyzer) if !ok { return nil, nil } @@ -48,7 +48,7 @@ func run(pass *analysis.Pass) (interface{}, error) { return nil, nil } - pluginStatus, ok := pass.ResultOf[published.Analyzer].(*published.PluginStatus) + pluginStatus, ok := analysis.GetResult[*published.PluginStatus](pass, published.Analyzer) if !ok { // in case of any error getting the online status, skip this check return nil, nil diff --git a/pkg/analysis/passes/virusscan/virusscan.go b/pkg/analysis/passes/virusscan/virusscan.go index d5f63f89..77b10e62 100644 --- a/pkg/analysis/passes/virusscan/virusscan.go +++ b/pkg/analysis/passes/virusscan/virusscan.go @@ -71,7 +71,7 @@ func run(pass *analysis.Pass) (interface{}, error) { } // scan the archive - archiveDir, ok := pass.ResultOf[archive.Analyzer].(string) + archiveDir, ok := analysis.GetResult[string](pass, archive.Analyzer) if !ok { return nil, nil @@ -86,7 +86,7 @@ func run(pass *analysis.Pass) (interface{}, error) { } // scan the source code - sourceCodeDir, ok := pass.ResultOf[sourcecode.Analyzer].(string) + sourceCodeDir, ok := analysis.GetResult[string](pass, sourcecode.Analyzer) if !ok { // no source code found so we can't scan return nil, nil diff --git a/pkg/cmd/plugincheck2/main.go b/pkg/cmd/plugincheck2/main.go index 1ed8f1ea..73399108 100644 --- a/pkg/cmd/plugincheck2/main.go +++ b/pkg/cmd/plugincheck2/main.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/bmatcuk/doublestar/v4" "gopkg.in/yaml.v3" @@ -66,6 +67,11 @@ func main() { false, "If set, outputs results in GitHub Actions format regardless of config file setting", ) + parallelFlag = flag.Bool( + "parallel", + false, + "Run the analyzers in parallel", + ) ) flag.Parse() @@ -159,6 +165,7 @@ func main() { } } + st := time.Now() diags, err := runner.Check( analyzers, analysis.CheckParams{ @@ -169,6 +176,7 @@ func main() { Checksum: *checksum, ArchiveCalculatedMD5: fmt.Sprintf("%x", md5hash), ArchiveCalculatedSHA1: fmt.Sprintf("%x", sha1hash), + Parallel: *parallelFlag, }, cfg, severity, @@ -177,6 +185,7 @@ func main() { // we don't exit on error. we want to still report the diagnostics logme.DebugFln("check failed: %v", err) } + logme.DebugFln("analyzers run in %+v", time.Since(st)) var outputMarshaler output.Marshaler diff --git a/pkg/runner/resultsbroker.go b/pkg/runner/resultsbroker.go new file mode 100644 index 00000000..0647a1ba --- /dev/null +++ b/pkg/runner/resultsbroker.go @@ -0,0 +1,39 @@ +package runner + +import ( + "sync" + + "github.com/grafana/plugin-validator/pkg/analysis" + "github.com/grafana/plugin-validator/pkg/logme" +) + +type resultsBroker struct { + mux sync.RWMutex + subscribers map[*analysis.Analyzer][]chan any +} + +func newResultsBroker() *resultsBroker { + return &resultsBroker{ + subscribers: make(map[*analysis.Analyzer][]chan any), + } +} + +func (r *resultsBroker) publish(analyzer *analysis.Analyzer, result any) { + r.mux.RLock() + defer r.mux.RUnlock() + logme.DebugFln("publishing result for analyzer %q to %d subscribers", analyzer.Name, len(r.subscribers[analyzer])) + for _, sub := range r.subscribers[analyzer] { + sub <- result + } +} + +func (r *resultsBroker) subscribe(analyzer *analysis.Analyzer) <-chan any { + r.mux.Lock() + defer r.mux.Unlock() + ch := make(chan any) + if _, ok := r.subscribers[analyzer]; !ok { + r.subscribers[analyzer] = []chan any{} + } + r.subscribers[analyzer] = append(r.subscribers[analyzer], ch) + return ch +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index e5b3d419..147d21fd 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -1,7 +1,10 @@ package runner import ( + "errors" "fmt" + "sync" + "time" "github.com/grafana/plugin-validator/pkg/analysis" "github.com/grafana/plugin-validator/pkg/logme" @@ -49,12 +52,123 @@ func Check( } initAnalyzers(analyzers, &cfg, pluginId, severityOverwrite) + if !params.Parallel { + logme.Debugln("running in sequential mode") + return checkSequential(analyzers, params) + } + logme.DebugFln("running in parallel mode") + return checkParallel(analyzers, params) +} + +func checkParallel(analyzers []*analysis.Analyzer, params analysis.CheckParams) (analysis.Diagnostics, error) { + // TODO: func because it's the same when sequential + diagnostics := make(analysis.Diagnostics) + var diagnosticsMux sync.Mutex + pass := &analysis.Pass{ + RootDir: params.ArchiveDir, + CheckParams: params, + ResultOf: sync.Map{}, + Report: func(name string, d analysis.Diagnostic) { + // Collect all diagnostics for presenting at the end. + diagnosticsMux.Lock() + defer diagnosticsMux.Unlock() + diagnostics[name] = append(diagnostics[name], d) + }, + } + + var seen sync.Map + + broker := newResultsBroker() + ready := make(chan struct{}, len(analyzers)) + errs := make(chan error, len(analyzers)) + for _, currentAnalyzer := range analyzers { + currentAnalyzer := currentAnalyzer + // Subscribe for all dependencies + depsChs := make([]<-chan any, 0, len(currentAnalyzer.Requires)) + for _, dep := range currentAnalyzer.Requires { + depsChs = append(depsChs, broker.subscribe(dep)) + } + + // Start goroutine that will run the analyzer + go func() { + // Wait for all analyzers to be ready (all pubsub dependency requests done) + <-ready + + // Wait for all dependencies to run + tickerQuit := make(chan struct{}) + go func() { + ticker := time.NewTicker(time.Second * 30) + defer ticker.Stop() + for { + select { + case <-ticker.C: + logme.DebugFln("analyzer %s: waiting for dependencies", currentAnalyzer.Name) + case <-tickerQuit: + return + } + } + }() + var wg sync.WaitGroup + wg.Add(len(depsChs)) + for _, depCh := range depsChs { + go func() { + defer wg.Done() + <-depCh + }() + } + tickerQuit <- struct{}{} + close(tickerQuit) + logme.DebugFln("analyzer %s: all dependencies done", currentAnalyzer.Name) + + // Do not run the same analyzer twice + if _, ok := seen.Load(currentAnalyzer); ok { + // Always return nil error to the main goroutine + logme.DebugFln("analyzer %s: analyzer already run", currentAnalyzer.Name) + errs <- nil + return + } + seen.Store(currentAnalyzer, true) + logme.DebugFln("Running analyzer %s", currentAnalyzer.Name) + + // Run the analyzer + // TODO: concurrent??? + pass.AnalyzerName = currentAnalyzer.Name + // TODO: ensure no concurrent access in analyzers + res, err := currentAnalyzer.Run(pass) + // Publish the result to all subscribers (dependent analyzers) + defer func() { + logme.DebugFln("analyzer %s: publishing result", currentAnalyzer.Name) + broker.publish(currentAnalyzer, res) + }() + if err != nil { + errs <- err + return + } + pass.ResultOf.Store(currentAnalyzer, res) + errs <- nil + }() + } + + // Signal all goroutines that subscription is done and analyzers are ready to run + for i := 0; i < len(analyzers); i++ { + ready <- struct{}{} + } + + // Await all errors from all goroutines and return combined error + var finalErr error + for i := 0; i < len(analyzers); i++ { + errors.Join(finalErr, <-errs) + } + return diagnostics, finalErr +} + +func checkSequential(analyzers []*analysis.Analyzer, params analysis.CheckParams) (analysis.Diagnostics, error) { diagnostics := make(analysis.Diagnostics) pass := &analysis.Pass{ RootDir: params.ArchiveDir, CheckParams: params, - ResultOf: make(map[*analysis.Analyzer]interface{}), + ResultOf: sync.Map{}, Report: func(name string, d analysis.Diagnostic) { // Collect all diagnostics for presenting at the end. diagnostics[name] = append(diagnostics[name], d) @@ -88,7 +202,7 @@ func Check( if err != nil { return err } - pass.ResultOf[currentAnalyzer] = res + pass.ResultOf.Store(currentAnalyzer, res) return nil } @@ -99,7 +213,6 @@ func Check( return diagnostics, err } } - return diagnostics, nil }