From 720b09617be8f10abe9b8cd8bef6e6e7b2258054 Mon Sep 17 00:00:00 2001 From: Aleksei Bekh-Ivanov Date: Thu, 11 Jul 2019 18:12:55 +0200 Subject: [PATCH 1/5] Add module configuration extendability --- .hawkeyerc | 13 +++++++ bin/hawkeye-modules | 17 ++++---- bin/hawkeye-modules-config | 41 ++++++++++++++++++++ bin/hawkeye-modules-ls | 13 +++++++ lib/modules/files-secrets/config-schema.json | 26 +++++++++++++ lib/modules/files-secrets/index.js | 7 +++- lib/rc.js | 9 ++++- lib/scan.js | 36 ++++++++++++++++- package.json | 4 +- 9 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 .hawkeyerc create mode 100755 bin/hawkeye-modules-config create mode 100755 bin/hawkeye-modules-ls create mode 100644 lib/modules/files-secrets/config-schema.json diff --git a/.hawkeyerc b/.hawkeyerc new file mode 100644 index 00000000..2ef84c73 --- /dev/null +++ b/.hawkeyerc @@ -0,0 +1,13 @@ +{ + "all": true, + "modules": ["files-secrets"], + "failOn": "low", + "showCode": false, + "moduleConfig": { + "files-secrets": { + "files": [ + "patterns.json" + ] + } + } +} diff --git a/bin/hawkeye-modules b/bin/hawkeye-modules index 74198fb2..bef0a9a5 100755 --- a/bin/hawkeye-modules +++ b/bin/hawkeye-modules @@ -2,12 +2,11 @@ 'use strict' -const logger = require('../lib/logger') -const modules = require('../lib/modules') -require('colors') - -logger.log('Module Status'.bold) -modules().forEach(m => { - logger.log(`${m.enabled ? 'Enabled: '.green : 'Disabled: '.red} ${m.key.bold}`) - logger.log(' ' + m.description) -}) +const program = require('commander') +program + .command('config ', 'Module configuration documentation') + .command('ls', 'Lists the currently install modules', {isDefault: true}) + .parse(process.argv) + + + diff --git a/bin/hawkeye-modules-config b/bin/hawkeye-modules-config new file mode 100755 index 00000000..c363bb6f --- /dev/null +++ b/bin/hawkeye-modules-config @@ -0,0 +1,41 @@ +#!/usr/bin/env node + +'use strict' + +const logger = require('../lib/logger') +const modules = require('../lib/modules') +require('colors') +const program = require('commander') + +program + .arguments('') + .action(printModuleConfigurationDocumentation); + +program.parse(process.argv); + +function printModuleConfigurationDocumentation(moduleKey) { + console.log(`${moduleKey} configuration:`.bold) + console.log('') + + let module = modules().filter(m => m.key === moduleKey)[0]; + const schema = module.configSchema; + for (const [k, v] of Object.entries(schema.properties)) { + console.log(`${k.bold} (${v.type}): ${v.title}`) + } + + const exmapleConfig = { + moduleConfig:{ + } + } + + exmapleConfig.moduleConfig[module.key] = schema.examples[0]; + + console.log('') + console.log('Example .hawkeyerc:'.bold) + + let example = JSON.stringify(exmapleConfig, undefined, 2).padStart(4); + console.log(example) + + +} + diff --git a/bin/hawkeye-modules-ls b/bin/hawkeye-modules-ls new file mode 100755 index 00000000..74198fb2 --- /dev/null +++ b/bin/hawkeye-modules-ls @@ -0,0 +1,13 @@ +#!/usr/bin/env node + +'use strict' + +const logger = require('../lib/logger') +const modules = require('../lib/modules') +require('colors') + +logger.log('Module Status'.bold) +modules().forEach(m => { + logger.log(`${m.enabled ? 'Enabled: '.green : 'Disabled: '.red} ${m.key.bold}`) + logger.log(' ' + m.description) +}) diff --git a/lib/modules/files-secrets/config-schema.json b/lib/modules/files-secrets/config-schema.json new file mode 100644 index 00000000..b2b82af2 --- /dev/null +++ b/lib/modules/files-secrets/config-schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Schema of Hawkeye module files-secrets", + "properties": { + "files": { + "type": "array", + "title": "List of files containing patterns of suspicious filenames", + "default": [], + "items": { + "type": "string", + "title": "Path to file", + "examples": [ + "path/to/patterns.json" + ] + } + } + }, + "examples": [ + { + "files": [ + "path/to/patterns.json" + ] + } + ] +} diff --git a/lib/modules/files-secrets/index.js b/lib/modules/files-secrets/index.js index 002c11ec..f909c55e 100644 --- a/lib/modules/files-secrets/index.js +++ b/lib/modules/files-secrets/index.js @@ -9,8 +9,11 @@ module.exports = { key, description: 'Scans for suspicious filenames that are likely to contain secrets', enabled: true, - handles: () => true, - run: fm => { + handles: (fm, config) => true, + configSchema: require('./config-schema.json'), + run: (fm, config) => { + // TODO Access `config.files` here to read custom patterns' definitions + const results = new ModuleResults(key) const checkers = items.map(item => { diff --git a/lib/rc.js b/lib/rc.js index 447d33c5..21807d97 100644 --- a/lib/rc.js +++ b/lib/rc.js @@ -17,6 +17,7 @@ module.exports = class RC { this.all = false this.staged = false this.showCode = false + this.moduleConfig = {} this.writers = [consoleWriter] } @@ -95,7 +96,8 @@ module.exports = class RC { http: this.withHttp.bind(this), json: this.withJson.bind(this), failOn: this.withFailOn.bind(this), - showCode: this.withShowCode.bind(this) + showCode: this.withShowCode.bind(this), + moduleConfig: this.withModuleConfig.bind(this) } Object.keys(hawkeyerc).forEach(key => { const handler = handlers[key] @@ -149,4 +151,9 @@ module.exports = class RC { this.writers.push({ ...httpWriter, opts: { url } }) return this } + + withModuleConfig (config) { + this.moduleConfig = config || {} + return this + } } diff --git a/lib/scan.js b/lib/scan.js index 3c3468c8..7cb691b3 100644 --- a/lib/scan.js +++ b/lib/scan.js @@ -4,6 +4,7 @@ const modules = require('./modules') const logger = require('./logger') const _ = require('lodash') require('colors') +const Ajv = require('ajv') module.exports = async (rc = {}) => { logger.log('Target for scan:', rc.target) @@ -26,9 +27,14 @@ module.exports = async (rc = {}) => { const activeModules = [] const inactiveModules = [] + knownModules + .filter(canBeConfigured) + .filter(m => rc.moduleConfig.hasOwnProperty(m.key)) + .forEach(m => validateAndNormalizeModuleConfig(m, rc.moduleConfig[m.key])) + for (const module of knownModules) { logger.log(`Checking ${module.key} for applicability`) - const isActive = await module.handles(fm) + const isActive = await module.handles(fm, rc.moduleConfig[module.key]) ;(isActive ? activeModules : inactiveModules).push(module) } inactiveModules.forEach(module => logger.log('Skipping module'.bold, module.key)) @@ -41,7 +47,7 @@ module.exports = async (rc = {}) => { .reduce((prom, { key, run }) => prom.then(async allRes => { logger.log('Running module'.bold, key) try { - const res = await run(fm) + const res = await run(fm, rc.moduleConfig[module.key]) return allRes.concat(res) } catch (e) { logger.error(key, 'returned an error!', e.message) @@ -73,3 +79,29 @@ module.exports = async (rc = {}) => { return results.length ? 1 : 0 } + +function canBeConfigured (module) { + return !!module.configSchema +} + +/** + * Not only validates the config, but also removes fields that are not defined in schema and + * puts default value in if property is undefined + */ +function validateAndNormalizeModuleConfig (module, config) { + const ajv = new Ajv({ + strictDefaults: true, + strictKeywords: true, + removeAdditional: true, + useDefaults: true + }) + if (!module.configSchema.hasOwnProperty('additionalProperties')) { + module.configSchema.additionalProperties = false + } + + const validate = ajv.compile(module.configSchema) + const valid = validate(config) + if (!valid) { + throw new Error(`Config for module '${module.key}' is invalid:\n` + validate.errors.map(JSON.stringify).join('\n')) + } +} diff --git a/package.json b/package.json index a86d4b06..c13baf55 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "watch": "nodemon --watch . --exec 'npm test'", "test": "npx npm-run-all test:lint test:unit", "test:lint": "npx standard", + "test:lint:fix": "npx standard --fix", "test:unit": "NODE_ENV=testing npx mocha 'lib/**/*-unit.js' -R 'spec' testutils.js" }, "bin": { @@ -56,7 +57,8 @@ "semver": "^6.1.1", "superagent": "^5.0.5", "tmp": "^0.1.0", - "xml2js": "^0.4.19" + "xml2js": "^0.4.19", + "ajv": "^6.10.0" }, "devDependencies": { "chai": "^4.2.0", From 68a8f366dee7f51a1bdcc5ea682880faa991a69c Mon Sep 17 00:00:00 2001 From: Aleksei Bekh-Ivanov Date: Wed, 17 Jul 2019 15:49:46 +0200 Subject: [PATCH 2/5] Add generic module configuration schema tests --- lib/__tests__/modules-unit.js | 56 +++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/lib/__tests__/modules-unit.js b/lib/__tests__/modules-unit.js index 799c3788..4a5c2a51 100644 --- a/lib/__tests__/modules-unit.js +++ b/lib/__tests__/modules-unit.js @@ -1,14 +1,66 @@ 'use strict' +/* eslint-disable no-unused-expressions */ + +const Ajv = require('ajv') + const modules = require('../modules') describe('Modules', () => { modules().forEach(module => { describe(module.key, () => { - it('should have the module signature', () => { + it('should have required properties', () => { let expectMethods = ['key', 'description', 'handles', 'run', 'enabled'].sort() - expect(Object.keys(module).sort()).to.deep.equal(expectMethods) + expect(module).to.include.all.keys(expectMethods) }) }) }) }) + +describe('Configurable Modules', () => { + modules() + .filter(m => m.configSchema) + .forEach(module => { + describe(`${module.key} config schema`, () => { + it('should be a valid JSON Schema', () => { + const ajv = new Ajv({ + strictDefaults: true, + strictKeywords: true + }) + + expect(ajv.validateSchema(module.configSchema)).to.be.true + }) + + it('should expect an object', () => { + expect(module.configSchema.type).to.be.equal('object') + }) + + it('should have at least one example', () => { + expect(module.configSchema.examples).to.have.lengthOf.at.least(1) + }) + + it('should have valid examples', () => { + const ajv = new Ajv({ + strictDefaults: true, + strictKeywords: true, + removeAdditional: true + }) + + if (!module.configSchema.hasOwnProperty('additionalProperties')) { + module.configSchema.additionalProperties = false + } + + module.configSchema.examples.forEach((e, i) => { + const validate = ajv.compile(module.configSchema) + expect(validate(e), `Example #${i}`).to.be.true + }) + }) + + it('should have some title for root level properties', () => { + for (const [property, definition] of Object.entries(module.configSchema.properties)) { + expect(definition.title, property).to.not.be.empty + } + }) + }) + }) +}) From c56b1649264ca3c87e20ba068e64cf5e1597f5a5 Mon Sep 17 00:00:00 2001 From: Aleksei Bekh-Ivanov Date: Wed, 17 Jul 2019 15:55:18 +0200 Subject: [PATCH 3/5] Add test that empty object is a valid config for every module Every module shoud have some defaults --- lib/__tests__/modules-unit.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/__tests__/modules-unit.js b/lib/__tests__/modules-unit.js index 4a5c2a51..7211043a 100644 --- a/lib/__tests__/modules-unit.js +++ b/lib/__tests__/modules-unit.js @@ -61,6 +61,22 @@ describe('Configurable Modules', () => { expect(definition.title, property).to.not.be.empty } }) + + it('should accept empty object as valid config', () => { + const ajv = new Ajv({ + strictDefaults: true, + strictKeywords: true, + removeAdditional: true, + useDefaults: true + }) + + if (!module.configSchema.hasOwnProperty('additionalProperties')) { + module.configSchema.additionalProperties = false + } + + const validate = ajv.compile(module.configSchema) + expect(validate({})).to.be.true + }) }) }) }) From fdd4c18c385b8b006d6f80478c4b5de29ac1fcdd Mon Sep 17 00:00:00 2001 From: Aleksei Bekh-Ivanov Date: Wed, 17 Jul 2019 16:09:37 +0200 Subject: [PATCH 4/5] Improve module configuration help output --- bin/hawkeye-modules-config | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/bin/hawkeye-modules-config b/bin/hawkeye-modules-config index c363bb6f..2fcc17ef 100755 --- a/bin/hawkeye-modules-config +++ b/bin/hawkeye-modules-config @@ -2,7 +2,6 @@ 'use strict' -const logger = require('../lib/logger') const modules = require('../lib/modules') require('colors') const program = require('commander') @@ -14,28 +13,31 @@ program program.parse(process.argv); function printModuleConfigurationDocumentation(moduleKey) { - console.log(`${moduleKey} configuration:`.bold) + console.log(`${moduleKey} module configuration properties:`.bold) console.log('') let module = modules().filter(m => m.key === moduleKey)[0]; const schema = module.configSchema; for (const [k, v] of Object.entries(schema.properties)) { - console.log(`${k.bold} (${v.type}): ${v.title}`) + let type = v.type; + if (type === 'array') { + type = v.items.type + '[]' + } + console.log(` ${k.bold} (${type}): ${v.title}`) } - const exmapleConfig = { + const exampleConfig = { moduleConfig:{ } } - exmapleConfig.moduleConfig[module.key] = schema.examples[0]; - console.log('') - console.log('Example .hawkeyerc:'.bold) - - let example = JSON.stringify(exmapleConfig, undefined, 2).padStart(4); - console.log(example) - - + console.log('.hawkeyerc examples:'.bold) + schema.examples.forEach((e) => { + exampleConfig.moduleConfig[module.key] = e; + let example = JSON.stringify(exampleConfig, undefined, 2).padStart(4); + console.log(example) + console.log('') + }) } From 0d3ccaa5547570084168ba7cd72c48b2ba39f40a Mon Sep 17 00:00:00 2001 From: Aleksei Bekh-Ivanov Date: Wed, 17 Jul 2019 16:15:21 +0200 Subject: [PATCH 5/5] Enforce type definition of root level properties --- lib/__tests__/modules-unit.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/__tests__/modules-unit.js b/lib/__tests__/modules-unit.js index 7211043a..21a1733f 100644 --- a/lib/__tests__/modules-unit.js +++ b/lib/__tests__/modules-unit.js @@ -62,6 +62,16 @@ describe('Configurable Modules', () => { } }) + it('should have some type defined for root level properties', () => { + for (const [property, definition] of Object.entries(module.configSchema.properties)) { + expect(definition.type, property).to.not.be.empty + + if (definition.type === 'array') { + expect(definition.items.type, `${property} items`).to.not.be.empty + } + } + }) + it('should accept empty object as valid config', () => { const ajv = new Ajv({ strictDefaults: true,