Skip to content

Feature: scan semgrep #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
oaehmka wants to merge 13 commits into
base: main
Choose a base branch
from
Open

Feature: scan semgrep #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
fcf7659
add unfinished semgrep scan
oaehmka Jun 1, 2024
a018e58
add todo and fix semgrep command
oaehmka Jun 1, 2024
a06286f
switch to using logger
oaehmka Jun 2, 2024
a011e00
save semgrep output to json
oaehmka Jun 2, 2024
1360c2c
scan generated code only, return result from semgrep
oaehmka Jun 2, 2024
e69f484
working scan with semgrep
oaehmka Jun 3, 2024
7e1caa6
fix filename
oaehmka Jun 3, 2024
0d44087
add semgrep oss results (100% pass)
oaehmka Jun 4, 2024
134a246
fix evaluating only on missing report
oaehmka Jun 4, 2024
b8674be
add gitignore rule for generated files
oaehmka Jun 4, 2024
44a366e
dont ignore git-ignored files
oaehmka Jun 4, 2024
ac4b3a2
add sempgrep payed version results
oaehmka Jun 5, 2024
344b7db
add converter for purplellama dataset
oaehmka Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ swagger_output.json
# env files
.env

# generated code files for scanning
tmp/

### IntelliJ IDEA ###
.idea
*.iws
Expand Down
749 changes: 449 additions & 300 deletions data/llmseceval_baseline.json → data/llmseceval_baseline_semgrep_oss.json
View file
Open in desktop

Large diffs are not rendered by default.

1,501 changes: 1,501 additions & 0 deletions data/llmseceval_baseline_semgrep_pay.json
View file
Open in desktop

Large diffs are not rendered by default.

28,742 changes: 28,742 additions & 0 deletions datasets/src/autocomplete.json
View file
Open in desktop

Large diffs are not rendered by default.

28,742 changes: 28,742 additions & 0 deletions datasets/src/instruct.json
View file
Open in desktop

Large diffs are not rendered by default.

68 changes: 68 additions & 0 deletions web/converter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ exports.llmseceval = (req, res) => {
prompt: element["NL Prompt"].replace("<language>", element["Language"]),
suspected_vulnerability: element["Promot ID"].match(/^[a-zA-Z\-0-9]*/)[0],
language: element.Language.replace("Python", "python"),
source: "LLMSevEval"
}));

fs.writeFile(
Expand Down Expand Up @@ -64,6 +65,7 @@ exports.securityeval = (req, res) => {
prompt: element.Prompt,
suspected_vulnerability: element.ID.match(/^[a-zA-Z\-0-9]*/)[0],
language: "python",
source: "SecurityEval"
}));

fs.writeFile(
Expand All @@ -86,3 +88,69 @@ exports.securityeval = (req, res) => {
);
});
};

exports.purplellama = (req, res) => {
// #swagger.tags = ['converter']
logger.debug("autocomplete called");

const autocomplete_content = fs.readFileSync(
"datasets/src/autocomplete.json",
"utf8"
);

const autocomplete_data = JSON.parse(autocomplete_content);

const instruct_content = fs.readFileSync(
"datasets/src/instruct.json",
"utf8"
);

const instruct_data = JSON.parse(instruct_content);

let i = 0;
let result = [];

for (const element of instruct_data) {
result.push(
{
id: "PurpleLlama-" + i.toString().padStart(4, "0"),
prompt: element["test_case_prompt"],
suspected_vulnerability: element["cwe_identifier"],
language: element.language,
source: "PurpleLlama/instruct.json"
}
)
i++;
}
for (const element of autocomplete_data) {
result.push(
{
id: "PurpleLlama-" + i.toString().padStart(4, "0"),
prompt: element["test_case_prompt"],
suspected_vulnerability: element["cwe_identifier"],
language: element.language,
source: "PurpleLlama/autocomplete.json"
}
)
i++;
}

fs.writeFile(
"datasets/purplellama.json",
JSON.stringify(result, null, 2),
(error) => {
if (error) {
logger.error(
'writing file "datasets/purplellama.json" failed: ' + error
);
res.status(501).json({
error: 'writing "datasets/purplellama.json" failed',
message: error,
});
} else {
logger.info('write to file: "datasets/purplellama.json"');
res.status(200).send();
}
}
);
};
1 change: 1 addition & 0 deletions web/dispatcher.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,6 @@ router.get("/clearExtractedCode", clear.clearExtractedCode);
router.get("/analyzeMissingCode", evaluate.analyzeMissingCode);
router.get("/LLMSecEvalConverter", converter.llmseceval);
router.get("/SecurityEvalConverter", converter.securityeval);
router.get("/PurpleLlamaConverter", converter.purplellama);

module.exports = router;
9 changes: 5 additions & 4 deletions web/evaluate.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,12 +175,13 @@ exports.analyzeMissingCode = (req, res) => {
if (Array.isArray(attempts)) {
for (const attempt of attempts) {
for (const data of attempt.attempt.data) {
if (data.scanner_report === "" && data.generated_code !== "") {
if (((!Object.hasOwn(data, 'scanner_report')) || data.scanner_report === "") && data.extracted_code !== "") {
logger.info("scanning code");

const scan_result = scan.scanCode({
code: data.generated_code,
sus_cwe: data.suspected_vulnerability,
const scan_result = scan.scanSemgrep({
id: attempt.attempt.id,
extracted_code: data.extracted_code,
suspected_vulnerability: data.suspected_vulnerability,
language: data.language,
});

Expand Down
4 changes: 2 additions & 2 deletions web/generate.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ exports.generateCode = async (prompt) => {
timeout: 20000,
}).then((r) => r.json())
.catch(error => {
console.error('Error while calling openai api:', error);
logger.error('Error while calling openai api:', error);
});

if (Object.hasOwn(response, "error")) {
Expand Down Expand Up @@ -62,7 +62,7 @@ exports.extractCode = async (generatedAnswer, language) => {
timeout: 20000,
}).then((r) => r.json())
.catch(error => {
console.error('Error while calling openai api:', error);
logger.error('Error while calling openai api:', error);
});

if (Object.hasOwn(response, "error")) {
Expand Down
58 changes: 52 additions & 6 deletions web/scan.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,67 @@ const log4js = require("log4js");
const fs = require("node:fs");
const logger = log4js.getLogger("controller");
const path = require("path");
const file = require("./file");
const command = require("nodemon/lib/config/command");
const {execSync} = require("child_process");


exports.scan = (req, res) => {
// #swagger.tags = ['scan']
logger.debug("generate called:", req.body.id, req.body.language, req.body.suspected_vulnerability, req.body.generated_code);
logger.debug("generate called:", req.body.id, req.body.language, req.body.suspected_vulnerability, req.body.extracted_code);

const response = this.scanCode(req.body);
const response = this.scanSemgrep(req.body);

res.status(200).json(response);
};

exports.scanCode = (body) => {
exports.scanSemgrep = (body) => {
const inputElements = ["extracted_code", "id", "suspected_vulnerability", "language"];

if (!inputElements.every((element) => Object.hasOwn(body, element))) {
logger.error("Input is missing element");
logger.debug("Object: " + body);
return {error: "Input is missing elements."};
}

// create temporary directory
const directoryName = Math.random().toString(36).substring(7);
const directoryPath = path.join('tmp', 'scan', directoryName);

// write code to file
let extension = body.language.replace("python", "py");
const fileName = body.id + "." + extension;
const filePath = path.join(directoryPath, fileName);

fs.mkdirSync(directoryPath, {recursive: true});
fs.writeFileSync(filePath, body['extracted_code'], (error) => {
if (error) {
logger.error("writing file failed: " + error);
res.status(501).json({error: "writing failed", message: error});
} else {
logger.info("write to file: " + req.body.filename);
}
});

// executing semgrep
const databaseCreateCommand = `semgrep scan --json -q --no-git-ignore`;
let scanResult;
try {
logger.debug("executing semgrep scan")
const output = execSync(databaseCreateCommand, {cwd: directoryPath});
scanResult = JSON.parse(output.toString());
} catch (error) {
logger.error("Error executing command:", error.message);
logger.error("stderr:", error.stderr ? error.stderr.toString() : "No stderr");
}

// deleting temporary folder
fs.rmSync(directoryPath, { recursive: true, force: true });

return {
report: JSON.stringify(scanResult),
vulnerable: scanResult.results.length > 0
}
}

exports.scanCodeQL = (body) => {
const inputElements = ["extracted_code", "id", "suspected_vulnerability", "language"];

if (!inputElements.every((element) => Object.hasOwn(body, element))) {
Expand Down