diff --git a/.dotstop.dot b/.dotstop.dot index 1cf5c087b4..291ebccc95 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -19,7 +19,6 @@ digraph G { "AOU-15" [sha=f4d2fdcacc3c3bd743af3ff5c2a08ceb650e59015ed846c35ddb4de104e80cec]; "AOU-16" [sha="4e08bac839cba01a4dc2a794bd2d1254d388945c6b79293684e00b9f8d03f31e"]; "AOU-17" [sha=ce26eef503179bbedca706f43fedc289d8c56580c4b885b3abab440d2de25870]; -"AOU-18" [sha="672193d261861effb34f845d6e3421f47ce85e79d7ef34f62ce5c10bef68e796"]; "AOU-19" [sha="877c346448621a81b2d1418e610575bfcd556d2f56a14f23f289b90828907928"]; "AOU-20" [sha="d17ea5c6a3f07b516abc7c51d4b15b1a130a28602fbd4eb6872006a2e5f939d0"]; "AOU-21" [sha="d1712fcf7a0a00659021a69784f697b5d7b404aeadb7be66553218ba2e6c0bf7"]; @@ -91,6 +90,7 @@ digraph G { "JLS-64" [sha="40f1382c156e308ee543c30df4dc7eb457ac14d472909c30eb6caae9a3bc1d68"]; "JLS-65" [sha="e413de6c831c1c019c67c3e3477b9dc9302cc79433ec894beaee0c95e053b545"]; "JLS-66" [sha="cf57eaf55654ef52589b1879c7294de13ddf1258ecdff4f6371178c6e8e6975b"]; +"JLS-74" [sha="c161214f0f206f3c0826750978fcc4c99e2765a0c3333592e1293b323434ca34"]; "NJF-01" [sha="548dc86014e093974f68660942daa231271496a471885bbed092a375b3079bd8"]; "NJF-02" [sha="6ea015646d696e3f014390ff41612eab66ac940f20cf27ce933cbadf8482d526"]; "NJF-03" [sha="4bd1f8210b7bba9a248055a437f377d9da0b7576c5e3ed053606cf8b5b2febe3"]; @@ -400,6 +400,9 @@ digraph G { "PJD-03" -> "PJD-02" [sha="34c3d9163590aec428486eef28fa3895a3c425b2ceb826d3c1d592d34eefe14e"]; "TA-ANALYSIS" -> "JLS-17" [sha=eac76c7d417f623d9dd1a220fc0b3ec6e2c2b9e16781eefae6d21dbd35d1402d]; "TA-ANALYSIS" -> "JLS-26" [sha="250c8c20314a242c1daac1278ef5a96d97bce8d943318430a91ee30580b5b29d"]; +"TA-ANALYSIS" -> "JLS-74" [sha="438058a274824cc8774588099abf500dcac8fa0215c486f79a43e00c6baa66a4"]; +"TA-ANALYSIS" -> "JLS-31" [sha="80b0be4318282eee6799e35ef618a077539ec0d4c783db2e50e68d00e9085018"]; +"TA-ANALYSIS" -> "JLS-27" [sha="cb77a6a19e50c33a85cc9b099e4401321f7460a9a6afef31bd2b9da29fe063af"]; "TA-BEHAVIOURS" -> "JLEX-01" [sha="8cd931ef61b7012140344adf54469e943bfc690ee54f12db12777464880061db"]; "TA-BEHAVIOURS" -> "JLEX-02" [sha=cb26451e31a56b1eb51a4d45283ba4a7c6e898efbd045b59cba10d3c6aa093de]; "TA-BEHAVIOURS" -> "JLS-03" [sha=cf9211c07452914cb2d0b455f859b26cb2724423eae5187e8cbfdff06d1b5ba3]; @@ -423,7 +426,6 @@ digraph G { "TA-CONSTRAINTS" -> "AOU-09" [sha="b47001a26392bd151e4a03feccb198c74fdfbdfaeb055fd682b59e6b92f0bed5"]; "TA-CONSTRAINTS" -> "AOU-15" [sha="936c697c984cfadd07db6e2eef5a3f5f540c57d099c7a773c00e551ea15bdf7a"]; "TA-CONSTRAINTS" -> "AOU-17" [sha="b9866189c44d25116cc7cd47432546f8ea2b3aee0b493b364a89872e11957a69"]; -"TA-CONSTRAINTS" -> "AOU-18" [sha="ca8930b0d06ea3987a85112508f49cfaf1cf77d9a405535f3173975175f15d78"]; "TA-CONSTRAINTS" -> "AOU-19" [sha="ef2ca2a552493258538bf72c975d4c0ec711cea193adb804765feea86382ec0a"]; "TA-CONSTRAINTS" -> "AOU-22" [sha="54608fda93406fa3546c2d38ef09d34f7252e2c40bb6bc982decf762ab1c68f0"]; "TA-CONSTRAINTS" -> "AOU-23" [sha="6a9aa9e8b107c684e68b21b44c363185c5f0668c39307a00758781d65f3e61d4"]; diff --git a/.dotstop_extensions/README.md b/.dotstop_extensions/README.md index 818acc446e..1aea0422f4 100644 --- a/.dotstop_extensions/README.md +++ b/.dotstop_extensions/README.md @@ -410,10 +410,10 @@ Note: Starting with trudag v2025.09.16, "Commit date/time" is a unix timestamp ( ## push -This functionality writes the generated data into an sqlite database `TrustableScoring.db` located in the folder `TSF`. This database contains two tables, `commit_info`, where the metadata of "info" are stored, and `scores`, where the scores are stored, and which references `commit_info` via the date as foreign key. +This functionality writes the generated data into an sqlite database stored under TSF/data_storage/ (e.g. TrustableScoring_YYYY-MM-DD.db). The file path can be configured via the environment variable TSF_SCORING_DB. This database contains two tables, `commit_info`, where the metadata of "info" are stored, and `scores`, where the scores are stored, and which references `commit_info` via the date as foreign key. It is intended to store data only once per commit. If, for any reason, the same commit generates data more than once, then only the most recent data are stored, and the obsolete data are deleted. This still ensures that the scoring history of the main branch is as complete as possible. ## pull -This functionality parses the information stored in `TrustableScoring.db` into the format which is expected by trudag. In case that no data is found, the empty history is returned. \ No newline at end of file +This functionality parses the information stored in the scoring database configured via TSF_SCORING_DB (stored under TSF/data_storage/) into the format which is expected by trudag. In case that no data is found, the empty history is returned. \ No newline at end of file diff --git a/.dotstop_extensions/data_store.py b/.dotstop_extensions/data_store.py index 335ac3a707..b5d22a7a83 100644 --- a/.dotstop_extensions/data_store.py +++ b/.dotstop_extensions/data_store.py @@ -3,7 +3,7 @@ import os # global variable -- path to persistent data storage -persistent_storage = "TSF/TrustableScoring.db" +persistent_storage = os.getenv("TSF_SCORING_DB", "TSF/TrustableScoring.db") def data_store_pull() -> list[dict]: data = get_my_data() diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index 0f4f538d99..8065ed2320 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -121,17 +121,58 @@ jobs: git commit -m "Updated issues list" || echo "No changes to commit" git push origin save_historical_data && git pull - - name: Load persistent data + - name: Select persistent scoring DB (rotate if near GitHub limit) + shell: bash run: | - if ! git ls-tree --name-only origin/save_historical_data TSF/TrustableScoring.db | grep TSF/TrustableScoring.db; then - mkdir -p TSF - touch TSF/TrustableScoring.db - git add TSF/TrustableScoring.db - git commit -m "Initialise persistent data storage" - git push origin save_historical_data + set -euo pipefail + + DB_DIR="TSF/data_storage" + DB_PREFIX="TrustableScoring" + LIMIT_BYTES=$((90 * 1024 * 1024)) # 90 MiB + today=$(date -u +%F) + + echo "[info] Rotation threshold: ${LIMIT_BYTES} bytes" + echo "[info] Today (UTC): ${today}" + + # newest dated DB in storage branch + latest="$(git ls-tree -r --name-only origin/save_historical_data "$DB_DIR" \ + | grep -E "^${DB_DIR}/${DB_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}(_[0-9]{3})?\.db$" \ + | sort | tail -n 1 || true)" + + mkdir -p "$DB_DIR" + + if [ -z "${latest}" ]; then + latest="${DB_DIR}/${DB_PREFIX}_${today}.db" + sqlite3 "$latest" "PRAGMA user_version = 1;" + echo "[info] No existing scoring DB -> created: $latest" + echo "TSF_SCORING_DB=${latest}" >> "$GITHUB_ENV" + exit 0 fi - git checkout $branch_name - git checkout save_historical_data -- TSF/TrustableScoring.db + + git checkout save_historical_data -- "$latest" + size=$(stat -c%s "$latest") + echo "[info] Current scoring DB: $latest (${size} bytes)" + + if [ "$size" -ge "$LIMIT_BYTES" ]; then + base="${DB_DIR}/${DB_PREFIX}_${today}.db" + new="$base" + if [ -e "$new" ]; then + i=1 + while [ -e "${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" ]; do + i=$((i+1)) + done + new="${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" + fi + sqlite3 "$new" "PRAGMA user_version = 1;" + latest="$new" + echo "[info] Rotation triggered -> new DB: $latest" + else + echo "[info] Rotation not needed." + fi + + echo "TSF_SCORING_DB=${latest}" >> "$GITHUB_ENV" + echo "[info] Using scoring DB: $latest" + - name: Generate trudag report run: | @@ -144,13 +185,14 @@ jobs: - name: Checkout data storage branch run: | - git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -v TSF/TrustableScoring.db) + git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -vE '^TSF/data_storage/TrustableScoring.*\.db$) git checkout save_historical_data git pull - name: Store persistent data run: | - git add TSF/TrustableScoring.db + echo "[info] Committing scoring DB: $TSF_SCORING_DB" + git add "$TSF_SCORING_DB" git commit -m "Append data storage" || echo "Historical data already up to date." git push origin save_historical_data diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 34c6ecc7b0..97ae14055b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -348,25 +348,75 @@ jobs: git fetch --all git checkout save_historical_data && git pull || git checkout -b save_historical_data - - name: Load persistent data + - name: Select persistent DB (rotate if near GitHub limit) + shell: bash run: | - if ! git ls-tree --name-only origin/save_historical_data TSF/MemoryEfficientTestResultData.db | grep TSF/MemoryEfficientTestResultData.db; then - touch TSF/MemoryEfficientTestResultData.db - git add TSF/MemoryEfficientTestResultData.db - git commit -m "Create persistent storage" - git push origin save_historical_data + set -euo pipefail + + # Ensure sqlite3 is available + if ! command -v sqlite3 >/dev/null 2>&1; then + echo "[info] sqlite3 not found -> installing" + sudo apt-get update + sudo apt-get install -y sqlite3 fi - # Handle fork PRs by staying on current working branch instead of switching - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - # For PRs, we're already on the right commit from actions/checkout - # Just get the file from save_historical_data branch - git checkout save_historical_data -- TSF/MemoryEfficientTestResultData.db + + DB_DIR="TSF/data_storage" + DB_PREFIX="MemoryEfficientTestResultData" + LIMIT_BYTES=$((90 * 1024 * 1024)) # rotate at 90 MB + today=$(date -u +%F) + + echo "[info] Event: ${GITHUB_EVENT_NAME:-}, Ref: ${GITHUB_REF_NAME:-}, SHA: ${GITHUB_SHA:-}" + echo "[info] Storage branch: origin/save_historical_data" + echo "[info] Rotation threshold: ${LIMIT_BYTES} bytes (~90 MB)" + echo "[info] Today (UTC): ${today}" + + # Find newest DB in storage branch (ISO dates sort correctly) + candidates="$(git ls-tree -r --name-only origin/save_historical_data "$DB_DIR" \ + | grep -E "^${DB_DIR}/${DB_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}(_[0-9]{3})?\.db$" \ + | sort || true)" + + latest="$(printf "%s\n" "${candidates}" | tail -n 1 || true)" + + mkdir -p "$DB_DIR" + + if [ -z "${latest}" ]; then + new="${DB_DIR}/${DB_PREFIX}_${today}.db" + sqlite3 "$new" "PRAGMA user_version = 1;" + echo "TSF_PERSIST_DB=${new}" >> "$GITHUB_ENV" + echo "[info] No existing DB found -> created initial DB: ${new}" + exit 0 + fi + + echo "[info] Latest DB in storage: ${latest}" + git checkout save_historical_data -- "$latest" + + size="$(stat -c%s "$latest")" + echo "[info] Latest DB size: ${size} bytes" + + # Rotate if size too large + if [ "$size" -ge "$LIMIT_BYTES" ]; then + base="${DB_DIR}/${DB_PREFIX}_${today}.db" + new="$base" + + if [ -e "$new" ]; then + i=1 + while [ -e "${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" ]; do + i=$((i+1)) + done + new="${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" + fi + + sqlite3 "$new" "PRAGMA user_version = 1;" + latest="$new" + echo "[info] Rotation triggered -> new DB: ${latest}" else - # For push events, switch to the branch and get the file - git checkout $branch_name - git checkout save_historical_data -- TSF/MemoryEfficientTestResultData.db + echo "[info] Rotation not needed." fi + echo "TSF_PERSIST_DB=${latest}" >> "$GITHUB_ENV" + echo "[info] Using persistent DB: ${latest}" + + - name: append test data run: | python3 TSF/scripts/capture_test_data_memory_sensitive.py successful @@ -374,13 +424,13 @@ jobs: - name: Checkout data storage branch if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref_name == 'main') run: | - git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -v TSF/MemoryEfficientTestResultData.db) + git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -vE '^TSF/data_storage/MemoryEfficientTestResultData_.*\.db$') git checkout save_historical_data - name: Commit and Push persistent data if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref_name == 'main') run: | - git add TSF/MemoryEfficientTestResultData.db + git add "$TSF_PERSIST_DB" git commit -m "Save test data" || echo "No test data to be saved" git push origin save_historical_data @@ -448,25 +498,74 @@ jobs: git fetch --all git checkout save_historical_data && git pull || git checkout -b save_historical_data - - name: Load persistent data + - name: Select persistent DB (rotate if near GitHub limit) + shell: bash run: | - if ! git ls-tree --name-only origin/save_historical_data TSF/MemoryEfficientTestResultData.db | grep TSF/MemoryEfficientTestResultData.db; then - touch TSF/MemoryEfficientTestResultData.db - git add TSF/MemoryEfficientTestResultData.db - git commit -m "Create persistent storage" - git push origin save_historical_data + set -euo pipefail + + # Ensure sqlite3 is available + if ! command -v sqlite3 >/dev/null 2>&1; then + echo "[info] sqlite3 not found -> installing" + sudo apt-get update + sudo apt-get install -y sqlite3 fi - # Handle fork PRs by staying on current working branch instead of switching - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - # For PRs, we're already on the right commit from actions/checkout - # Just get the file from save_historical_data branch - git checkout save_historical_data -- TSF/MemoryEfficientTestResultData.db + + DB_DIR="TSF/data_storage" + DB_PREFIX="MemoryEfficientTestResultData" + LIMIT_BYTES=$((90 * 1024 * 1024)) # rotate at 90 MB + today=$(date -u +%F) + + echo "[info] Event: ${GITHUB_EVENT_NAME:-}, Ref: ${GITHUB_REF_NAME:-}, SHA: ${GITHUB_SHA:-}" + echo "[info] Storage branch: origin/save_historical_data" + echo "[info] Rotation threshold: ${LIMIT_BYTES} bytes (~90 MB)" + echo "[info] Today (UTC): ${today}" + + # Find newest DB in storage branch (ISO dates sort correctly) + candidates="$(git ls-tree -r --name-only origin/save_historical_data "$DB_DIR" \ + | grep -E "^${DB_DIR}/${DB_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}(_[0-9]{3})?\.db$" \ + | sort || true)" + + latest="$(printf "%s\n" "${candidates}" | tail -n 1 || true)" + + mkdir -p "$DB_DIR" + + if [ -z "${latest}" ]; then + new="${DB_DIR}/${DB_PREFIX}_${today}.db" + sqlite3 "$new" "PRAGMA user_version = 1;" + echo "TSF_PERSIST_DB=${new}" >> "$GITHUB_ENV" + echo "[info] No existing DB found -> created initial DB: ${new}" + exit 0 + fi + + echo "[info] Latest DB in storage: ${latest}" + git checkout save_historical_data -- "$latest" + + size="$(stat -c%s "$latest")" + echo "[info] Latest DB size: ${size} bytes" + + # Rotate if size too large + if [ "$size" -ge "$LIMIT_BYTES" ]; then + base="${DB_DIR}/${DB_PREFIX}_${today}.db" + new="$base" + + if [ -e "$new" ]; then + i=1 + while [ -e "${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" ]; do + i=$((i+1)) + done + new="${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" + fi + + sqlite3 "$new" "PRAGMA user_version = 1;" + latest="$new" + echo "[info] Rotation triggered -> new DB: ${latest}" else - # For push events, switch to the branch and get the file - git checkout $branch_name - git checkout save_historical_data -- TSF/MemoryEfficientTestResultData.db + echo "[info] Rotation not needed." fi + echo "TSF_PERSIST_DB=${latest}" >> "$GITHUB_ENV" + echo "[info] Using persistent DB: ${latest}" + - name: append test data run: | python3 TSF/scripts/capture_test_data_memory_sensitive.py failed @@ -474,13 +573,13 @@ jobs: - name: Checkout data storage branch if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref_name == 'main') run: | - git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -v TSF/MemoryEfficientTestResultData.db) + git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -vE '^TSF/data_storage/MemoryEfficientTestResultData_.*\.db$') git checkout save_historical_data - name: Commit and Push persistent data if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref_name == 'main') run: | - git add TSF/MemoryEfficientTestResultData.db + git add "$TSF_PERSIST_DB" git commit -m "Save test data" || echo "No test data to be saved" git push origin save_historical_data @@ -548,25 +647,74 @@ jobs: git fetch --all git checkout save_historical_data && git pull || git checkout -b save_historical_data - - name: Load persistent data + - name: Select persistent DB (rotate if near GitHub limit) + shell: bash run: | - if ! git ls-tree --name-only origin/save_historical_data TSF/MemoryEfficientTestResultData.db | grep TSF/MemoryEfficientTestResultData.db; then - touch TSF/MemoryEfficientTestResultData.db - git add TSF/MemoryEfficientTestResultData.db - git commit -m "Create persistent storage" - git push origin save_historical_data + set -euo pipefail + + # Ensure sqlite3 is available + if ! command -v sqlite3 >/dev/null 2>&1; then + echo "[info] sqlite3 not found -> installing" + sudo apt-get update + sudo apt-get install -y sqlite3 fi - # Handle fork PRs by staying on current working branch instead of switching - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - # For PRs, we're already on the right commit from actions/checkout - # Just get the file from save_historical_data branch - git checkout save_historical_data -- TSF/MemoryEfficientTestResultData.db + + DB_DIR="TSF/data_storage" + DB_PREFIX="MemoryEfficientTestResultData" + LIMIT_BYTES=$((90 * 1024 * 1024)) # rotate at 90 MB + today=$(date -u +%F) + + echo "[info] Event: ${GITHUB_EVENT_NAME:-}, Ref: ${GITHUB_REF_NAME:-}, SHA: ${GITHUB_SHA:-}" + echo "[info] Storage branch: origin/save_historical_data" + echo "[info] Rotation threshold: ${LIMIT_BYTES} bytes (~90 MB)" + echo "[info] Today (UTC): ${today}" + + # Find newest DB in storage branch (ISO dates sort correctly) + candidates="$(git ls-tree -r --name-only origin/save_historical_data "$DB_DIR" \ + | grep -E "^${DB_DIR}/${DB_PREFIX}_[0-9]{4}-[0-9]{2}-[0-9]{2}(_[0-9]{3})?\.db$" \ + | sort || true)" + + latest="$(printf "%s\n" "${candidates}" | tail -n 1 || true)" + + mkdir -p "$DB_DIR" + + if [ -z "${latest}" ]; then + new="${DB_DIR}/${DB_PREFIX}_${today}.db" + sqlite3 "$new" "PRAGMA user_version = 1;" + echo "TSF_PERSIST_DB=${new}" >> "$GITHUB_ENV" + echo "[info] No existing DB found -> created initial DB: ${new}" + exit 0 + fi + + echo "[info] Latest DB in storage: ${latest}" + git checkout save_historical_data -- "$latest" + + size="$(stat -c%s "$latest")" + echo "[info] Latest DB size: ${size} bytes" + + # Rotate if size too large + if [ "$size" -ge "$LIMIT_BYTES" ]; then + base="${DB_DIR}/${DB_PREFIX}_${today}.db" + new="$base" + + if [ -e "$new" ]; then + i=1 + while [ -e "${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" ]; do + i=$((i+1)) + done + new="${DB_DIR}/${DB_PREFIX}_${today}_$(printf '%03d' "$i").db" + fi + + sqlite3 "$new" "PRAGMA user_version = 1;" + latest="$new" + echo "[info] Rotation triggered -> new DB: ${latest}" else - # For push events, switch to the branch and get the file - git checkout $branch_name - git checkout save_historical_data -- TSF/MemoryEfficientTestResultData.db + echo "[info] Rotation not needed." fi + echo "TSF_PERSIST_DB=${latest}" >> "$GITHUB_ENV" + echo "[info] Using persistent DB: ${latest}" + - name: append test data run: | python3 TSF/scripts/capture_test_data_memory_sensitive.py cancelled @@ -574,13 +722,13 @@ jobs: - name: Checkout data storage branch if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref_name == 'main') run: | - git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -v TSF/MemoryEfficientTestResultData.db) + git stash push --include-untracked -- $(git status --porcelain | awk '{print $2}' | grep -vE '^TSF/data_storage/MemoryEfficientTestResultData_.*\.db$') git checkout save_historical_data - name: Commit and Push persistent data if: github.event_name == 'schedule' || (github.event_name == 'push' && github.ref_name == 'main') run: | - git add TSF/MemoryEfficientTestResultData.db + git add "$TSF_PERSIST_DB" git commit -m "Save test data" || echo "No test data to be saved" git push origin save_historical_data diff --git a/TSF/docs/ci_failure_rate_analysis.md b/TSF/docs/ci_failure_rate_analysis.md index 0c945140fc..5bfc377820 100644 --- a/TSF/docs/ci_failure_rate_analysis.md +++ b/TSF/docs/ci_failure_rate_analysis.md @@ -11,6 +11,16 @@ GitHub’s “failure rate” counts all non-successful job runs, including canc jobs (e.g. quickly closed PRs) and infrastructure/tooling problems, not only true regressions in the JSON library. +In this document we distinguish between: +- (a) **test-result failures/regressions** (unit/integration tests failing), and +- (b) **CI/environment/infrastructure failures** (runner/tooling/network issues, publishing steps, label synchronization, etc.). + +**Methodology note:** For the jobs with non-zero failure rates discussed below, +we inspected logs of failed workflow runs to determine whether failures +originated from failing unit/integration tests (test-result failures/regressions) +or from CI/environment/tooling steps. Only the former is treated as +behaviour-related evidence. + ## Jobs with the highest reported failure rates From the Jobs table (sorted by Failure rate): @@ -74,6 +84,16 @@ GitHub’s “failure rate” counts all non-successful job runs, including canc jobs (e.g. quickly closed PRs) and infrastructure/tooling problems, not only true regressions in the JSON library. +In this document we distinguish between: +- (a) **test-result failures/regressions** (unit/integration tests failing), and +- (b) **CI/environment/infrastructure failures** (runner/tooling/network issues, publishing steps, label synchronization, etc.). + +**Methodology note:** For the jobs with non-zero failure rates discussed below, +we inspected logs of failed workflow runs to determine whether failures +originated from failing unit/integration tests (test-result failures/regressions) +or from CI/environment/tooling steps. Only the former is treated as +behaviour-related evidence. + ## Jobs with the highest reported failure rates From the Jobs table (sorted by Failure rate): @@ -90,7 +110,7 @@ All other Parent-Workflow jobs in the last 90 days report a failure rate of The job `publish_test_data_success` is the final step of the Ubuntu child workflow that publishes the persistent test-result database -(`TSF/MemoryEfficientTestResultData.db`) back to the `save_historical_data` +(`TSF/data_storage/MemoryEfficientTestResultData*.db`) back to the `save_historical_data` branch. The observed 5.26 % failure rate corresponds to 2 failed runs out of 38, and these failures are limited to this publishing step (e.g. git push / branch / permission issues) rather than to the execution of the tests @@ -102,6 +122,9 @@ defaults. Its 2.63 % failure rate corresponds to 1 failed run out of 38, and is related to repository/label management rather than to building or testing the JSON library. +No high failure rates in this period were caused by failing unit/integration tests. All +observed high failure rates were confined to publishing or repository-management steps. + ## Conclusion Taken together, a period of there months show that all test-related @@ -110,4 +133,3 @@ failure rates are confined to meta-jobs that handle publishing of historical test data and label synchronization. This indicates a stable CI setup for `inc_nlohmann_json`, with the only reported failures occurring in infrastructure-side integration steps rather than in the core test pipeline. - diff --git a/TSF/scripts/capture_test_data_memory_sensitive.py b/TSF/scripts/capture_test_data_memory_sensitive.py index 260299e41a..e2423303cf 100644 --- a/TSF/scripts/capture_test_data_memory_sensitive.py +++ b/TSF/scripts/capture_test_data_memory_sensitive.py @@ -144,7 +144,9 @@ def find_most_recent_results(target: str, name: str, compiler: str, cpp_standard # Step 1: store metadata of workflow run persistently # initiate connection to database - connector = sqlite3.connect("TSF/MemoryEfficientTestResultData.db") + persist_db = os.environ.get("TSF_PERSIST_DB", "TSF/MemoryEfficientTestResultData.db") + connector = sqlite3.connect(persist_db) + connector.execute("PRAGMA foreign_keys = ON") # load expected tables diff --git a/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md b/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md index 4ca97050fb..24d2a44437 100644 --- a/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-ANALYSIS_CONTEXT.md @@ -43,9 +43,9 @@ by confirming that test results remain unchanged when no changes are intended. **Evidence** - Analysis of test data, including thresholds in relation to appropriate statistical properties. - - **Answer**: The analysis of test data is captured by JLS-17, but no explicit quantitative thresholds are defined. + - **Answer**: The analysis of test data includes CI failure-rate/trend analysis (JLS-17), quantitative CI thresholds such as the lcov coverage gate (see JLS-54), and checking that expectations are supported by tests via Trustable traceability (supporting statements under JLEX-01/02 reference CI tests and are validated by SME reviewers during scoring, see JLS-74). In addition, CI runs include automated static analysis and sanitizers, which provide further evidence by detecting issues (see, JLS-31). - Analysis of failures - - **Answer**: Provided by JLS-26 and JLS-17. + - **Answer**: Provided by JLS-26, JLS-17 and JLS-31. - Analysis of spikes and trends - **Answer**: CI test failure rates for the upstream `nlohmann/json` repository and `eclipse-score/inc_nlohmann_json` are analysed using the time-series based GitHub Actions metrics views. This analysis is performed manually (see JLS-17). There is currently no fully automated, continuous analysis of failures. - Validation of analysis methods used @@ -59,11 +59,11 @@ that may indicate problems in development, test, or production. **CHECKLIST** - What fraction of Expectations are covered by the test data? - - **Answer**: The two expectations are JLEX-01 and JLEX-02. Every statement supporting either of these expectations is ultimately supported by a test, except for WFJ-06. WFJ-06 specifies that `basic_json::accept` must accept exactly JSON values for all possible inputs. Since there are infinitely many possible inputs, this cannot be tested exhaustively. Indirect tests are provided by the rejection of ill-formed json data. + - **Answer**: The two expectations are JLEX-01 and JLEX-02. Every statement supporting either of these expectations is ultimately supported by a test, except for WFJ-06. WFJ-06 specifies that `basic_json::accept` must accept exactly JSON values for all possible inputs. Since there are infinitely many possible inputs, this cannot be tested exhaustively. Indirect tests are provided by the rejection of ill-formed json data. This traceability is established by requiring each supporting statement under JLEX-01/02 to reference the relevant CI test(s), and the suitability of the referenced tests as evidence is validated during SME review as part of the scoring process (see JLS-74). - What fraction of Misbehaviours are covered by the monitored indicator data? - - **Answer**: Currently none, because there is no implemented monitoring of deployed instances yet. This is a future integrator responsibility (see AOU-09, AOU-18 and AOU-19). + - **Answer**: Currently there is no indicators implemented, that focus on runtime behavior. The only indicators implemented are a coverage gate and PR count gate that are both part of the CI. The data therefore is available via the GitHub actions history. - How confident are we that the indicator data are accurate and timely? - - **Answer**: No indicator data is collected (see the previous question). + - **Answer**: See the previous question. Since we just implemented a coverage gate and PR count gate as general indicators, that data is produced automatically by the CI and therefore is generated consistently for every run. We are confident that the values are timely, as they are updated on each CI execution, and accurate to the extent that GitHub Actions reflects the executed workflows and their recorded outputs. - How reliable is the monitoring process? - **Answer**: See the previous question. - How well does the production data correlate with our test data? diff --git a/TSF/trustable/assertions/TA-DATA_CONTEXT.md b/TSF/trustable/assertions/TA-DATA_CONTEXT.md index 3bdddf0685..3b0bc734c1 100644 --- a/TSF/trustable/assertions/TA-DATA_CONTEXT.md +++ b/TSF/trustable/assertions/TA-DATA_CONTEXT.md @@ -27,13 +27,13 @@ Appropriate storage strategies safeguard availability across the product lifecyc - Time-stamped and traceable result records for each test execution, linked to associated system under test version and specification references. - **Answer**: Provided by JLS-18 and JLS-45. - List of monitored indicators, linked to associated specification version references. - - **Answer**: Runtime monitoring indicators for deployed instances are not yet implemented and are expected to be defined by the integrator (see AOU-09, AOU-18 and AOU-19). + - **Answer**: For eclipse-score/inc_nlohmann_json, no runtime monitoring indicators are defined because the component is a statically integrated, header-only library without long-running runtime behaviour in this repository context. The monitored indicators that are currently specified and collected are CI-/process-based: Coverage threshold gate (see JLS-54) and PR-count / review-load limit gate (see JLS-55). - Time-stamped and traceable test-derived data for each indicator, linked to associated system under test version and indicator specifications references. - - **Answer**: Indicator-level data is not yet collected and should be done by the integrator (see AOU-09). + - **Answer**: The CI collects time-stamped indicator data and links it to the tested commit SHA. The indicator specifications are referenced in JLS-54 (coverage gate) and JLS-55 (PR-count gate). - List of monitored deployments, linked to associated version and configuration references. - - **Answer**: Not available. Monitoring of deployed instances should be specified by the integrator (see AOU-09, AOU-18 and AOU-19). + - **Answer**: Monitoring is performed via CI runs (coverage gate and PR-count gate) and is traceable to the tested commit SHA and the CI workflow configuration. There is no separate monitoring of production deployments in this repository context. - Time-stamped and traceable production data for each indicator, linked to associated deployment metadata and specification references. - - **Answer**: Not available. Should be done by the integrator (see AOU-09, AOU-18 and AOU-19). + - **Answer**: Not available. No production/runtime monitoring data is collected, only CI-derived, time-stamped indicator data is available via the CI artefacts and run history. **Confidence scoring** @@ -44,22 +44,22 @@ monitored deployments. **Checklist** - Is all test data stored with long-term accessibility? - - **Answer**: No. Test results are collected into a persistent database as part of the CI workflows, which is done as a proof of concept. Due to GitHub storage constraints, the storage of this database is currently limited. + - **Answer**: Yes, the test results are collected into a persistent database as part of the CI workflows and pushed to the save_historical_data branch. To avoid hitting GitHub file size limits, the persistent database is automatically rotated into date-stamped files, while older files remain available for long-term access. - Is all monitoring data stored with long-term accessibility? - - **Answer**: No. Dedicated monitoring data from deployed software is not collected yet. But it is expected to be implemented by the integrator (see AOU-09, AOU-18 and AOU-19). + - **Answer**: Monitoring data is currently collected via the CI and stored with long-term accessibility in the persistent CI data store on the save_historical_data branch. However, there is still no dedicated monitoring for runtime behaviour (and related aspects), so this part of monitoring data is not collected yet. - Are extensible data models implemented? - **Answer**: Test-result data is stored in a SQLite database with separate tables for workflow runs and individual test results (see JLS-18). This schema can be extended with additional fields or tables if needed. - Is sensitive data handled correctly (broadcasted, stored, discarded, or anonymised) with appropriate encryption and redundancy? - **Answer**: This is not explicitly applicable. The captured test data does not include personal or sensitive data. - Are proper backup mechanisms in place? - - **Answer**: No explicit project-level backup mechanism is defined for the test results database beyond GitHub’s own infrastructure. + - **Answer**: No explicit project-level backup mechanism is defined beyond GitHub’s own infrastructure. The persistent test/scoring databases are stored and versioned on the save_historical_data branch, which provides history and recoverability via Git, but there is no separate off-platform backup process in place. - Are storage and backup limits tested? - - **Answer**: The `capture_test_data_memory_sensitive.py` script enforces size limits for the persistent database and fails the workflow if they are exceeded. There is no backup mechanism. + - **Answer**: Storage limits are addressed in CI by checking the size of the persistent databases and rotating to a new date-stamped database file once a threshold is reached, to avoid hitting GitHub file size limits. There is no separate backup mechanism beyond GitHub/Git history. - Are all data changes traceable? - - **Answer**: Yes, for test data. Updates to `TSF/MemoryEfficientTestResultData.db` are performed by CI workflows and committed to the `save_historical_data` branch, so Git history records each change. + - **Answer**: Yes, for both test and scoring data. Updates to the persistent databases (e.g. TSF/persistent/MemoryEfficientTestResultData_*.db and TSF/persistent/TrustableScoring_*.db) are performed by CI workflows and committed to the save_historical_data branch, so Git history records each change. - Are concurrent changes correctly managed and resolved? - **Answer**: Largely yes for test data. The ubuntu workflow uses a concurrency group that cancels in-progress runs for the same reference, so typically only one job updates the persistent database at a time and remaining conflicts would surface as failed pushes and require manual resolution. - Is data accessible only to intended parties? - **Answer**: Since the library is open source, there are no unintended parties. - Are any subsets of our data being published? - - **Answer**: Yes, as a proof of concept, CI test result data is committed to the `save_historical_data` branch in the SQLite database `TSF/MemoryEfficientTestResultData.db`, which is publicly accessible via this GitHub repository. + - **Answer**: Yes, as a proof of concept, CI test result data is committed to the `save_historical_data` branch in the SQLite database `TSF/data_storage/MemoryEfficientTestResultData*.db`, which is publicly accessible via this GitHub repository. diff --git a/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md b/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md index 71fcc7932d..22df475105 100644 --- a/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md +++ b/TSF/trustable/assertions/TA-VALIDATION_CONTEXT.md @@ -58,11 +58,11 @@ results for all expected tests (both pass / fail and performance). - How many pass/fail results would be expected, based on the scheduled tests? - **Answer**: All scheduled unit and integration tests are expected to pass (zero failures). - Do we have all of the expected results? - - **Answer**: Yes, for the selected workflows we typically have results for each run, potential gaps arise when CI runs are skipped or cancelled, or due to storge limitation. + - **Answer**: Yes, for the selected workflows we typically have results for each run, potential gaps arise when CI runs are skipped or cancelled, or due to storage limitation. - Do we have time-series data for all of those results? - - **Answer**: Stored test results are timestamped and can be queried as a time series (JLS-18, JLS-45), but due to storage limits we only keep a truncated history rather than a complete time series over the whole project lifetime. + - **Answer**: Stored test results are timestamped and can be queried as a time series (JLS-18, JLS-45). The persistent storage uses a memory-efficient strategy that records changes in test outcomes rather than storing every run as a separate data point. - If there are any gaps, do we understand why? - - **Answer**: Yes, gaps arise by design from the memory sensitive storage strategy (only initial snapshots and relevant changes are kept, see TSF/scripts/README.md and JLS-45), from the fixed size limits on the persistent database, and from the fact that we do not collect runtime monitoring data for deployed instances yet (see AOU-09, AOU-18 and AOU-19). + - **Answer**: Yes. Gaps can arise by design from the memory-sensitive storage strategy (only initial snapshots and relevant changes are kept, see TSF/scripts/README.md and JLS-45) and when CI runs are skipped/cancelled or fail before artefacts are generated. - Are the test validation strategies credible and appropriate? - **Answer**: Yes, the upstream nlohmann/json test suite is extensive and is complemented by additional TSF-related tests. These tests are executed on each change and on a daily schedule via CI (JLS-01, JLS-22), and their results are stored and analysed through TA-DATA and TA-ANALYSIS. - What proportion of the implemented tests are validated? diff --git a/TSF/trustable/assumptions-of-use/AOU-09.md b/TSF/trustable/assumptions-of-use/AOU-09.md index b2fb8ac3d8..6e4668e49f 100644 --- a/TSF/trustable/assumptions-of-use/AOU-09.md +++ b/TSF/trustable/assumptions-of-use/AOU-09.md @@ -3,4 +3,4 @@ level: 1.1 normative: true --- -The integrator shall ensure inside eclipse-score/inc_nlohmann_json that advanced warning indicators for misbehaviours are identified, and monitoring mechanisms are specified, verified and validated based on analysis. \ No newline at end of file +The integrator shall ensure, within eclipse-score/inc_nlohmann_json, that the data produced by the implemented advanced warning indicators are verified and validated based on analysis. \ No newline at end of file diff --git a/TSF/trustable/assumptions-of-use/AOU-18.md b/TSF/trustable/assumptions-of-use/AOU-18.md deleted file mode 100644 index c8c2f9dfce..0000000000 --- a/TSF/trustable/assumptions-of-use/AOU-18.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -level: 1.1 -normative: true ---- - -The integrator shall ensure that monitoring data from deployed software is accurately captured, securely stored, and well-documented for analysis within eclipse-score/inc_nlohmann_json, as long as the nlohmann/json library is actively used within eclipse-score/inc_nlohmann_json. diff --git a/TSF/trustable/statements/JLS-20.md b/TSF/trustable/statements/JLS-20.md index 09ed880405..4d8fd1d65a 100644 --- a/TSF/trustable/statements/JLS-20.md +++ b/TSF/trustable/statements/JLS-20.md @@ -9,14 +9,14 @@ references: path: ./.github/workflows/publish_documentation.yml description: "github workflow executing calculation and storage of trustable scores" - type: website - url: "https://github.com/eclipse-score/inc_nlohmann_json/blob/save_historical_data/TSF/TrustableScoring.db" - description: "the database containing the trustable scores" + url: "https://github.com/eclipse-score/inc_nlohmann_json/blob/save_historical_data/TSF/data_storage" + description: "the folder that contains databases containing the trustable scores" evidence: type: https_response_time configuration: target_seconds: 2 urls: - - https://github.com/eclipse-score/inc_nlohmann_json/blob/save_historical_data/TSF/TrustableScoring.db + - "https://github.com/eclipse-score/inc_nlohmann_json/blob/save_historical_data/TSF/data_storage" score: Jonas-Kirchhoff: 1.0 --- diff --git a/TSF/trustable/statements/JLS-31.md b/TSF/trustable/statements/JLS-31.md index ab6b857366..a7ac6e88c2 100644 --- a/TSF/trustable/statements/JLS-31.md +++ b/TSF/trustable/statements/JLS-31.md @@ -2,17 +2,29 @@ level: 1.1 normative: true references: - - type: project_website - url: "https://scorecard.dev/viewer/?uri=github.com%2Fnlohmann%2Fjson" - description: "OpenSSF Scorecard Report for nlohmann/json, where score for 'SAST' supports this statement." + - type: project_website + url: "https://json.nlohmann.me/community/quality_assurance/" + description: "Quality-assurance overview that lists the static analysis tools (e.g., clang-tidy / Clang Static Analyzer) and dynamic analysis tools (e.g., Valgrind / Clang Sanitizers) and states that violations fail the build." + - type: project_website + url: "https://github.com/nlohmann/json/actions" + description: "Public GitHub Actions run history showing the outcomes of CI jobs over time including the logs for the static code analysis tools." + - type: website + url: "https://app.codacy.com/gh/nlohmann/json/dashboard" + description: "Codacy dashboard for nlohmann/json" + - type: website + url: "https://coveralls.io/github/nlohmann/json" + description: "Coveralls dashboard for nlohmann/json, including coverage history/statistics (trend view)." evidence: - type: https_response_time - configuration: - target_seconds: 2 - urls: - - "https://scorecard.dev/viewer/?uri=github.com%2Fnlohmann%2Fjson" + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/community/quality_assurance/" + - "https://github.com/nlohmann/json/actions" + - "https://app.codacy.com/gh/nlohmann/json/dashboard" + - "https://coveralls.io/github/nlohmann/json" score: Erikhu1: 0.9 --- -The nlohmann/json repository uses a static code analysis tool. \ No newline at end of file +The nlohmann/json repository uses static code analysis tools and sanitizer. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-74.md b/TSF/trustable/statements/JLS-74.md new file mode 100644 index 0000000000..8b3fb53457 --- /dev/null +++ b/TSF/trustable/statements/JLS-74.md @@ -0,0 +1,20 @@ +--- +level: '1.1' +normative: true +references: + - type: website + url: "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-01" + description: "Generated item page for JLEX-01, including its supporting subtree and references." + - type: website + url: "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-02" + description: "Generated item page for JLEX-02, including its supporting subtree and references." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-01" + - "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-02" +--- + +Each supporting statement for the Expectations includes references to the CI test evidence that exercises it. The suitability of the referenced tests as evidence for each statement is validated by SME reviewers as part of the scoring process. \ No newline at end of file