diff --git a/.github/workflows/ada-spark-ci.yml b/.github/workflows/ada-spark-ci.yml index 18dc866..6381c28 100644 --- a/.github/workflows/ada-spark-ci.yml +++ b/.github/workflows/ada-spark-ci.yml @@ -5,7 +5,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup GNAT run: | @@ -32,7 +33,8 @@ jobs: runs-on: ubuntu-latest if: hashFiles('**/*.ads') != '' steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: SPARK proof run: | echo "SPARK proof would run here with gnatprove" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index df5de58..76cb89b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -54,8 +54,9 @@ jobs: # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages steps: + # SHA-pinned for supply chain security (RSR requirement) - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` @@ -64,8 +65,9 @@ jobs: # uses: actions/setup-example@v1 # Initializes the CodeQL tools for scanning. + # SHA-pinned for supply chain security (RSR requirement) - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@4f3212b61783c3340571c2c5c7cd92c19a472c42 # v3.28.0 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -93,7 +95,8 @@ jobs: echo ' make release' exit 1 + # SHA-pinned for supply chain security (RSR requirement) - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@4f3212b61783c3340571c2c5c7cd92c19a472c42 # v3.28.0 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/guix-nix-policy.yml b/.github/workflows/guix-nix-policy.yml index a776006..f8dd533 100644 --- a/.github/workflows/guix-nix-policy.yml +++ b/.github/workflows/guix-nix-policy.yml @@ -4,7 +4,8 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Enforce Guix primary / Nix fallback run: | # Check for package manager files diff --git a/.github/workflows/jekyll-gh-pages.yml b/.github/workflows/jekyll-gh-pages.yml index e31d81c..a02bb87 100644 --- a/.github/workflows/jekyll-gh-pages.yml +++ b/.github/workflows/jekyll-gh-pages.yml @@ -26,17 +26,21 @@ jobs: build: runs-on: ubuntu-latest steps: + # SHA-pinned for supply chain security (RSR requirement) - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # SHA-pinned for supply chain security (RSR requirement) - name: Setup Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + # SHA-pinned for supply chain security (RSR requirement) - name: Build with Jekyll - uses: actions/jekyll-build-pages@v1 + uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 with: source: ./ destination: ./_site + # SHA-pinned for supply chain security (RSR requirement) - name: Upload artifact - uses: actions/upload-pages-artifact@v3 + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 # Deployment job deploy: @@ -46,6 +50,7 @@ jobs: runs-on: ubuntu-latest needs: build steps: + # SHA-pinned for supply chain security (RSR requirement) - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/.github/workflows/npm-bun-blocker.yml b/.github/workflows/npm-bun-blocker.yml index e19c080..f8da7a0 100644 --- a/.github/workflows/npm-bun-blocker.yml +++ b/.github/workflows/npm-bun-blocker.yml @@ -4,7 +4,8 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Block npm/bun run: | if [ -f "package-lock.json" ] || [ -f "bun.lockb" ] || [ -f ".npmrc" ]; then diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index eb78d2c..8399759 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -5,14 +5,16 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Check file permissions run: | find . -type f -perm /111 -name "*.sh" | head -10 || true + # SHA-pinned for supply chain security (RSR requirement) - name: Check for secrets - uses: trufflesecurity/trufflehog@main + uses: trufflesecurity/trufflehog@8ac2561ada5fea3e56d45e2e657efe4dfcdbc4a1 # v3.88.1 with: path: ./ base: ${{ github.event.pull_request.base.sha || github.event.before }} @@ -28,14 +30,16 @@ jobs: run: | find . -type f -size +1M -not -path "./.git/*" | head -10 || echo "No large files" + # SHA-pinned for supply chain security (RSR requirement) - name: EditorConfig check - uses: editorconfig-checker/action-editorconfig-checker@main + uses: editorconfig-checker/action-editorconfig-checker@e2e2455cfb4a316de2a7eef40c1b4b32f0b03eeb # v2.0.0 continue-on-error: true docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Check documentation run: | MISSING="" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index a073b17..c04acfa 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -14,17 +14,20 @@ jobs: security-events: write id-token: write steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - + + # SHA-pinned for supply chain security (RSR requirement) - name: Run Scorecard - uses: ossf/scorecard-action@v2.3.1 + uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0 with: results_file: results.sarif results_format: sarif + # SHA-pinned for supply chain security (RSR requirement) - name: Upload results - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@4f3212b61783c3340571c2c5c7cd92c19a472c42 # v3.28.0 with: sarif_file: results.sarif diff --git a/.github/workflows/security-policy.yml b/.github/workflows/security-policy.yml index c20bbc7..1da2466 100644 --- a/.github/workflows/security-policy.yml +++ b/.github/workflows/security-policy.yml @@ -4,7 +4,8 @@ jobs: check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Security checks run: | FAILED=false diff --git a/.github/workflows/wellknown-enforcement.yml b/.github/workflows/wellknown-enforcement.yml index 809d209..3aeeb2a 100644 --- a/.github/workflows/wellknown-enforcement.yml +++ b/.github/workflows/wellknown-enforcement.yml @@ -17,8 +17,9 @@ jobs: validate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - + # SHA-pinned for supply chain security (RSR requirement) + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: RFC 9116 security.txt validation run: | SECTXT="" diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..f84f77a --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,127 @@ + + + +# Vexometer Roadmap + +**Version**: 0.2.0-dev +**Phase**: Extending Metrics +**Updated**: 2025-12-17 + +## Current Status + +Vexometer is the **hub** of the irritation surface analysis ecosystem. It measures AI assistant friction, annoyances, and failures through quantified metrics. **It diagnoses; it does not prescribe treatment.** + +### Completed (v0.1.0) + +- Core ISA (Irritation Surface Analyser) framework +- Original 6 metrics implemented: + - **TII** - Temporal Intrusion Index (time-wasting behaviours) + - **LPS** - Linguistic Pathology Score (verbal tics, padding, sycophancy) + - **EFR** - Epistemic Failure Rate (hallucination, false confidence) + - **PQ** - Paternalism Quotient (over-helping, unsolicited warnings) + - **TAI** - Telemetry Anxiety Index (privacy concerns) + - **ICS** - Interaction Coherence Score (conversation flow) +- RSR compliance infrastructure +- Basic measurement pipeline +- Pattern detection framework +- Model comparison and ranking +- GtkAda GUI framework +- Multi-provider API client (local + remote LLMs) + +--- + +## In Progress (v0.2.0) + +### Extended Metrics (v2) + +Four additional metrics to complete diagnostic coverage: + +| Metric | Name | Status | Description | +|--------|------|--------|-------------| +| **CII** | Completion Integrity Index | Specification complete | Incomplete outputs, placeholders, lazy generation | +| **SRS** | Strategic Rigidity Score | Specification complete | Backtrack resistance, sunk-cost patching | +| **SFR** | Scope Fidelity Ratio | Specification complete | Scope creep/collapse, request alignment | +| **RCI** | Recovery Competence Index | Specification complete | Error recovery quality, strategy variation | + +### Next Steps + +1. Implement CII detection patterns for common languages +2. Implement SRS event classification and tracking +3. Implement SFR scope comparison algorithm +4. Implement RCI approach fingerprinting +5. Create satellite integration interface specification +6. Document metric calculation methodology + +--- + +## Planned (v0.3.0) + +### Satellite Integration Interface + +- **vexometer-trace-v1** protocol specification +- **vexometer-efficacy-v1** protocol for satellite reporting +- **vexometer-metrics-v1** subscription protocol +- Before/after trace validation +- Metric reduction percentage reporting + +### Satellite Ecosystem (Independent Repos) + +| Satellite | Reduces | Status | Purpose | +|-----------|---------|--------|---------| +| vex-lazy-eliminator | CII, LPS | Planned | Completeness enforcement | +| vex-hallucination-guard | EFR | Planned | Factual verification layer | +| vex-sycophancy-shield | LPS, EFR | Planned | Epistemic commitment tracking | +| vex-confidence-calibrator | EFR | Planned | Structured uncertainty | +| vex-specification-anchor | SFR, ICS | Planned | Immutable requirements ledger | +| vex-instruction-persistence | TII, ICS | Planned | System instruction compliance | +| vex-backtrack-enabler | SRS, ICS | Planned | Low-friction restart support | +| vex-context-firewall | EFR, ICS | Planned | Truth maintenance | +| vex-scope-governor | SFR, PQ | Planned | Scope contract enforcement | +| vex-error-recovery | RCI | Planned | Strategy variation on failure | +| vex-verbosity-compressor | LPS, TII | Planned | Information density optimisation | +| vex-clarification-gate | PQ, TII | Planned | Risk-weighted ambiguity handling | + +--- + +## Future Considerations (v1.0.0) + +- SPARK formal verification for metric calculations +- Full AUnit test coverage +- Container distribution (Podman/Docker) +- API bindings for integration (Rust, Elixir) +- Real-time analysis mode +- Benchmark suite with curated LLM interactions +- Public metric comparison database + +--- + +## Architecture Decisions + +| ADR | Decision | Status | Rationale | +|-----|----------|--------|-----------| +| ADR-001 | RSR Compliance | Accepted | RSR Gold target, SHA-pinned actions, SPDX headers | +| ADR-002 | Satellite Architecture | Accepted | Keep vexometer pure diagnostic; interventions in satellites | +| ADR-003 | Metric Normalisation | Accepted | All metrics 0-1 scale, lower is better | +| ADR-004 | Language Choice | Accepted | Ada/SPARK for formal verification of metric calculations | + +--- + +## Technical Stack + +- **Language**: Ada 2022 with SPARK annotations +- **Build**: gprbuild + Alire +- **GUI**: GtkAda +- **Package Management**: Guix (primary) / Nix (fallback) +- **CI/CD**: GitHub Actions (SHA-pinned) + GitLab CI +- **Standard**: RSR (Rhodium Standard Repository) + +--- + +## Contributing + +See [CONTRIBUTING.adoc](CONTRIBUTING.adoc) for guidelines. Vexometer follows a cathedral development model. + +## Related Projects + +- [rhodium-standard-repositories](https://github.com/hyperpolymath/rhodium-standard-repositories) - Repository standard +- [vexometer-satellites](https://gitlab.com/hyperpolymath/vexometer-satellites) - Satellite index diff --git a/flake.nix b/flake.nix index c6bda0d..e329ce4 100644 --- a/flake.nix +++ b/flake.nix @@ -98,7 +98,7 @@ packages.default = pkgs.stdenv.mkDerivation { pname = "vexometer"; - version = "0.1.0"; + version = "0.2.0-dev"; src = ./.; diff --git a/guix.scm b/guix.scm new file mode 100644 index 0000000..34690c5 --- /dev/null +++ b/guix.scm @@ -0,0 +1,90 @@ +;; SPDX-FileCopyrightText: 2025 Jonathan D.A. Jewell +;; SPDX-License-Identifier: AGPL-3.0-or-later +;; +;; Vexometer - Irritation Surface Analyser +;; GNU Guix package definition (RSR primary package manager) +;; +;; Usage: +;; guix shell -f guix.scm # Enter dev environment +;; guix build -f guix.scm # Build package +;; guix pack -f guix.scm # Create relocatable tarball + +(use-modules (guix packages) + (guix gexp) + (guix git-download) + (guix build-system gnu) + ((guix licenses) #:prefix license:) + (gnu packages ada) + (gnu packages gtk) + (gnu packages pkg-config) + (gnu packages version-control) + (gnu packages build-tools) + (gnu packages curl) + (gnu packages tls) + (gnu packages documentation) + (gnu packages shellutils)) + +(define-public vexometer + (package + (name "vexometer") + (version "0.2.0-dev") + (source (local-file "." "vexometer-checkout" + #:recursive? #t + #:select? (lambda (file stat) + (not (or (string-suffix? ".git" file) + (string-suffix? "obj" file) + (string-suffix? "bin" file)))))) + (build-system gnu-build-system) + (arguments + (list + #:phases + #~(modify-phases %standard-phases + (delete 'configure) + (replace 'build + (lambda _ + (invoke "gprbuild" "-P" "vexometer.gpr" + "-XVEXOMETER_BUILD_MODE=release"))) + (replace 'check + (lambda* (#:key tests? #:allow-other-keys) + (when tests? + ;; AUnit tests would run here + (display "Tests not yet implemented\n")))) + (replace 'install + (lambda* (#:key outputs #:allow-other-keys) + (let* ((out (assoc-ref outputs "out")) + (bin (string-append out "/bin")) + (share (string-append out "/share/vexometer"))) + (mkdir-p bin) + (mkdir-p share) + (install-file "bin/vexometer" bin) + (copy-recursively "data" share))))))) + (native-inputs + (list gprbuild + pkg-config)) + (inputs + (list gnat + gtkada + gtk+ + gnatcoll + aws-ada + curl + openssl)) + (propagated-inputs + (list just)) + (synopsis "Irritation Surface Analyser for AI assistants") + (description + "Vexometer measures AI assistant 'irritation surfaces' - the friction, +annoyances, and failures that make AI interactions frustrating. It provides +quantified metrics across dimensions including temporal intrusion (TII), +linguistic pathology (LPS), epistemic failure (EFR), paternalism (PQ), +telemetry anxiety (TAI), interaction coherence (ICS), completion integrity +(CII), strategic rigidity (SRS), scope fidelity (SFR), and recovery +competence (RCI). + +Vexometer diagnoses; it does not prescribe treatment. Intervention tools +(satellites) are developed in separate repositories.") + (home-page "https://gitlab.com/hyperpolymath/vexometer") + (license license:agpl3+))) + +;; Return the package for `guix build -f guix.scm` +vexometer diff --git a/justfile b/justfile index 68a9bce..12867ac 100644 --- a/justfile +++ b/justfile @@ -12,7 +12,7 @@ default: build # Project metadata project := "vexometer" -version := "0.1.0" +version := "0.2.0-dev" # Build modes build_mode := env_var_or_default("VEXOMETER_BUILD_MODE", "debug") @@ -95,7 +95,8 @@ docs-html: validate: @echo "Validating RSR compliance..." @echo "Checking required files..." - @test -f flake.nix && echo "✓ flake.nix" || echo "✗ flake.nix missing" + @test -f guix.scm && echo "✓ guix.scm (primary)" || echo "✗ guix.scm missing (RSR primary)" + @test -f flake.nix && echo "✓ flake.nix (fallback)" || echo "✗ flake.nix missing" @test -f justfile && echo "✓ justfile" || echo "✗ justfile missing" @test -f README.adoc && echo "✓ README.adoc" || echo "✗ README.adoc missing" @test -f LICENSE.txt && echo "✓ LICENSE.txt" || echo "✗ LICENSE.txt missing"