Add http-mitm-proxy for HTTPS interception

.gitignore

@@ -35,3 +35,6 @@ report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
 
 # Agent runtime data (contexts, logs)
 .mini-agent/
+.http-mitm-proxy/
+.mkcert-ca/
+.mitm-mkcert-certs/

docs/MITM-PROXY.md

@@ -0,0 +1,183 @@
+# MITM Proxy Setup
+
+This document describes how to set up an HTTPS-intercepting MITM proxy for testing and development.
+
+## Overview
+
+The proxy intercepts HTTPS traffic, allowing you to:
+- Inject headers on outgoing requests
+- Modify response bodies (for text content)
+- Log all HTTPS traffic
+
+## Quick Start
+
+```bash
+# Start the proxy
+bun run test/mitm-proxy.ts
+
+# In another terminal, test it:
+export https_proxy=http://localhost:8080
+curl --cacert .http-mitm-proxy/certs/ca.pem https://httpbin.org/headers
+```
+
+## Client Compatibility
+
+| Client | Works | Notes |
+|--------|-------|-------|
+| **curl** | ✅ | `--cacert path/to/ca.pem` |
+| **wget** | ✅ | `--ca-certificate=path/to/ca.pem` |
+| **Python urllib** | ✅ | `ssl.load_verify_locations(ca_path)` |
+| **Node.js/Bun** | ✅ | `NODE_EXTRA_CA_CERTS=path/to/ca.pem` |
+| **npm** | ✅ | Requires explicit config (see below) |
+| **Go http.Client** | ⚠️ | See "Go Compatibility" below |
+| **gh CLI** | ⚠️ | Uses Go internally |
+
+## Configuration by Client
+
+### curl
+
+```bash
+export https_proxy=http://localhost:8080
+curl --cacert .http-mitm-proxy/certs/ca.pem https://example.com
+```
+
+### wget
+
+```bash
+export HTTPS_PROXY=http://localhost:8080
+wget --ca-certificate=.http-mitm-proxy/certs/ca.pem https://example.com
+```
+
+### Python
+
+```python
+import urllib.request
+import ssl
+
+proxy = urllib.request.ProxyHandler({'https': 'http://localhost:8080'})
+ctx = ssl.create_default_context()
+ctx.load_verify_locations('.http-mitm-proxy/certs/ca.pem')
+https = urllib.request.HTTPSHandler(context=ctx)
+opener = urllib.request.build_opener(proxy, https)
+
+response = opener.open('https://example.com')
+```
+
+### Node.js / Bun
+
+```bash
+export NODE_EXTRA_CA_CERTS=.http-mitm-proxy/certs/ca.pem
+export https_proxy=http://localhost:8080
+node my-script.js
+```
+
+Or programmatically with undici:
+
+```typescript
+import { ProxyAgent } from 'undici'
+import { readFileSync } from 'node:fs'
+
+const dispatcher = new ProxyAgent({
+  uri: 'http://localhost:8080',
+  requestTls: { ca: readFileSync('.http-mitm-proxy/certs/ca.pem') }
+})
+
+const response = await fetch('https://example.com', { dispatcher })
+```
+
+### npm
+
+npm ignores `https_proxy` env var. Configure explicitly:
+
+```bash
+npm config set proxy http://localhost:8080
+npm config set https-proxy http://localhost:8080
+npm config set cafile .http-mitm-proxy/certs/ca.pem
+npm config set strict-ssl false
+
+npm install some-package
+```
+
+To clean up after testing:
+
+```bash
+npm config delete proxy https-proxy cafile strict-ssl
+```
+
+## Go Compatibility
+
+**Issue:** http-mitm-proxy uses node-forge for certificate generation, which can produce certificates with negative serial numbers. Go's `crypto/x509` package rejects these as invalid.
+
+**Workaround:** Use mkcert for certificate generation:
+
+```bash
+# Install mkcert
+apt-get install mkcert
+
+# Create CA
+CAROOT=.mkcert-ca mkcert -install
+
+# Use mkcert proxy variant
+bun run test/mitm-proxy-mkcert.ts
+
+# Set SSL_CERT_FILE for Go clients
+export SSL_CERT_FILE=.mkcert-ca/rootCA.pem
+export HTTPS_PROXY=http://localhost:8080
+```
+
+**Note:** Even with mkcert certs, Go clients may encounter "too many transfer encodings" errors due to HTTP/1.x transport strictness. This is a known limitation.
+
+## System-Wide Trust
+
+To avoid passing CA cert path to every command:
+
+### Linux (Debian/Ubuntu)
+
+```bash
+cp .http-mitm-proxy/certs/ca.pem /usr/local/share/ca-certificates/mitm-proxy.crt
+update-ca-certificates
+```
+
+### macOS
+
+```bash
+sudo security add-trusted-cert -d -r trustRoot \
+  -k /Library/Keychains/System.keychain \
+  .http-mitm-proxy/certs/ca.pem
+```
+
+## Response Modification
+
+The proxy uses `Proxy.gunzip` to automatically decompress gzip/deflate responses before modification. Modifies these content types:
+- `text/html`
+- `text/plain`
+- `application/json`
+
+Binary content (images, tarballs) passes through unmodified.
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `test/mitm-proxy.ts` | Basic proxy (node-forge certs) |
+| `test/mitm-proxy-mkcert.ts` | Proxy with mkcert certs (Go-compatible) |
+| `.http-mitm-proxy/certs/` | Auto-generated certificates |
+| `.mkcert-ca/` | mkcert CA (if using mkcert variant) |
+
+## Troubleshooting
+
+### "certificate verify failed"
+- Ensure CA cert is trusted (system-wide or per-command)
+- Check the correct CA path is being used
+
+### npm install fails with Z_DATA_ERROR
+- Earlier versions corrupted compressed responses
+- Fixed by using `Proxy.gunzip` for automatic decompression
+
+### Go "negative serial number"
+- Use mkcert proxy variant instead of default
+- Or upgrade to http-mitm-proxy version that fixes node-forge cert generation
+
+### Go "too many transfer encodings"
+- Known issue with Go's strict HTTP/1.x parsing
+- Workaround: use HTTP/2 where possible

package.json

@@ -34,6 +34,7 @@
     "eslint-plugin-sort-destructure-keys": "^2.0.0",
     "semver": "^7.7.3",
     "tuistory": "^0.0.2",
+    "undici": "^7.18.2",
     "vitest": "^3.2.0"
   },
   "peerDependencies": {
@@ -55,6 +56,7 @@
     "@opentui/core": "^0.1.55",
     "@opentui/react": "^0.1.55",
     "effect": "^3.19.8",
+    "http-mitm-proxy": "^1.1.0",
     "react": "19",
     "react-dom": "19",
     "yaml": "^2.7.0"

test/mitm-proxy-mkcert.ts

@@ -0,0 +1,175 @@
+/**
+ * MITM proxy using mkcert for certificate generation.
+ *
+ * This version uses mkcert instead of node-forge for cert generation,
+ * which produces certs that Go's crypto/x509 accepts (proper serial numbers).
+ */
+
+import { Proxy as MitmProxy } from "http-mitm-proxy"
+import { execSync, spawnSync } from "node:child_process"
+import { existsSync, mkdirSync, readFileSync } from "node:fs"
+
+export const INJECTED_HEADER = "X-Proxy-Injected"
+export const INJECTED_HEADER_VALUE = "mitm-active"
+export const RESPONSE_MARKER = "\n<!-- MITM_PROXY_MARKER -->"
+
+const MKCERT_CAROOT = process.env.MKCERT_CAROOT || "/home/user/mini-agent/.mkcert-ca"
+const CERTS_DIR = "/home/user/mini-agent/.mitm-mkcert-certs"
+
+// Ensure CA exists
+function ensureMkcertCa() {
+  if (!existsSync(`${MKCERT_CAROOT}/rootCA.pem`)) {
+    console.log("Creating mkcert CA...")
+    execSync(`CAROOT=${MKCERT_CAROOT} mkcert -install`, { stdio: "inherit" })
+  }
+}
+
+// Generate cert for hostname using mkcert
+function generateCert(hostname: string): { keyFileData: string; certFileData: string } {
+  mkdirSync(CERTS_DIR, { recursive: true })
+
+  const keyFile = `${CERTS_DIR}/${hostname}-key.pem`
+  const certFile = `${CERTS_DIR}/${hostname}.pem`
+
+  // Check if cert already exists
+  if (existsSync(keyFile) && existsSync(certFile)) {
+    return {
+      keyFileData: readFileSync(keyFile, "utf-8"),
+      certFileData: readFileSync(certFile, "utf-8")
+    }
+  }
+
+  // Generate new cert with mkcert
+  const result = spawnSync(
+    "mkcert",
+    ["-key-file", keyFile, "-cert-file", certFile, hostname],
+    {
+      env: { ...process.env, CAROOT: MKCERT_CAROOT },
+      encoding: "utf-8"
+    }
+  )
+
+  if (result.status !== 0) {
+    throw new Error(`mkcert failed for ${hostname}: ${result.stderr}`)
+  }
+
+  return {
+    keyFileData: readFileSync(keyFile, "utf-8"),
+    certFileData: readFileSync(certFile, "utf-8")
+  }
+}
+
+export function createMitmProxyWithMkcert(port: number) {
+  ensureMkcertCa()
+
+  const proxy = new MitmProxy()
+
+  proxy.onError((_ctx, err) => {
+    console.error("[PROXY ERROR]", err?.message || err)
+  })
+
+  // Use mkcert for cert generation instead of node-forge
+  proxy.onCertificateMissing = (ctx, _files, callback) => {
+    console.log(`[PROXY] Generating cert for ${ctx.hostname}`)
+    try {
+      const { certFileData, keyFileData } = generateCert(ctx.hostname)
+      callback(null, { keyFileData, certFileData })
+    } catch (err) {
+      console.error(`[PROXY] Cert generation failed:`, err)
+      callback(err as Error)
+    }
+  }
+
+  proxy.onRequest((ctx, callback) => {
+    const host = ctx.clientToProxyRequest.headers.host || "unknown"
+    const url = ctx.clientToProxyRequest.url || "/"
+    console.log(`[PROXY] → ${ctx.clientToProxyRequest.method} ${host}${url}`)
+
+    // Inject header on outgoing request
+    if (ctx.proxyToServerRequestOptions?.headers) {
+      ctx.proxyToServerRequestOptions.headers[INJECTED_HEADER] = INJECTED_HEADER_VALUE
+    }
+
+    // Track chunks for this specific request
+    let pendingChunk: Buffer | null = null
+    let shouldModifyResponse = false
+
+    // Check content type to decide if we should modify response
+    ctx.onResponse((ctx, callback) => {
+      const contentType = ctx.serverToProxyResponse?.headers["content-type"] || ""
+      const contentEncoding = ctx.serverToProxyResponse?.headers["content-encoding"]
+
+      // Only modify uncompressed text responses
+      shouldModifyResponse = !contentEncoding &&
+        (contentType.includes("text/html") ||
+          contentType.includes("text/plain") ||
+          contentType.includes("application/json"))
+
+      callback()
+    })
+
+    // Buffer chunks to append marker to the final one (only for text)
+    ctx.onResponseData((_ctx, chunk, callback) => {
+      if (!shouldModifyResponse) {
+        callback(null, chunk)
+        return
+      }
+      const toSend = pendingChunk
+      pendingChunk = chunk
+      callback(null, toSend ?? Buffer.alloc(0))
+    })
+
+    ctx.onResponseEnd((ctx, callback) => {
+      if (shouldModifyResponse && pendingChunk) {
+        const withMarker = Buffer.concat([pendingChunk, Buffer.from(RESPONSE_MARKER)])
+        ctx.proxyToClientResponse.write(withMarker)
+      }
+      callback()
+    })
+
+    callback()
+  })
+
+  proxy.onResponse((ctx, callback) => {
+    const status = ctx.serverToProxyResponse?.statusCode
+    console.log(`[PROXY] ← ${status}`)
+    callback()
+  })
+
+  return {
+    start: () =>
+      new Promise<void>((resolve, reject) => {
+        proxy.listen({ port }, (err: Error | null | undefined) => {
+          if (err) reject(err)
+          else resolve()
+        })
+      }),
+    stop: () =>
+      new Promise<void>((resolve) => {
+        proxy.close()
+        resolve()
+      }),
+    port,
+    caPath: `${MKCERT_CAROOT}/rootCA.pem`
+  }
+}
+
+// Run standalone if executed directly
+if (import.meta.main) {
+  const PORT = Number(process.env.PROXY_PORT) || 8080
+  const proxy = createMitmProxyWithMkcert(PORT)
+
+  await proxy.start()
+  console.log(`MITM proxy (mkcert) listening on port ${PORT}`)
+  console.log(`CA cert: ${proxy.caPath}`)
+  console.log("")
+  console.log("Usage:")
+  console.log(`  export https_proxy=http://localhost:${PORT}`)
+  console.log(`  export SSL_CERT_FILE=${proxy.caPath}  # for Go`)
+  console.log(`  curl https://httpbin.org/headers`)
+
+  process.on("SIGINT", async () => {
+    await proxy.stop()
+    process.exit(0)
+  })
+}