From d3987c0f68d490a095580deca18b2388a44da0f9 Mon Sep 17 00:00:00 2001 From: dadukhankevin Date: Sun, 15 Feb 2026 14:49:27 -0600 Subject: [PATCH] Reduce A/B testing and attention check frequency from 15% to 1% The default probability was triggering tests too frequently. Changed the default from 0.15 (1 in ~7) to 0.01 (1 in 100) across the setting default, the runtime fallback, and documentation. Co-Authored-By: Claude Opus 4.6 --- docs/AB_TESTING.md | 4 ++-- package.json | 2 +- src/providers/translationSuggestions/llmCompletion.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/AB_TESTING.md b/docs/AB_TESTING.md index 2f0766eb7..212d57f3e 100644 --- a/docs/AB_TESTING.md +++ b/docs/AB_TESTING.md @@ -3,7 +3,7 @@ A/B testing in Codex shows two translation suggestions side‑by‑side once in a while so you can pick the better one. This helps us learn which retrieval and prompting strategies work best without slowing you down. ## How it works -- Triggering: Tests run at random with a small probability (default 15%). +- Triggering: Tests run at random with a small probability (default 1%). - Variants: When triggered, two candidates are generated in parallel. - Auto‑apply: If the two results are effectively identical, we apply one automatically and no modal is shown. - Choosing: If they differ, a simple chooser appears; click the option that reads best. Dismissing the modal after choosing just closes it. @@ -16,7 +16,7 @@ A/B testing in Codex shows two translation suggestions side‑by‑side once in ## Settings - `codex-editor-extension.abTestingEnabled`: turn A/B testing on/off. -- `codex-editor-extension.abTestingProbability`: probability (0–1) for running a true A/B test. Default: `0.15` (15%). +- `codex-editor-extension.abTestingProbability`: probability (0–1) for running a true A/B test. Default: `0.01` (1%). Change these in VS Code Settings → Extensions → Codex Editor. diff --git a/package.json b/package.json index ccef3077b..1038660bd 100644 --- a/package.json +++ b/package.json @@ -940,7 +940,7 @@ "codex-editor-extension.abTestingProbability": { "title": "A/B Testing Probability", "type": "number", - "default": 0.15, + "default": 0.01, "minimum": 0, "maximum": 1, "description": "Probability (0-1) that any eligible event will run a true A/B test. When not triggered, the system returns identical variants to keep UX consistent without doubling compute." diff --git a/src/providers/translationSuggestions/llmCompletion.ts b/src/providers/translationSuggestions/llmCompletion.ts index 17215eb4f..8d3d4490d 100644 --- a/src/providers/translationSuggestions/llmCompletion.ts +++ b/src/providers/translationSuggestions/llmCompletion.ts @@ -251,7 +251,7 @@ export async function llmCompletion( const extConfig = vscode.workspace.getConfiguration("codex-editor-extension"); const abEnabled = Boolean(extConfig.get("abTestingEnabled") ?? true) && !isBatchOperation; const abProbabilityRaw = extConfig.get("abTestingProbability"); - const abProbability = Math.max(0, Math.min(1, typeof abProbabilityRaw === "number" ? abProbabilityRaw : 0.15)); + const abProbability = Math.max(0, Math.min(1, typeof abProbabilityRaw === "number" ? abProbabilityRaw : 0.01)); const randomValue = Math.random(); const triggerAB = abEnabled && randomValue < abProbability;