chore: Playwright cloud test

.github/workflows/e2e.yml

@@ -1,14 +1,28 @@
 name: E2E Tests
+
 on:
   workflow_call:
+    secrets:
+      AZURE_CLIENT_ID:
+        required: false
+      AZURE_TENANT_ID:
+        required: false
+      AZURE_SUBSCRIPTION_ID:
+        required: false
+      PLAYWRIGHT_SERVICE_URL:
+        required: false
+      # Add other secrets if necessary
   workflow_dispatch:
 
+env:
+  TELEMETRY_DISABLED: 1
+
 permissions:
+  id-token: write
   contents: read
   actions: read
-  checks: write    
-env:
-  TELEMETRY_DISABLED: 1
+  checks: write
+
 jobs:
   build:
     name: Run E2E Tests
@@ -88,7 +102,31 @@ jobs:
       - name: Install Playwright
         run: pnpm exec playwright install --with-deps
 
-      - name: Run E2E Tests
+      - name: Set Azure Secret Variables
+        run: |
+          if [[ -n "${{ secrets.AZURE_CLIENT_ID }}" && -n "${{ secrets.AZURE_TENANT_ID }}" && -n "${{ secrets.AZURE_SUBSCRIPTION_ID }}" ]]; then
+            echo "AZURE_ENABLED=true" >> $GITHUB_ENV
+          else
+            echo "AZURE_ENABLED=false" >> $GITHUB_ENV
+          fi
+
+      - name: Azure login
+        if: env.AZURE_ENABLED == 'true'
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Run E2E Tests (Azure)
+        if: env.AZURE_ENABLED == 'true'
+        env:
+          PLAYWRIGHT_SERVICE_URL: ${{ secrets.PLAYWRIGHT_SERVICE_URL }}
+        run: |
+          pnpm test-e2e:azure
+
+      - name: Run E2E Tests (Local)
+        if: env.AZURE_ENABLED == 'false'
         run: |
           pnpm test:e2e
 

.github/workflows/pr.yml

@@ -6,6 +6,7 @@ permissions:
   pull-requests: read
   actions: read
   checks: write
+  id-token: write
 
 on:
   pull_request:
@@ -19,53 +20,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  changes:
-    name: Detect changes
-    runs-on: ubuntu-latest
-    outputs:
-      has-files-requiring-all-checks: ${{ steps.filter.outputs.has-files-requiring-all-checks }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/dangerous-git-checkout
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            has-files-requiring-all-checks:
-              - "!(**.md|.github/CODEOWNERS)"
-
   test:
     name: Run Unit Tests
-    needs: [changes]
-    if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
   lint:
     name: Run Linters
-    needs: [changes]
-    if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/lint.yml
     secrets: inherit
 
   build:
     name: Build Formbricks-web
-    needs: [changes]
-    if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/build-web.yml
     secrets: inherit
 
   docs:
     name: Build Docs
-    needs: [changes]
-    if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/build-docs.yml
     secrets: inherit
 
   e2e-test:
     name: Run E2E Tests
-    needs: [changes]
-    if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/e2e.yml
     secrets: inherit
 

apps/web/app/(app)/(survey-editor)/environments/[environmentId]/surveys/[surveyId]/edit/components/MatrixQuestionForm.tsx

@@ -166,7 +166,7 @@ export const MatrixQuestionForm = ({
               <div
                 className="flex items-center"
                 onKeyDown={(e) => handleKeyDown(e, "row")}
-                key={`row-${index}-${question.rows.length}`}>
+                key={`row-${index}`}>
                 <QuestionFormInput
                   id={`row-${index}`}
                   label={""}
@@ -219,7 +219,7 @@ export const MatrixQuestionForm = ({
               <div
                 className="flex items-center"
                 onKeyDown={(e) => handleKeyDown(e, "column")}
-                key={`column-${index}-${question.columns.length}`}>
+                key={`column-${index}`}>
                 <QuestionFormInput
                   id={`column-${index}`}
                   label={""}

apps/web/playwright/js.spec.ts

@@ -1,125 +1,144 @@
 import { expect } from "@playwright/test";
+import http from "http";
 import { test } from "./lib/fixtures";
-import { replaceEnvironmentIdInHtml } from "./utils/helper";
+
+const HTML_TEMPLATE = `<head>
+  <script type="text/javascript">
+    !(function () {
+      var t = document.createElement("script");
+      (t.type = "text/javascript"), (t.async = !0), (t.src = "http://localhost:3000/js/formbricks.umd.cjs");
+      var e = document.getElementsByTagName("script")[0];
+      e.parentNode.insertBefore(t, e),
+        setTimeout(function () {
+          formbricks.init({
+            environmentId: "ENVIRONMENT_ID",
+            userId: "RANDOM_USER_ID",
+            apiHost: "http://localhost:3000",
+          });
+        }, 500);
+    })();
+  </script>
+</head>
+
+<body style="background-color: #fff">
+  <p>This is my sample page using the Formbricks JS javascript widget</p>
+</body>
+`;
 
 test.describe("JS Package Test", async () => {
+  let server: http.Server;
   let environmentId: string;
 
-  test("Tests", async ({ page, users }) => {
-    await test.step("Admin creates an In-App Survey", async () => {
-      const user = await users.create();
-      await user.login();
-
-      await page.waitForURL(/\/environments\/[^/]+\/surveys/);
-
-      await page.getByRole("heading", { name: "Product Market Fit (Superhuman)" }).isVisible();
-      await page.getByRole("heading", { name: "Product Market Fit (Superhuman)" }).click();
-
-      await page.getByRole("button", { name: "Use this template" }).isVisible();
-      await page.getByRole("button", { name: "Use this template" }).click();
-
-      await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/edit/);
-
-      await page.getByRole("button", { name: "Settings", exact: true }).click();
-
-      await expect(page.locator("#howToSendCardTrigger")).toBeVisible();
-      await page.locator("#howToSendCardTrigger").click();
-
-      await expect(page.locator("#howToSendCardOption-app")).toBeVisible();
-      await page.locator("#howToSendCardOption-app").click();
+  test.beforeAll(async () => {
+    // Create a simple HTTP server
+    server = http.createServer((_, res) => {
+      const htmlContent = HTML_TEMPLATE.replace("ENVIRONMENT_ID", environmentId || "");
+      res.writeHead(200, { "Content-Type": "text/html" });
+      res.end(htmlContent);
+    });
 
-      await page.locator("#whenToSendCardTrigger").click();
+    await new Promise<void>((resolve) => {
+      server.listen(3004, () => resolve());
+    });
+  });
 
-      await page.getByRole("button", { name: "Add action" }).click();
-      await page.getByText("New SessionGets fired when a").click();
+  test.afterAll(async () => {
+    // Cleanup: close the server
+    await new Promise((resolve) => server.close(resolve));
+  });
 
-      await page.locator("#recontactOptionsCardTrigger").click();
+  test("Create, display and validate PMF survey", async ({ page, users }) => {
+    // Create and login user
+    const user = await users.create();
+    await user.login();
 
-      await page.locator("label").filter({ hasText: "Keep showing while conditions" }).click();
-      await page.locator("#recontactDays").check();
+    await page.waitForURL(/\/environments\/[^/]+\/surveys/);
 
-      await page.getByRole("button", { name: "Publish" }).click();
+    // Extract environmentId early in the test
+    environmentId =
+      /\/environments\/([^/]+)\/surveys/.exec(page.url())?.[1] ??
+      (() => {
+        throw new Error("Unable to parse environmentId from URL");
+      })();
 
-      environmentId =
-        /\/environments\/([^/]+)\/surveys/.exec(page.url())?.[1] ??
-        (() => {
-          throw new Error("Unable to parse environmentId from URL");
-        })();
+    // Create survey from template
+    await page.getByRole("heading", { name: "Product Market Fit (Superhuman)" }).isVisible();
+    await page.getByRole("heading", { name: "Product Market Fit (Superhuman)" }).click();
+    await page.getByRole("button", { name: "Use this template" }).isVisible();
+    await page.getByRole("button", { name: "Use this template" }).click();
 
-      await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary/);
-    });
+    // Configure survey settings
+    await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/edit/);
+    await page.getByRole("button", { name: "Settings", exact: true }).click();
 
-    await test.step("JS display survey on page and submit response", async () => {
-      let currentDir = process.cwd();
-      let htmlFilePath = currentDir + "/packages/js/index.html";
-
-      let htmlFile = replaceEnvironmentIdInHtml(htmlFilePath, environmentId);
-      await page.goto(htmlFile);
-
-      // Formbricks In App Sync has happened
-      const syncApi = await page.waitForResponse(
-        (response) => {
-          return response.url().includes("/environment");
-        },
-        {
-          timeout: 120000,
-        }
-      );
-
-      expect(syncApi.status()).toBe(200);
-
-      // Formbricks Modal exists in the DOM
-      await expect(page.locator("#formbricks-modal-container")).toHaveCount(1);
-
-      // Formbricks Modal is visible
-      await expect(
-        page.locator("#questionCard-0").getByRole("link", { name: "Powered by Formbricks" })
-      ).toBeVisible();
-
-      // Fill the Survey
-      await page.getByRole("button", { name: "Happy to help!" }).click();
-      await page.locator("label").filter({ hasText: "Somewhat disappointed" }).click();
-      await page.locator("#questionCard-1").getByRole("button", { name: "Next" }).click();
-      await page.locator("label").filter({ hasText: "Founder" }).click();
-      await page.locator("#questionCard-2").getByRole("button", { name: "Next" }).click();
-      await page
-        .locator("#questionCard-3")
-        .getByLabel("textarea")
-        .fill("People who believe that PMF is necessary");
-      await page.locator("#questionCard-3").getByRole("button", { name: "Next" }).click();
-      await page.locator("#questionCard-4").getByLabel("textarea").fill("Much higher response rates!");
-      await page.locator("#questionCard-4").getByRole("button", { name: "Next" }).click();
-      await page.locator("#questionCard-5").getByLabel("textarea").fill("Make this end to end test pass!");
-      await page.getByRole("button", { name: "Finish" }).click();
-
-      // loading spinner -> wait for it to disappear
-      await page.getByTestId("loading-spinner").waitFor({ state: "hidden" });
-      await page.waitForLoadState("networkidle");
-    });
+    await expect(page.locator("#howToSendCardTrigger")).toBeVisible();
+    await page.locator("#howToSendCardTrigger").click();
+    await expect(page.locator("#howToSendCardOption-app")).toBeVisible();
+    await page.locator("#howToSendCardOption-app").click();
 
-    await test.step("Admin validates Displays & Response", async () => {
-      await page.goto("/");
-      await page.waitForURL(/\/environments\/[^/]+\/surveys/);
+    await page.locator("#whenToSendCardTrigger").click();
+    await page.getByRole("button", { name: "Add action" }).click();
+    await page.getByText("New SessionGets fired when a").click();
 
-      await page.getByRole("link", { name: "product Market Fit (Superhuman)" }).click();
-      (await page.waitForSelector("text=Responses")).isVisible();
+    await page.locator("#recontactOptionsCardTrigger").click();
+    await page.locator("label").filter({ hasText: "Keep showing while conditions" }).click();
+    await page.locator("#recontactDays").check();
 
-      await page.waitForLoadState("networkidle");
-      await page.waitForTimeout(2000);
+    await page.getByRole("button", { name: "Publish" }).click();
 
-      const impressionsCount = await page.getByRole("button", { name: "Impressions" }).innerText();
-      expect(impressionsCount).toEqual("Impressions\n\n1");
+    await page.waitForURL(/\/environments\/[^/]+\/surveys\/[^/]+\/summary/);
 
-      await expect(page.getByRole("link", { name: "Responses (1)" })).toBeVisible();
-      await expect(page.getByRole("button", { name: "Completed 100%" })).toBeVisible();
+    // No need for file operations anymore, just use the server
+    await page.goto("http://localhost:3004");
 
-      await expect(page.getByText("1 Responses", { exact: true }).first()).toBeVisible();
-      await expect(page.getByText("CTR100%")).toBeVisible();
-      await expect(page.getByText("Somewhat disappointed100%")).toBeVisible();
-      await expect(page.getByText("Founder100%")).toBeVisible();
-      await expect(page.getByText("People who believe that PMF").first()).toBeVisible();
-      await expect(page.getByText("Much higher response rates!").first()).toBeVisible();
-      await expect(page.getByText("Make this end to end test").first()).toBeVisible();
+    const syncApi = await page.waitForResponse((response) => response.url().includes("/environment"), {
+      timeout: 120000,
     });
+    expect(syncApi.status()).toBe(200);
+
+    await expect(page.locator("#formbricks-modal-container")).toHaveCount(1);
+    await expect(
+      page.locator("#questionCard-0").getByRole("link", { name: "Powered by Formbricks" })
+    ).toBeVisible();
+
+    // Fill the survey
+    await page.getByRole("button", { name: "Happy to help!" }).click();
+    await page.locator("label").filter({ hasText: "Somewhat disappointed" }).click();
+    await page.locator("#questionCard-1").getByRole("button", { name: "Next" }).click();
+    await page.locator("label").filter({ hasText: "Founder" }).click();
+    await page.locator("#questionCard-2").getByRole("button", { name: "Next" }).click();
+    await page
+      .locator("#questionCard-3")
+      .getByLabel("textarea")
+      .fill("People who believe that PMF is necessary");
+    await page.locator("#questionCard-3").getByRole("button", { name: "Next" }).click();
+    await page.locator("#questionCard-4").getByLabel("textarea").fill("Much higher response rates!");
+    await page.locator("#questionCard-4").getByRole("button", { name: "Next" }).click();
+    await page.locator("#questionCard-5").getByLabel("textarea").fill("Make this end to end test pass!");
+    await page.getByRole("button", { name: "Finish" }).click();
+
+    await page.getByTestId("loading-spinner").waitFor({ state: "hidden" });
+    await page.waitForLoadState("networkidle");
+
+    // Validate displays and response
+    await page.goto("/");
+    await page.waitForURL(/\/environments\/[^/]+\/surveys/);
+    await page.getByRole("link", { name: "product Market Fit (Superhuman)" }).click();
+    await page.waitForSelector("text=Responses");
+    await page.waitForLoadState("networkidle");
+    await page.waitForTimeout(2000);
+
+    const impressionsCount = await page.getByRole("button", { name: "Impressions" }).innerText();
+    expect(impressionsCount).toEqual("Impressions\n\n1");
+
+    await expect(page.getByRole("link", { name: "Responses (1)" })).toBeVisible();
+    await expect(page.getByRole("button", { name: "Completed 100%" })).toBeVisible();
+    await expect(page.getByText("1 Responses", { exact: true }).first()).toBeVisible();
+    await expect(page.getByText("CTR100%")).toBeVisible();
+    await expect(page.getByText("Somewhat disappointed100%")).toBeVisible();
+    await expect(page.getByText("Founder100%")).toBeVisible();
+    await expect(page.getByText("People who believe that PMF").first()).toBeVisible();
+    await expect(page.getByText("Much higher response rates!").first()).toBeVisible();
+    await expect(page.getByText("Make this end to end test").first()).toBeVisible();
   });
 });