WIP: Run E2E against Grafana Cloud

Author: adamyeats

.github/workflows/cloud-e2e.yml

@@ -0,0 +1,141 @@
+name: Grafana Cloud E2E tests
+
+on:
+  schedule:
+    - cron: '0 9 * * *' # Daily at 09:00 UTC
+
+  # TODO: test, remove later
+  push:
+    branches:
+      - main
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  # @TODO: Remove the `build` job later
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: step-security/harden-runner@v2 # zizmor: ignore[unpinned-uses]
+        with:
+          egress-policy: audit
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: 'stable'
+
+      - name: Build backend
+        uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b
+        with:
+          args: buildAll
+          version: latest
+
+      - name: Install frontend dependencies
+        run: npm ci --no-audit --fund=false
+
+      - name: Build frontend
+        run: npm run build
+        env:
+          NODE_OPTIONS: '--max_old_space_size=4096'
+
+      - name: Upload dist artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          if-no-files-found: error
+          name: dist-artifacts
+          path: dist
+          retention-days: 2
+
+  e2e:
+    name: Bench Tests
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: step-security/harden-runner@v2 # zizmor: ignore[unpinned-uses]
+        with:
+          egress-policy: audit
+          
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+          cache: 'npm'
+      
+      - name: Download GitHub artifact
+        uses: actions/download-artifact@v5
+        with:
+          name: dist-artifacts
+          path: dist
+
+      - name: Get secrets from Vault
+        id: get-secrets
+        # Only execute this action in a trusted context
+        if: |
+          github.event_name != 'pull_request' || 
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: grafana/shared-workflows/actions/get-vault-secrets@get-vault-secrets/v1.3.0
+        with:
+          common_secrets: |
+            PLAYWRIGHT_GRAFANA_PASSWORD=data-sources/e2e:grafana-pw
+            PLAYWRIGHT_GRAFANA_USERNAME=data-sources/e2e:grafana-username
+          repo_secrets: |
+            DS_INSTANCE_HOST=ds-instance:host
+            DS_INSTANCE_PASSWORD=ds-instance:password
+            DS_INSTANCE_PORT=ds-instance:port
+            DS_INSTANCE_USERNAME=ds-instance:username
+          export_env: false
+
+      - name: Install npm dependencies
+        run: |
+          if [ -f yarn.lock ]; then
+            yarn install --frozen-lockfile --non-interactive
+          else
+            npm ci --no-audit --fund=false
+          fi
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: Wait for Grafana to start
+        uses: grafana/plugin-actions/wait-for-grafana@wait-for-grafana/v1.0.2
+        with:
+          url: "https://datasourcese2e.grafana-dev.net/login"
+
+      - name: Run Grafana Bench tests
+        run: |
+          set -euo pipefail 
+          docker run \
+            --rm \
+            --network=host \
+            --volume "$PWD:/tests" \
+            us-docker.pkg.dev/grafanalabs-global/docker-grafana-bench-prod/grafana-bench-playwright:v0.6.3 test \
+            --grafana-admin-password "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_PASSWORD }}" \
+            --grafana-admin-user "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_USERNAME }}" \
+            --grafana-url "https://datasourcese2e.grafana-dev.net" \
+            --log-level debug \
+            --pw-execute "npm run e2e" \
+            --pw-prepare "npm install --frozen-lockfile; npx playwright install" \
+            --test-env-vars "CI=true" \
+            --test-runner playwright
+

.github/workflows/push.yml

@@ -16,6 +16,6 @@ jobs:
       contents: read
       id-token: write
     with:
-      plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
       go-version: '1.24'
       golangci-lint-version: '2.1.6'
+      plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}

playwright.config.ts

@@ -1,3 +1,5 @@
+import { dirname } from 'path';
+
 import { defineConfig, devices } from '@playwright/test';
 import type { PluginOptions } from '@grafana/plugin-e2e';
 
@@ -42,14 +44,14 @@ export default defineConfig<PluginOptions>({
   projects: [
     {
       name: 'auth',
-      testDir: 'node_modules/@grafana/plugin-e2e/dist/auth',
+      testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`,
       testMatch: [/.*\.js/],
     },
     {
       name: 'run-tests',
       use: {
         ...devices['Desktop Chrome'],
-        storageState: 'playwright/.auth/admin.json',
+        storageState: `playwright/.auth/${process.env.GRAFANA_ADMIN_USER || 'admin'}.json`,
       },
       dependencies: ['auth'],
     },