WIP: Run E2E against Grafana Cloud

Author: adamyeats

.github/workflows/cloud-e2e.yml

@@ -0,0 +1,121 @@
+name: Grafana Cloud E2E tests
+
+on:
+  schedule:
+    - cron: '0 9 * * *' # Daily at 09:00 UTC
+
+  # TODO: test, remove later
+  push:
+    branches:
+      - main
+  pull_request:
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  # @TODO: Remove the `build` job later
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: 'stable'
+
+      - name: Build backend
+        uses: magefile/mage-action@6f50bbb8ea47d56e62dee92392788acbc8192d0b
+        with:
+          args: buildAll
+          version: latest
+
+      - name: Install frontend dependencies
+        run: npm ci --no-audit --fund=false
+
+      - name: Build frontend
+        run: npm run build
+        env:
+          NODE_OPTIONS: '--max_old_space_size=4096'
+
+  e2e:
+    name: Bench Tests
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+          
+      - name: Setup Node.js environment
+        uses: actions/setup-node@v6
+        with:
+          node-version-file: .nvmrc
+          cache: 'npm'
+
+      - name: Get secrets from Vault
+        id: get-secrets
+        # Only execute this action in a trusted context
+        if: |
+          github.event_name != 'pull_request' || 
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: grafana/shared-workflows/actions/get-vault-secrets@get-vault-secrets/v1.3.0
+        with:
+          common_secrets: |
+            PLAYWRIGHT_GRAFANA_PASSWORD=data-sources/e2e:grafana-pw
+            PLAYWRIGHT_GRAFANA_USERNAME=data-sources/e2e:grafana-username
+          repo_secrets: |
+            DS_INSTANCE_HOST=ds-instance:host
+            DS_INSTANCE_PASSWORD=ds-instance:password
+            DS_INSTANCE_PORT=ds-instance:port
+            DS_INSTANCE_USERNAME=ds-instance:username
+          export_env: false
+
+      - name: Install npm dependencies
+        run: |
+          if [ -f yarn.lock ]; then
+            yarn install --frozen-lockfile --non-interactive
+          else
+            npm ci --no-audit --fund=false
+          fi
+
+      - name: Install Playwright Browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: Run Grafana Bench tests
+        run: |
+          set -euo pipefail 
+          docker run \
+            --rm \
+            --network=host \
+            --volume "$PWD:/tests" \
+            us-docker.pkg.dev/grafanalabs-global/docker-grafana-bench-prod/grafana-bench-playwright:v0.6.3 test \
+            --grafana-admin-password "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_PASSWORD }}" \
+            --grafana-admin-user "${{ fromJSON(steps.get-secrets.outputs.secrets).PLAYWRIGHT_GRAFANA_USERNAME }}" \
+            --grafana-url "https://datasourcese2e.grafana-dev.net" \
+            --log-level debug \
+            --pw-execute "npm run e2e" \
+            --pw-prepare "npm ci --no-audit --fund=false; npx playwright install" \
+            --test-env-vars 'CI=true DS_INSTANCE_HOST=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_HOST }} DS_INSTANCE_PASSWORD=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PASSWORD }} DS_INSTANCE_PORT=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_PORT }} DS_INSTANCE_USERNAME=${{ fromJSON(steps.get-secrets.outputs.secrets).DS_INSTANCE_USERNAME }}' \
+            --test-runner playwright
+
+      - name: Archive Playwright artifacts
+        uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.github/workflows/push.yml

@@ -16,6 +16,6 @@ jobs:
       contents: read
       id-token: write
     with:
-      plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
       go-version: '1.24'
       golangci-lint-version: '2.1.6'
+      plugin-version-suffix: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}

playwright.config.ts

@@ -1,3 +1,5 @@
+import { dirname } from 'path';
+
 import { defineConfig, devices } from '@playwright/test';
 import type { PluginOptions } from '@grafana/plugin-e2e';
 
@@ -42,14 +44,14 @@ export default defineConfig<PluginOptions>({
   projects: [
     {
       name: 'auth',
-      testDir: 'node_modules/@grafana/plugin-e2e/dist/auth',
+      testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`,
       testMatch: [/.*\.js/],
     },
     {
       name: 'run-tests',
       use: {
         ...devices['Desktop Chrome'],
-        storageState: 'playwright/.auth/admin.json',
+        storageState: `playwright/.auth/${process.env.GRAFANA_ADMIN_USER || 'admin'}.json`,
       },
       dependencies: ['auth'],
     },

tests/e2e/configEditor.spec.ts

@@ -1,21 +1,30 @@
 import { expect, test } from '@grafana/plugin-e2e';
 
 const PLUGIN_UID = 'grafana-clickhouse-datasource';
-const CLICKHOUSE_DB_URL = Boolean(process.env.CI) ? 'clickhouse-server' : 'localhost';
+
+// Determine ClickHouse DB URL based on environment
+function resolveClickhouseUrl(env = process.env) {
+  const { CI, DS_INSTANCE_HOST } = env;
+  return CI ? (DS_INSTANCE_HOST || 'clickhouse-server') : 'localhost';
+}
 
 test.describe('Config Editor', () => {
   test('invalid credentials should return an error', async ({ createDataSourceConfigPage, page }) => {
     const configPage = await createDataSourceConfigPage({ type: PLUGIN_UID });
-    await page.getByPlaceholder('Server address').fill(CLICKHOUSE_DB_URL);
+    await page.getByPlaceholder('Server address').fill(resolveClickhouseUrl());
     await expect(configPage.saveAndTest()).not.toBeOK();
   });
 
   test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage, page }) => {
     const configPage = await createDataSourceConfigPage({ type: PLUGIN_UID });
 
-    await page.getByPlaceholder('Server address').fill(CLICKHOUSE_DB_URL);
-    await page.getByPlaceholder('9000').fill('9000');
-    await page.getByPlaceholder('default').fill('default');
+    await page.getByPlaceholder('Server address').fill(resolveClickhouseUrl());
+    await page.getByPlaceholder('9000').fill(process.env.DS_INSTANCE_PORT ?? '9000');
+    await page.getByPlaceholder('default').fill(process.env.DS_INSTANCE_USERNAME ?? 'default');
+
+    if (process.env.DS_INSTANCE_PASSWORD) {
+      await page.getByPlaceholder('password').fill(process.env.DS_INSTANCE_PASSWORD);
+    }
 
     await configPage.saveAndTest();
     await expect(configPage).toHaveAlert('success', { hasNotText: 'Datasource updated' });