Create and configure new actions workflow for E2E tests

.github/workflows/e2e.yaml

@@ -0,0 +1,230 @@
+name: E2E Tests
+on:
+  workflow_call:
+    secrets:
+      CONNECT_LICENSE:
+        required: true
+
+jobs:
+  cypress:
+    name: e2e tests (cypress)
+    runs-on: ubuntu-latest
+    env:
+      DEBUG_CYPRESS: "false" # Set to true to enable extra logging and video recording replays
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: pip
+          cache-dependency-path: test/e2e/requirements.txt
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: test/e2e/package-lock.json
+
+      - name: Install Python dependencies
+        working-directory: test/e2e
+        run: pip install -r requirements.txt
+
+      - name: Install npm dependencies
+        working-directory: test/e2e
+        run: npm ci
+
+      - uses: extractions/setup-just@v2
+
+      - name: Build publisher binary
+        run: just build
+
+      - name: Download VSIX artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist
+
+      - name: Write Connect license file
+        if: env.CONNECT_LICENSE != ''
+        run: |
+          mkdir -p ./test/e2e/licenses
+          echo "$CONNECT_LICENSE" > ./test/e2e/licenses/connect-license.lic
+        env:
+          CONNECT_LICENSE: ${{ secrets.CONNECT_LICENSE }}
+
+      - name: Build Docker images
+        working-directory: test/e2e
+        run: |
+          docker build --build-arg GH_DOWNLOAD_TOKEN=${{ secrets.GH_DOWNLOAD_TOKEN }} -f Dockerfile.base -t e2ebase --platform linux/amd64 .
+          docker compose build connect-publisher-e2e code-server
+        env:
+          CONNECT_LICENSE: ${{ secrets.CONNECT_LICENSE }}
+
+      - name: "[DEBUG] Print host user and group info"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        run: |
+          echo "USER: $(whoami)"
+          echo "UID: $(id -u)"
+          echo "GID: $(id -g)"
+          id
+          ls -ld .
+
+      - name: Ensure content-workspace exists
+        run: mkdir -p ./test/e2e/content-workspace
+
+      - name: Start containers
+        working-directory: test/e2e
+        run: |
+          just start "connect-publisher-e2e"
+          just start "code-server"
+
+      - name: Wait for Connect server to be ready
+        run: |
+          echo "Waiting for Connect server to be ready..."
+          for i in {1..30}; do
+            if curl -sf http://localhost:3939/__ping__ > /dev/null; then
+              echo "Connect server is ready!"
+              break
+            fi
+            if [ $i -eq 30 ]; then
+              echo "WARNING: Timeout waiting for Connect server to be ready"
+              # Continue anyway, don't exit with error
+            fi
+            echo "Waiting for Connect server (attempt $i/30)..."
+            sleep 2
+          done
+
+      - name: Stop any processes using port 8080
+        run: |
+          PID=$(docker exec publisher-e2e.code-server sh -c "lsof -ti :8080 || netstat -tulnp 2>/dev/null | grep ':8080' | awk '{print \$7}' | cut -d'/' -f1 | head -n1" || true)
+          if [ -n "$PID" ]; then
+            echo "Killing process $PID using port 8080"
+            docker exec publisher-e2e.code-server kill -9 $PID || true
+          else
+            echo "No process using port 8080"
+          fi
+
+      - name: Wait for code-server to be ready
+        run: |
+          for i in {1..30}; do
+            if curl -sf http://localhost:8080 > /dev/null; then
+              echo "code-server is ready!"
+              break
+            fi
+            if [ $i -eq 30 ]; then
+              echo "code-server did not become ready in time"
+              exit 1
+            fi
+            sleep 2
+          done
+
+      - name: Install and wait for VS Code extension to be ready
+        working-directory: test/e2e
+        run: |
+          # Install the Publisher extension
+          VSIX_FILENAME=$(ls -Art ../../dist | grep linux-amd64 | tail -n 1)
+          docker compose exec code-server code-server --install-extension "/home/coder/vsix/${VSIX_FILENAME}"
+
+          # Wait for the extension to be installed (more reliable than process grep)
+          echo "Waiting for Publisher extension to be installed..."
+          for i in {1..60}; do
+            if docker compose exec code-server code-server --list-extensions | grep -q posit.publisher; then
+              echo "Publisher extension appears to be installed!"
+              break
+            fi
+            if [ $i -eq 60 ]; then
+              echo "WARNING: Timeout waiting for Publisher extension to be installed"
+            fi
+            echo "Waiting for extension installation (attempt $i/60)..."
+            sleep 2
+          done
+
+      - name: "[DEBUG] Print workspace file ownership before tests"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        run: |
+          docker exec publisher-e2e.code-server ls -lR /home/coder/workspace || true
+
+      - name: Prepare test environment
+        working-directory: test/e2e
+        run: |
+          # Clean up any static TOML files before tests
+          docker exec publisher-e2e.code-server rm -f /home/coder/workspace/static*.toml || true
+
+      - name: Run Cypress tests
+        working-directory: test/e2e
+        run: npx cypress run
+        env:
+          CONNECT_LICENSE: ${{ secrets.CONNECT_LICENSE }}
+          CONNECT_CLOUD_ENV: ${{ env.CONNECT_CLOUD_ENV }} # defaults to staging
+          CI: true
+          DEBUG_CYPRESS: ${{ env.DEBUG_CYPRESS }}
+          ACTIONS_STEP_DEBUG: ${{ runner.debug && 'true' || 'false' }}
+
+      - name: Upload Cypress screenshots on failure
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: cypress-screenshots
+          path: test/e2e/cypress/screenshots
+          if-no-files-found: ignore
+
+      - name: "[DEBUG] Upload Cypress videos"
+        uses: actions/upload-artifact@v4
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        with:
+          name: cypress-videos
+          path: test/e2e/cypress/videos
+          if-no-files-found: ignore
+
+      - name: "[DEBUG] List installed VS Code extensions"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        working-directory: test/e2e
+        run: docker compose exec code-server code-server --list-extensions --show-versions || true
+
+      - name: "[DEBUG] Display test/e2e directory tree"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        run: |
+          echo "Directory tree for test/e2e:"
+          if command -v tree > /dev/null; then
+            tree -I "node_modules|.venv" test/e2e
+          else
+            find test/e2e -path "*/node_modules" -prune -o -path "*/.venv" -prune -o -print
+          fi
+
+      - name: "[DEBUG] Print code-server logs"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        run: |
+          echo "=== Code Server Log Files ==="
+          docker exec publisher-e2e.code-server bash -c 'find /root/.local -type f -name "*.log" 2>/dev/null | while read -r file; do echo "FILE: $file"; done'
+          echo "=== Main Code Server Logs (sample) ==="
+          docker exec publisher-e2e.code-server bash -c 'cat /root/.local/share/code-server/coder-logs/code-server-*.log 2>/dev/null | tail -n 50 || echo "No logs found"'
+          echo "=== Extension Host Logs (sample) ==="
+          docker exec publisher-e2e.code-server bash -c 'find /root/.local -name "remoteexthost.log" -type f 2>/dev/null | head -n 1 | xargs cat | tail -n 50 || echo "No logs found"'
+
+      - name: "[DEBUG] Print Publisher extension logs"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        run: |
+          echo "=== Finding Publisher logs ==="
+          docker exec publisher-e2e.code-server bash -c 'find /root -type f -name "*.log" 2>/dev/null | grep -i "posit\|publisher" || echo "No publisher logs found"'
+          docker exec publisher-e2e.code-server bash -c 'find /home -type f -name "*.log" 2>/dev/null | grep -i "posit\|publisher" || echo "No publisher logs found"'
+          echo "======================================="
+
+      - name: "[DEBUG] Print code-server container logs"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        working-directory: test/e2e
+        run: docker compose logs code-server || true
+
+      - name: "[DEBUG] List /home/coder directory contents"
+        if: always() && (env.DEBUG_CYPRESS == 'true' || runner.debug)
+        run: docker exec publisher-e2e.code-server ls -lR /home/coder || true
+
+      - name: Stop containers
+        if: always()
+        working-directory: test/e2e
+        run: just stop

.github/workflows/pull-request.yaml

@@ -16,6 +16,11 @@ jobs:
     uses: ./.github/workflows/agent.yaml
   vscode:
     uses: ./.github/workflows/vscode.yaml
+  # E2E Tests
+  e2e:
+    needs: [home-view-unit-tests, agent, vscode, package]
+    uses: ./.github/workflows/e2e.yaml
+    secrets: inherit
 
   # Build
   build:

.gitignore

@@ -10,6 +10,10 @@ node_modules
 
 .DS_Store
 
+*renv/
+*.venv/
+.envrc
+
 # Visual Studio Code
 .vscode/*
 !.vscode/recommended.settings.json
@@ -31,6 +35,7 @@ cmd/publisher/__debug_bin*
 
 # Possible e2e tests deployment assets
 test/e2e/content-workspace/**/.posit
+screenshots/
 
 # license files should not be commited to this repository
 *.lic

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added Cypress E2E CI Setup and Test Reliability Improvements (#2721)
 - Added endpoints for performing OAuth Device Authorization Grant with Posit Cloud login. (#2692)
 - Added support for one-click token authentication with Connect. (#2769)
 - Added schema and agent support for publishing to Connect Cloud. (#2729, #2747, #2771)

CONTRIBUTING.md

@@ -94,7 +94,9 @@ Once complete, a coverage report will open in your default browser.
 
 End-to-end tests are written in JavaScript and utilize Cypress for testing the Posit Publisher VSCode extension.
 
-They are not currently run within the CI pipeline, but can be run locally to verify that the extension works as expected in a Connect environment.
+These tests can be run locally to verify that the extension works as expected in a Connect environment.
+
+These tests also run automatically in the GitHub Actions CI pipeline for pull requests after the unit tests have passed. The workflow uses the `CONNECT_LICENSE` secret stored in the repository settings to authenticate with Connect during testing. Results, including screenshots of failed tests of test runs, are uploaded as artifacts for troubleshooting. (Video replays can be optionally enabled by setting local environment variable `DEBUG_CYPRESS` or `ACTIONS_STEP_DEBUG` to true).
 
 #### Requirements
 
@@ -190,6 +192,14 @@ just stop
 **NOTE: ** If you are updating the images in any way, where you need to rebuild the images with `just build-images`,
 you will need to run the `just stop` command to remove the existing containers before running `just dev`.
 
+#### Repeat Tests Headless Script
+
+Allows you to run your Cypress E2E tests multiple times in headless mode, either for all tests or for specific test files. It is useful for detecting flaky tests and verifying test suite stability.
+
+```bash
+./repeat-cypress-headless.sh [REPEAT=N] [spec1] [spec2] [...]
+```
+
 ## Development
 
 ### Build Tools

test/e2e/Dockerfile.base

@@ -123,34 +123,41 @@ RUN curl -O https://cdn.rstudio.com/r/${OS_IDENTIFIER}/pkgs/r-${R_VERSION}_1_amd
 
 FROM base AS quarto-1.4.556
 ENV QUARTO_VERSION=1.4.556
-# from https://docs.posit.co/resources/install-quarto.html#download-install-quarto-tar
+ARG GH_DOWNLOAD_TOKEN
 RUN mkdir -p /opt/quarto/${QUARTO_VERSION} \
-    && curl -o quarto.tar.gz -L \
-        "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && curl -H "Authorization: token ${GH_DOWNLOAD_TOKEN}" -fSL -o quarto.tar.gz \
+    "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && file quarto.tar.gz | grep 'gzip compressed data' \
     && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \
     && rm quarto.tar.gz
 
 FROM base AS quarto-1.5.52
 ENV QUARTO_VERSION=1.5.52
+ARG GH_DOWNLOAD_TOKEN
 RUN mkdir -p /opt/quarto/${QUARTO_VERSION} \
-    && curl -o quarto.tar.gz -L \
-        "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && curl -H "Authorization: token ${GH_DOWNLOAD_TOKEN}" -fSL -o quarto.tar.gz \
+    "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && file quarto.tar.gz | grep 'gzip compressed data' \
     && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \
     && rm quarto.tar.gz
 
 FROM base AS quarto-1.6.42
 ENV QUARTO_VERSION=1.6.42
+ARG GH_DOWNLOAD_TOKEN
 RUN mkdir -p /opt/quarto/${QUARTO_VERSION} \
-    && curl -o quarto.tar.gz -L \
-        "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && curl -H "Authorization: token ${GH_DOWNLOAD_TOKEN}" -fSL -o quarto.tar.gz \
+    "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && file quarto.tar.gz | grep 'gzip compressed data' \
     && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \
     && rm quarto.tar.gz
 
 FROM base AS quarto-1.7.6
 ENV QUARTO_VERSION=1.7.6
+ARG GH_DOWNLOAD_TOKEN
 RUN mkdir -p /opt/quarto/${QUARTO_VERSION} \
-    && curl -o quarto.tar.gz -L \
-        "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && curl -H "Authorization: token ${GH_DOWNLOAD_TOKEN}" -fSL -o quarto.tar.gz \
+    "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz" \
+    && file quarto.tar.gz | grep 'gzip compressed data' \
     && tar -zxvf quarto.tar.gz -C "/opt/quarto/${QUARTO_VERSION}" --strip-components=1 \
     && rm quarto.tar.gz
 
@@ -195,4 +202,4 @@ ARG QUARTO_VERSION=1.4.556
 RUN ln -s /opt/quarto/${QUARTO_VERSION}/bin/quarto /usr/bin/quarto
 
 # clean up from our apt-update
-RUN rm -rf /var/lib/apt/lists/* 
+RUN rm -rf /var/lib/apt/lists/*

test/e2e/code-server-entry.sh

@@ -20,5 +20,5 @@ cat <<EOF > /home/coder/.local/share/code-server/User/settings.json
 
 EOF
 
-# Run the original code server entrypoint that starts the service
-/usr/bin/code-server --disable-workspace-trust --auth none --bind-addr 0.0.0.0:8080 .
+# Code server entrypoint that starts the service
+exec /usr/bin/code-server --disable-workspace-trust --auth none --bind-addr 0.0.0.0:8080 .