Merge pull request #8406 from sagemathinc/github-tests-report-status

github test dorny/test-reporter

Author: williamstein

.github/workflows/make-and-test.yml

@@ -1,4 +1,4 @@
-name: Make all packages and run their tests
+name: Test
 
 # newer commits in the same PR abort running ones for the same workflow
 concurrency:
@@ -43,7 +43,7 @@ jobs:
       - name: Check doc links
         run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py
 
-  build:
+  test:
     runs-on: ubuntu-latest
 
     services:
@@ -94,7 +94,7 @@ jobs:
       - name: Download and install Valkey
         run: |
           VALKEY_VERSION=8.1.2
-          curl -LO https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
+          curl -LOq https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
           tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz
           sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/
 
@@ -106,6 +106,52 @@ jobs:
           pip install ipykernel
           python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)"
 
-      - run: cd src && npm install -g pnpm
-      - run: cd src && pnpm run make
-      - run: source venv/bin/activate && cd src && pnpm run test-github-ci
+
+      - name: install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+          run_install: false
+
+      - name: build
+        run: cd src && pnpm run make
+
+      # This runs all the tests with text output and jest-junit reporters to generate junit.xml reports
+      # That test-github-ci target runs workspaces.py for testing, which in turn runs the depchecks
+      - name: test
+        run: source venv/bin/activate && cd src && pnpm run test-github-ci --report
+
+      - name: upload test results
+        uses: actions/upload-artifact@v4  # upload test results
+        if: ${{ !cancelled() }}           # run this step even if previous step failed
+        with:
+          name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}"
+          path: 'src/packages/*/junit.xml'
+
+  report:
+    runs-on: ubuntu-latest
+
+    needs: [test]
+
+    if: ${{ !cancelled() }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download all test artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "test-results-*"
+          merge-multiple: true
+          path: test-results/
+
+      - name: Test Report
+        uses: dorny/test-reporter@v2
+        with:
+          name: CoCalc Jest Tests
+          path: 'test-results/**/junit.xml'
+          reporter: jest-junit
+          use-actions-summary: 'true'
+          fail-on-error: false
+

.gitignore

@@ -159,3 +159,6 @@ src/packages/frontend/i18n/trans/*.compiled.json
 **/*.db
 **/project-env.sh
 **/*.bash_history
+
+# test reports by jest-junit
+junit.xml

src/CLAUDE.md

@@ -1,18 +1,19 @@
-# CLAUDE.md
+# CLAUDE.md and GEMINI.md
 
-This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+This file provides guidance to Claude Code (claude.ai/code) and also Gemini CLI (https://github.com/google-gemini/gemini-cli) when working with code in this repository.
 
 # CoCalc Source Repository
 
-* This is the source code of CoCalc in a Git repository
-* It is a complex JavaScript/TypeScript SaaS application
-* CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
-* The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
+- This is the source code of CoCalc in a Git repository
+- It is a complex JavaScript/TypeScript SaaS application
+- CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
+- The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
 
 ## Code Style
 
 - Everything is written in TypeScript code
 - Indentation: 2-spaces
+- Run `pretter -w [filename]` after modifying a file (ts, tsx, md, json, ...) to format it correctly.
 - All .js and .ts files are formatted by the tool prettier
 - Add suitable types when you write code
 - Variable name styles are "camelCase" for local and "FOO_BAR" for global variables. If you edit older code not following these guidlines, adjust this rule to fit the files style.
@@ -23,28 +24,32 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 ## Development Commands
 
 ### Essential Commands
+
 - `pnpm build-dev` - Build all packages for development
 - `pnpm clean` - Clean all node_modules and dist directories
-- `pnpm database` - Start PostgreSQL database server
-- `pnpm hub` - Start the main hub server
-- `pnpm psql` - Connect to the PostgreSQL database
 - `pnpm test` - Run full test suite
-- `pnpm test-parallel` - Run tests in parallel across packages
 - `pnpm depcheck` - Check for dependency issues
+- `prettier -w [filename]` to format the style of a file after editing it
+- after creating a file, run `git add [filename]` to start tracking it
 
 ### Package-Specific Commands
-- `cd packages/[package] && pnpm tsc` - Watch TypeScript compilation for a specific package
+
+- `cd packages/[package] && pnpm build` - Build and compile a specific package
+  - for packages/next and packages/static, run `cd packages/[package] && pnpm build-dev`
+- `cd packages/[package] && pnpm tsc:watch` - TypeScript compilation in watch mode for a specific package
 - `cd packages/[package] && pnpm test` - Run tests for a specific package
 - `cd packages/[package] && pnpm build` - Build a specific package
+- **IMPORTANT**: When modifying packages like `util` that other packages depend on, you must run `pnpm build` in the modified package before typechecking dependent packages
 
-### Development Setup
-1. Start database: `pnpm database`
-2. Start hub: `pnpm hub`
-3. For TypeScript changes, run `pnpm tsc` in the relevant package directory
+### Development
+
+- **IMPORTANT**: Always run `prettier -w [filename]` immediately after editing any .ts, .tsx, .md, or .json file to ensure consistent styling
+- After TypeScript or `*.tsx` changes, run `pnpm build` in the relevant package directory
 
 ## Architecture Overview
 
 ### Package Structure
+
 CoCalc is organized as a monorepo with key packages:
 
 - **frontend** - React/TypeScript frontend application using Redux-style stores and actions
@@ -62,25 +67,30 @@ CoCalc is organized as a monorepo with key packages:
 ### Key Architectural Patterns
 
 #### Frontend Architecture
+
 - **Redux-style State Management**: Uses custom stores and actions pattern (see `packages/frontend/app-framework/actions-and-stores.ts`)
 - **TypeScript React Components**: All frontend code is TypeScript with proper typing
 - **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
 - **WebSocket Communication**: Real-time communication with backend via WebSocket messages
 
 #### Backend Architecture
+
 - **PostgreSQL Database**: Primary data store with sophisticated querying system
 - **WebSocket Messaging**: Real-time communication between frontend and backend
 - **Conat System**: Container orchestration for compute servers
 - **Event-Driven Architecture**: Extensive use of EventEmitter patterns
 - **Microservice-like Packages**: Each package handles specific functionality
 
 #### Communication Patterns
+
 - **WebSocket Messages**: Primary communication method (see `packages/comm/websocket/types.ts`)
 - **Database Queries**: Structured query system with typed interfaces
 - **Event Emitters**: Inter-service communication within backend
 - **REST-like APIs**: Some HTTP endpoints for specific operations
+- **API Schema**: API endpoints in `packages/next/pages/api/v2/` use Zod schemas in `packages/next/lib/api/schema/` for validation. These schemas must be kept in harmony with the TypeScript types sent from frontend applications using `apiPost` (in `packages/next/lib/api/post.ts`) or `api` (in `packages/frontend/client/api.ts`). When adding new fields to API requests, both the frontend types and the API schema validation must be updated.
 
 ### Key Technologies
+
 - **TypeScript**: Primary language for all new code
 - **React**: Frontend framework
 - **PostgreSQL**: Database
@@ -91,40 +101,56 @@ CoCalc is organized as a monorepo with key packages:
 - **SASS**: CSS preprocessing
 
 ### Database Schema
+
 - Comprehensive schema in `packages/util/db-schema`
 - Query abstractions in `packages/database/postgres/`
 - Type-safe database operations with TypeScript interfaces
 
 ### Testing
+
 - **Jest**: Primary testing framework
 - **ts-jest**: TypeScript support for Jest
 - **jsdom**: Browser environment simulation for frontend tests
 - Test files use `.test.ts` or `.spec.ts` extensions
 - Each package has its own jest.config.js
 
 ### Import Patterns
+
 - Use absolute imports with `@cocalc/` prefix for cross-package imports
 - Example: `import { cmp } from "@cocalc/util/misc"`
 - Type imports: `import type { Foo } from "./bar"`
 - Destructure imports when possible
 
 ### Development Workflow
-1. Changes to TypeScript require compilation (`pnpm tsc` in relevant package)
+
+1. Changes to TypeScript require compilation (`pnpm build` in relevant package)
 2. Database must be running before starting hub
 3. Hub coordinates all services and should be restarted after changes
 4. Use `pnpm clean && pnpm build-dev` when switching branches or after major changes
 
 # Workflow
-- Be sure to typecheck when you're done making a series of code changes
+
+- Be sure to build when you're done making a series of code changes
 - Prefer running single tests, and not the whole test suite, for performance
 
 ## Git Workflow
 
+- Never modify a file when in the `master` or `main` branch
+- All changes happen through feature branches, which are pushed as pull requests to GitHub
+- When creating a new file, run `git add [filename]` to track the file.
 - Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code.
 - When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo".
 
-# important-instruction-reminders
+# Important Instruction Reminders
+
 - Do what has been asked; nothing more, nothing less.
 - NEVER create files unless they're absolutely necessary for achieving your goal.
 - ALWAYS prefer editing an existing file to creating a new one.
-- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.
+- REFUSE to modify files when the git repository is on the `master` or `main` branch.
+- NEVER proactively create documentation files (`*.md`) or README files. Only create documentation files if explicitly requested by the User.
+
+# Ignore
+
+- Ignore files covered by `.gitignore`
+- Ignore everything in `node_modules` or `dist` directories
+- Ignore all files not tracked by Git, unless they are newly created files

src/GEMINI.md

@@ -0,0 +1 @@
+CLAUDE.md

src/packages/backend/package.json

@@ -10,7 +10,10 @@
     "./auth/*": "./dist/auth/*.js",
     "./auth/tokens/*": "./dist/auth/tokens/*.js"
   },
-  "keywords": ["utilities", "cocalc"],
+  "keywords": [
+    "utilities",
+    "cocalc"
+  ],
   "scripts": {
     "preinstall": "npx only-allow pnpm",
     "clean": "rm -rf dist node_modules",
@@ -28,7 +31,12 @@
     "conat-persist": "DEBUG=cocalc:*  node ./bin/conat-persist.cjs",
     "conat-test-server": "node ./bin/conat-test-server.cjs"
   },
-  "files": ["dist/**", "bin/**", "README.md", "package.json"],
+  "files": [
+    "dist/**",
+    "bin/**",
+    "README.md",
+    "package.json"
+  ],
   "author": "SageMath, Inc.",
   "license": "SEE LICENSE.md",
   "dependencies": {

src/packages/hub/package.json

@@ -64,6 +64,7 @@
     "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem",
     "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=80 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0",
     "tsc": "tsc --watch  --pretty --preserveWatchOutput",
+    "test": "jest dist/",
     "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'",
     "prepublishOnly": "test"
   },

src/packages/package.json

@@ -8,6 +8,7 @@
     "@types/jest": "^29.5.14",
     "check-dependency-version-consistency": "^5.0.0",
     "jest": "^29.7.0",
+    "jest-junit": "^16.0.0",
     "ts-jest": "^29.2.3",
     "typescript": "^5.7.3"
   },