Merge branch 'master' into i18n-pt_br-eu

Author: haraldschilly

src/.claude/settings.local.json

@@ -0,0 +1,22 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(pnpm tsc:*)",
+      "Bash(pnpm build:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(node:*)",
+      "Bash(grep:*)",
+      "Bash(find:*)",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:cocalc.com)",
+      "WebFetch(domain:doc.cocalc.com)",
+      "Bash(npm show:*)",
+      "Bash(prettier -w:*)",
+      "Bash(npx tsc:*)",
+      "Bash(gh pr view:*)",
+      "Bash(gh:*)"
+    ],
+    "deny": []
+  }
+}

src/CLAUDE.md

@@ -0,0 +1,130 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+# CoCalc Source Repository
+
+* This is the source code of CoCalc in a Git repository
+* It is a complex JavaScript/TypeScript SaaS application
+* CoCalc is organized as a monorepository (multi-packages) in the subdirectory "./packages"
+* The packages are managed as a pnpm workspace in "./packages/pnpm-workspace.yaml"
+
+## Code Style
+
+- Everything is written in TypeScript code
+- Indentation: 2-spaces
+- All .js and .ts files are formatted by the tool prettier
+- Add suitable types when you write code
+- Variable name styles are "camelCase" for local and "FOO_BAR" for global variables. If you edit older code not following these guidlines, adjust this rule to fit the files style.
+- Some older code is JavaScript or CoffeeScript, which will be translated to TypeScript
+- Use ES modules (import/export) syntax, not CommonJS (require)
+- Organize the list of imports in such a way: installed npm packages are on top, newline, then are imports from @cocalc's code base. Sorted alphabetically.
+
+## Development Commands
+
+### Essential Commands
+- `pnpm build-dev` - Build all packages for development
+- `pnpm clean` - Clean all node_modules and dist directories
+- `pnpm database` - Start PostgreSQL database server
+- `pnpm hub` - Start the main hub server
+- `pnpm psql` - Connect to the PostgreSQL database
+- `pnpm test` - Run full test suite
+- `pnpm test-parallel` - Run tests in parallel across packages
+- `pnpm depcheck` - Check for dependency issues
+
+### Package-Specific Commands
+- `cd packages/[package] && pnpm tsc` - Watch TypeScript compilation for a specific package
+- `cd packages/[package] && pnpm test` - Run tests for a specific package
+- `cd packages/[package] && pnpm build` - Build a specific package
+
+### Development Setup
+1. Start database: `pnpm database`
+2. Start hub: `pnpm hub`
+3. For TypeScript changes, run `pnpm tsc` in the relevant package directory
+
+## Architecture Overview
+
+### Package Structure
+CoCalc is organized as a monorepo with key packages:
+
+- **frontend** - React/TypeScript frontend application using Redux-style stores and actions
+- **backend** - Node.js backend services and utilities
+- **hub** - Main server orchestrating the entire system
+- **database** - PostgreSQL database layer with queries and schema
+- **util** - Shared utilities and types used across packages
+- **comm** - Communication layer including WebSocket types
+- **conat** - CoCalc's container/compute orchestration system
+- **sync** - Real-time synchronization system for collaborative editing
+- **project** - Project-level services and management
+- **static** - Static assets and build configuration
+- **next** - Next.js server components
+
+### Key Architectural Patterns
+
+#### Frontend Architecture
+- **Redux-style State Management**: Uses custom stores and actions pattern (see `packages/frontend/app-framework/actions-and-stores.ts`)
+- **TypeScript React Components**: All frontend code is TypeScript with proper typing
+- **Modular Store System**: Each feature has its own store/actions (AccountStore, BillingStore, etc.)
+- **WebSocket Communication**: Real-time communication with backend via WebSocket messages
+
+#### Backend Architecture
+- **PostgreSQL Database**: Primary data store with sophisticated querying system
+- **WebSocket Messaging**: Real-time communication between frontend and backend
+- **Conat System**: Container orchestration for compute servers
+- **Event-Driven Architecture**: Extensive use of EventEmitter patterns
+- **Microservice-like Packages**: Each package handles specific functionality
+
+#### Communication Patterns
+- **WebSocket Messages**: Primary communication method (see `packages/comm/websocket/types.ts`)
+- **Database Queries**: Structured query system with typed interfaces
+- **Event Emitters**: Inter-service communication within backend
+- **REST-like APIs**: Some HTTP endpoints for specific operations
+
+### Key Technologies
+- **TypeScript**: Primary language for all new code
+- **React**: Frontend framework
+- **PostgreSQL**: Database
+- **Node.js**: Backend runtime
+- **WebSockets**: Real-time communication
+- **pnpm**: Package manager and workspace management
+- **Jest**: Testing framework
+- **SASS**: CSS preprocessing
+
+### Database Schema
+- Comprehensive schema in `packages/util/db-schema`
+- Query abstractions in `packages/database/postgres/`
+- Type-safe database operations with TypeScript interfaces
+
+### Testing
+- **Jest**: Primary testing framework
+- **ts-jest**: TypeScript support for Jest
+- **jsdom**: Browser environment simulation for frontend tests
+- Test files use `.test.ts` or `.spec.ts` extensions
+- Each package has its own jest.config.js
+
+### Import Patterns
+- Use absolute imports with `@cocalc/` prefix for cross-package imports
+- Example: `import { cmp } from "@cocalc/util/misc"`
+- Type imports: `import type { Foo } from "./bar"`
+- Destructure imports when possible
+
+### Development Workflow
+1. Changes to TypeScript require compilation (`pnpm tsc` in relevant package)
+2. Database must be running before starting hub
+3. Hub coordinates all services and should be restarted after changes
+4. Use `pnpm clean && pnpm build-dev` when switching branches or after major changes
+
+# Workflow
+- Be sure to typecheck when you're done making a series of code changes
+- Prefer running single tests, and not the whole test suite, for performance
+
+## Git Workflow
+
+- Prefix git commits with the package and general area. e.g. 'frontend/latex: ...' if it concerns latex editor changes in the packages/frontend/... code.
+- When pushing a new branch to Github, track it upstream. e.g. `git push --set-upstream origin feature-foo` for branch "feature-foo".
+
+# important-instruction-reminders
+- Do what has been asked; nothing more, nothing less.
+- NEVER create files unless they're absolutely necessary for achieving your goal.
+- ALWAYS prefer editing an existing file to creating a new one.
+- NEVER proactively create documentation files (*.md) or README files. Only create documentation files if explicitly requested by the User.

src/package.json

@@ -18,11 +18,10 @@
     "version-check": "pip3 install typing_extensions mypy  || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py",
     "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test",
     "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test",
-    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter --retries=1",
+    "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1",
     "depcheck": "cd packages && pnpm run -r --parallel depcheck",
     "prettier-all": "cd packages/",
     "local-ci": "./scripts/ci.sh",
-    "conat-server": "cd packages/server && pnpm conat-server",
     "conat-connections": "cd packages/backend && pnpm conat-connections",
     "conat-watch": "cd packages/backend && pnpm conat-watch",
     "conat-inventory": "cd packages/backend && pnpm conat-inventory"

src/packages/backend/get-listing.ts

@@ -4,7 +4,7 @@
  */
 
 /*
-Server directory listing through the HTTP server and Websocket API.
+This is used by backends to serve directory listings to clients:
 
 {files:[..., {size:?,name:?,mtime:?,isdir:?}]}
 

src/packages/backend/logger.ts

@@ -128,7 +128,7 @@ function initTransports() {
     // Similar as in debug source code, except I stuck a timestamp
     // at the beginning, which I like... except also aware of
     // non-printf formatting.
-    const line = `${new Date().toISOString()}: ${myFormat(...args)}\n`;
+    const line = `${new Date().toISOString()} (${process.pid}):${myFormat(...args)}\n`;
 
     if (transports.console) {
       // the console transport:

src/packages/conat/core/client.ts

@@ -510,7 +510,10 @@ export class Client extends EventEmitter {
       reconnection: true,
     });
 
-    this.conn.on("info", (info) => {
+    this.conn.on("info", (info, ack) => {
+      if (typeof ack == "function") {
+        ack();
+      }
       const firstTime = this.info == null;
       this.info = info;
       this.emit("info", info);

src/packages/conat/core/server.ts

@@ -53,7 +53,7 @@ import {
 import { Patterns } from "./patterns";
 import { is_array } from "@cocalc/util/misc";
 import { UsageMonitor } from "@cocalc/conat/monitor/usage";
-import { once } from "@cocalc/util/async-utils";
+import { once, until } from "@cocalc/util/async-utils";
 import {
   clusterLink,
   type ClusterLink,
@@ -74,6 +74,7 @@ import { type SysConatServer, sysApiSubject, sysApi } from "./sys";
 import { forkedConatServer } from "./start-server";
 import { stickyChoice } from "./sticky";
 import { EventEmitter } from "events";
+import { Metrics } from "../types";
 
 const logger = getLogger("conat:core:server");
 
@@ -310,6 +311,10 @@ export class ConatServer extends EventEmitter {
     });
   };
 
+  public getUsage = (): Metrics => {
+    return this.usage.getMetrics();
+  };
+
   // this is for the Kubernetes health check -- I haven't
   // thought at all about what to do here, really.
   // Hopefully experience can teach us.
@@ -602,7 +607,9 @@ export class ConatServer extends EventEmitter {
       return;
     }
     if (!(await this.isAllowed({ user, subject, type: "sub" }))) {
-      const message = `permission denied subscribing to '${subject}' from ${JSON.stringify(user)}`;
+      const message = `permission denied subscribing to '${subject}' from ${JSON.stringify(
+        user,
+      )}`;
       this.log(message);
       throw new ConatError(message, {
         code: 403,
@@ -706,7 +713,9 @@ export class ConatServer extends EventEmitter {
     }
 
     if (!(await this.isAllowed({ user: from, subject, type: "pub" }))) {
-      const message = `permission denied publishing to '${subject}' from ${JSON.stringify(from)}`;
+      const message = `permission denied publishing to '${subject}' from ${JSON.stringify(
+        from,
+      )}`;
       this.log(message);
       throw new ConatError(message, {
         // this is the http code for permission denied, and having this
@@ -931,7 +940,7 @@ export class ConatServer extends EventEmitter {
       this.subscriptions[id] = new Set<string>();
     }
 
-    socket.emit("info", { ...this.info(), user });
+    this.sendInfo(socket, user);
 
     socket.on("stats", ({ recv0 }) => {
       const s = this.stats[socket.id];
@@ -950,7 +959,9 @@ export class ConatServer extends EventEmitter {
           return;
         }
         if (!(await this.isAllowed({ user, subject, type: "pub" }))) {
-          const message = `permission denied waiting for interest in '${subject}' from ${JSON.stringify(user)}`;
+          const message = `permission denied waiting for interest in '${subject}' from ${JSON.stringify(
+            user,
+          )}`;
           this.log(message);
           respond({ error: message, code: 403 });
         }
@@ -1075,6 +1086,38 @@ export class ConatServer extends EventEmitter {
     });
   };
 
+  sendInfo = async (socket, user) => {
+    // we send info with an ack because I think sometimes the initial "info"
+    // message gets dropped, leaving a broken hanging connection that never
+    // does anything (until the user explicitly refreshes their browser).
+    // I did see what is probably this in production frequently.
+    try {
+      await until(
+        async () => {
+          if (!socket.conn?.readyState.startsWith("o")) {
+            // logger.debug(`failed to send "info" message to ${socket.id}`);
+            // readyState not defined or not opened or opening, so connection must
+            // have been closed before success.
+            return true;
+          }
+          try {
+            await socket
+              .timeout(7500)
+              .emitWithAck("info", { ...this.info(), user });
+            return true;
+          } catch (err) {
+            // logger.debug(`error sending "info" message to ${socket.id}`, err);
+            return false;
+          }
+        },
+        { min: 5000, max: 30000, timeout: 120_000 },
+      );
+    } catch {
+      // never ack'd "info" after a few minutes -- could just be an old client,
+      // so don't do anything at this point.
+    }
+  };
+
   address = () => getServerAddress(this.options);
 
   // create new client in the same process connected to this server.
@@ -1338,8 +1381,8 @@ export class ConatServer extends EventEmitter {
     const usage = async () => {
       return { [this.id]: this.usage.stats() };
     };
-    // user has to explicitly refresh there browser after
-    // being disconnected this way
+    // user has to explicitly refresh their browser after
+    // being disconnected this way:
     const disconnect = async (ids: string | string[]) => {
       if (typeof ids == "string") {
         ids = [ids];
@@ -1791,7 +1834,9 @@ export function updateSticky(update: StickyUpdate, sticky: Sticky): boolean {
 function getServerAddress(options: Options) {
   const port = options.port;
   const path = options.path?.slice(0, -"/conat".length) ?? "";
-  return `http${options.ssl || port == 443 ? "s" : ""}://${options.clusterIpAddress ?? "localhost"}:${port}${path}`;
+  return `http${options.ssl || port == 443 ? "s" : ""}://${
+    options.clusterIpAddress ?? "localhost"
+  }:${port}${path}`;
 }
 
 /*