use separate process for each persist server; also include pid in logger output

Author: williamstein

src/packages/backend/logger.ts

@@ -128,7 +128,7 @@ function initTransports() {
     // Similar as in debug source code, except I stuck a timestamp
     // at the beginning, which I like... except also aware of
     // non-printf formatting.
-    const line = `${new Date().toISOString()}: ${myFormat(...args)}\n`;
+    const line = `${new Date().toISOString()} (${process.pid}):${myFormat(...args)}\n`;
 
     if (transports.console) {
       // the console transport:

src/packages/server/conat/index.ts

@@ -4,8 +4,8 @@ import { init as initChangefeedServer } from "@cocalc/database/conat/changefeed-
 import { init as initLLM } from "./llm";
 import { loadConatConfiguration } from "./configuration";
 import { createTimeService } from "@cocalc/conat/service/time";
-import { initPersistServer } from "@cocalc/backend/conat/persist";
-import { conatPersistCount, conatApiCount } from "@cocalc/backend/data";
+export { initConatPersist } from "./persist";
+import { conatApiCount } from "@cocalc/backend/data";
 
 export { loadConatConfiguration };
 
@@ -19,16 +19,6 @@ export async function initConatChangefeedServer() {
   initChangefeedServer();
 }
 
-export async function initConatPersist() {
-  logger.debug("initPersistServer: sqlite3 stream persistence", {
-    conatPersistCount,
-  });
-  await loadConatConfiguration();
-  for (let i = 0; i < conatPersistCount; i++) {
-    initPersistServer();
-  }
-}
-
 export async function initConatApi() {
   logger.debug("initConatApi: the central api services", { conatApiCount });
   await loadConatConfiguration();

src/packages/server/conat/persist/index.ts

@@ -1,6 +1,7 @@
-import { loadConatConfiguration } from "./configuration";
+import { loadConatConfiguration } from "../configuration";
 import { initPersistServer } from "@cocalc/backend/conat/persist";
 import { conatPersistCount } from "@cocalc/backend/data";
+import { createForkedPersistServer } from "./start-server";
 
 import getLogger from "@cocalc/backend/logger";
 
@@ -28,8 +29,8 @@ async function createPersistCluster() {
     conatPersistCount,
     "nodes",
   );
-  await loadConatConfiguration();
   for (let i = 0; i < conatPersistCount; i++) {
-    initPersistServer();
+    logger.debug("initPersistServer: starting node ", i);
+    createForkedPersistServer();
   }
 }

src/packages/server/conat/persist/start-persist-node.ts

@@ -0,0 +1,16 @@
+import { loadConatConfiguration } from "../configuration";
+import { initPersistServer } from "@cocalc/backend/conat/persist";
+import { getLogger } from "@cocalc/backend/logger";
+import { addErrorListeners } from "@cocalc/server/metrics/error-listener";
+
+const logger = getLogger("server:conat:persist:start-persist-node");
+
+async function main() {
+  logger.debug("starting forked persist node in process", process.pid);
+  console.log("starting forked persist node in process", process.pid);
+  addErrorListeners();
+  await loadConatConfiguration();
+  await initPersistServer();
+}
+
+main();

src/packages/server/conat/socketio/start-cluster-node.ts

@@ -1,3 +1,23 @@
+/*
+The code here is run when conatSocketioCount > 1 (i.e., env var CONAT_SOCKETIO_COUNT).
+This does NOT use the socketio cluster adapter or the nodejs cluster module.
+Every worker process this spawns runs independently after it starts and there is
+no single node coordinating communications like with the socketio cluster adapter,
+and traffic across the cluster is minimal. This will thus *scale* much better,
+though this is also just using normal TCP networking for communication instead of
+IPC (like socketios cluster adapter).  Also, the traffic between nodes is precisely
+what is needed for Conat, so it's really solving a differnet problem than socketio's
+cluster adapter, and under the hood what this actually does is much more sophisticated,
+with each node maintaining and serving sqlite backed streams of data about their state.
+
+This code exists mainly for testing and potentially also for scaling cocalc to
+more traffic when running on a single machine without Kubernetes.
+
+One cpu support several hundred simultaneous active connections -- if you want to
+have 500 active users all using projects (that also means additional connections) --
+you will definitely need more than one node.
+*/
+
 import "@cocalc/backend/conat/persist";
 import { init, type Options } from "@cocalc/conat/core/server";
 import { getUser, isAllowed } from "./auth";
@@ -6,7 +26,7 @@ import { loadConatConfiguration } from "../configuration";
 import { getLogger } from "@cocalc/backend/logger";
 
 async function main() {
-  console.log("main");
+  console.log("conat server: starting a cluster node");
 
   addErrorListeners();
   const configDone = loadConatConfiguration();