✨(k8s) add jigasi scaling and transcription feature

We want to deploy jigasi on our kubernetes cluster.
This enables transcription via VOSK.

The settings are stored in .env files used by Kustomize to generate ConfigMaps.
Values can be overriden in the overlay if necessary.

Author: annanas

bin/init-overlay

@@ -38,13 +38,15 @@ JIBRI_RECORDER_PASSWORD=$(random_password 64)
 JIBRI_XMPP_PASSWORD=$(random_password 64)
 JICOFO_AUTH_PASSWORD=$(random_password 64)
 JVB_AUTH_PASSWORD=$(random_password 64)
+JIGASI_XMPP_PASSWORD=$(random_password 64)
 echo OK
 
 export BASE_DOMAIN
 export JIBRI_RECORDER_PASSWORD
 export JIBRI_XMPP_PASSWORD
 export JICOFO_AUTH_PASSWORD
 export JVB_AUTH_PASSWORD
+export JIGASI_XMPP_PASSWORD
 export LETSENCRYPT_ACCOUNT_EMAIL
 
 mkdir "$NEW_OVERLAY_PATH"

k8s/base/env/jigasi.env

@@ -0,0 +1,31 @@
+# This disables Jigasi SIP connections
+JIGASI_DISABLE_SIP=false
+
+# SIP password for jigasi
+JIGASI_SIP_PASSWORD=jigasi
+
+#JIGASI_SIP_SERVER=
+
+#Port and Transport method used for Jigasi SIP
+JIGASI_SIP_PORT=5060
+JIGASI_SIP_TRANSPORT=UDP
+
+#JIGASI_SIP_DEFAULT_ROOM=
+#JIGASI_HEALTH_CHECK_SIP_URI=
+#JIGASI_HEALTH_CHECK_INTERVAL=
+#JIGASI_SIP_KEEP_ALIVE_METHOD=
+#JIGASI_ENABLE_SDES_SRTP=
+
+JIGASI_TRANSCRIBER_ADVERTISE_URL=true
+
+# Determines whether the transcriber records audio.
+JIGASI_TRANSCRIBER_RECORD_AUDIO=false
+
+# This variable changes whether Jigasi sends the transcribed text in chat or not.
+JIGASI_TRANSCRIBER_SEND_TXT=false
+
+# Custom transcription service to use with Jigasi
+JIGASI_CUSTOM_TRANSCRIPTION_SERVICE=org.jitsi.jigasi.transcription.VoskTranscriptionService
+
+# Websocket for the VOSK transcription service
+VOSK_WEBSOCKET_URL=ws://127.0.0.1:2700

k8s/base/env/jitsi-common.env

@@ -9,7 +9,7 @@ PUBLIC_URL=https://YOUR_DOMAIN
 XMPP_DOMAIN=YOUR_DOMAIN
 
 # Internal XMPP server
-XMPP_SERVER=prosody.jitsi.svc
+XMPP_SERVER=prosody.jitsi.svc.cluster.local
 
 # Internal XMPP domain for authenticated services.
 XMPP_AUTH_DOMAIN=auth.YOUR_DOMAIN
@@ -23,9 +23,12 @@ XMPP_INTERNAL_MUC_DOMAIN=internal-muc.auth.YOUR_DOMAIN
 # XMPP domain for the MUC.
 XMPP_MUC_DOMAIN=muc.YOUR_DOMAIN
 
-# XMPP domain for the jibri recorder
+# XMPP domain for the jibri and jigasi recorder
 XMPP_RECORDER_DOMAIN=recorder.YOUR_DOMAIN
 
+# XMPP port
+XMPP_PORT=5222
+
 # MUC for the JVB pool.
 JVB_BREWERY_MUC=jvbbrewery
 
@@ -35,6 +38,9 @@ JIBRI_BREWERY_MUC=jibribrewery
 # MUC name for the Jigasi pool.
 JIGASI_BREWERY_MUC=jigasibrewery
 
+# Jigasi SIP URI for jigasi and jicofo.
+JIGASI_SIP_URI=[email protected]_DOMAIN
+
 # System time zone
 TZ=UTC
 
@@ -62,6 +68,9 @@ ENABLE_AUTH=0
 # Enable guest access
 ENABLE_GUESTS=1
 
+# Enable transcriptions
+ENABLE_TRANSCRIPTIONS=true
+
 # Select authentication type: internal, jwt or ldap
 AUTH_TYPE=internal
 

k8s/base/env/jitsi-meet-front.env

@@ -9,10 +9,10 @@ DISABLE_HTTPS=1
 ENABLE_HTTP_REDIRECT=0
 
 # Internal XMPP server URL
-XMPP_BOSH_URL_BASE=http://prosody.jitsi.svc:5280
+XMPP_BOSH_URL_BASE=http://prosody.jitsi.svc.cluster.local:5280
 
 # Default language to use
-#DEFAULT_LANGUAGE=
+DEFAULT_LANGUAGE=en
 
 # URL used to receive branding specific information in JSON.
 # None of the fields are mandatory and the response must have the shape:
@@ -243,7 +243,7 @@ ENABLE_STATS_ID=false
 # Nginx configuration
 #
 
-NGINX_RESOLVER=127.0.0.1
+NGINX_RESOLVER=coredns.kube-system.svc.cluster.local
 
 # Defines the number of worker processes.
 NGINX_WORKER_PROCESSES=4
@@ -252,4 +252,10 @@ NGINX_WORKER_PROCESSES=4
 # worker process. It should be kept in mind that this number includes all
 # connections (e.g. connections with proxied servers, among others), not only
 # connections with clients.
-NGINX_WORKER_CONNECTIONS=2048
+NGINX_WORKER_CONNECTIONS=768
+
+#
+USE_APP_LANGUAGE=false
+
+# Transcription languages available in the drop down menu
+TRANSLATION_LANGUAGES=["en"]

k8s/base/env/jitsi-secrets.env

@@ -22,6 +22,12 @@ JIBRI_XMPP_PASSWORD=
 # XMPP password for Jibri client connections.
 JIBRI_XMPP_USER=jibri
 
+# XMPP user for Jigasi client connections.
+JIGASI_XMPP_USER=jigasi
+
+#XMPP password for Jigasi client connections.
+JIGASI_XMPP_PASSWORD=
+
 # Secret used to sign/verify JWT tokens
 #JWT_APP_SECRET=my_jitsi_app_secret
 

k8s/base/jicofo-deployment.yml

@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: jitsi-meet
-          image: jitsi/jicofo:stable-7648-4
+          image: jitsi/jicofo:stable-8044
           imagePullPolicy: IfNotPresent
           envFrom:
           - configMapRef:

k8s/base/jigasi-deployment.yml

@@ -0,0 +1,84 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: jigasi
+  name: jigasi
+spec:
+  selector:
+    matchLabels:
+      app: jigasi
+# The goal of this matchExpressions selector is to exclude the jigasi
+# pods with a label status=busy from their ReplicaSet.
+# A sidecar container (metadata-updater) is updating the status
+# label according to jibri's state.
+#
+# This mechanism drastically reduces the risk of terminating
+# a busy jibri pod when scaling down the deployment.
+#
+# For more details :
+# https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/#isolating-pods-from-a-replicaset
+    matchExpressions:
+      - {key: status, operator: In, values: [idle, unknown]}
+  strategy:
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 1
+    type: RollingUpdate
+  template:
+    metadata:
+      annotations:
+      labels:
+        app: jigasi
+        status: idle
+    spec:
+      serviceAccountName: jigasi
+      containers:
+      - image: annagrigoriu/testing:jigasi-transcription-final
+        name: jigasi
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 20000
+          protocol: UDP
+        - containerPort: 8788
+          name: api
+          protocol: TCP
+        envFrom:
+          - configMapRef:
+              name: jitsi-common
+          - configMapRef:
+              name: jigasi
+        env:
+          - name: JIGASI_XMPP_USER
+            valueFrom:
+              secretKeyRef:
+                name: jitsi-secrets
+                key: JIGASI_XMPP_USER
+          - name: JIGASI_XMPP_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: jitsi-secrets
+                key: JIGASI_XMPP_PASSWORD
+        volumeMounts:
+        - name: jigasi-transcripts
+          mountPath: /tmp/transcripts
+      - image: python:3.8
+        name: metadata-updater
+        command: ["/bin/sh","-c"]
+        args: [ "pip install websockets && python3 opt/jigasi-metadata-updater/jigasi-metadata-updater.py"]
+        volumeMounts:
+        - name: jigasi-metadata-updater
+          mountPath: /opt/jigasi-metadata-updater
+      - image: alphacep/kaldi-en:latest
+        name: vosk-en
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 2700
+      nodeSelector:
+        k8s.scaleway.com/pool-name: jigasi
+      volumes:
+        - name: jigasi-transcripts
+          emptyDir: {}
+        - name: jigasi-metadata-updater
+          configMap:
+            name: jigasi-metadata-updater

k8s/base/jigasi-hpa.yaml

@@ -0,0 +1,63 @@
+# With this Horizontal Pod Autoscaler, we want to ensure that there is
+# always at least:
+#  - a specific count of jigasi pods available (TARGET_MIN_VALUE)
+#  - a specific percentage of jigasi pods available across all jigasi pods (TARGET_PERCENT)
+#
+# The formula applied by HPA to compute the desired replicas is :
+# desiredReplicas = ceil[currentReplicas * ( currentMetricValue / desiredMetricValue )]
+# (see https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details)
+#
+# If multiple metrics are specified in HPA, the formula is applied for each of
+# them and the higher desiredReplicas is taken into account.
+#
+# To guarantee that we always have at least TARGET_MIN_VALUE pods available, we
+# just have to set this value as minReplicas because the Deployment manages
+# only available jigasis. When a jigasi pod is busy, it gets orphaned and is
+# ignored by the Deployment.
+#
+# To ensure that we have a certain percentage of available pods
+# (TARGET_PERCENT), a rule is defined in this HPA based on the "jigasi_busy"
+# metric, which takes into account all jigasi pods in the namespace (those
+# managed by the deployment + the orphaned pods that are busy)
+
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: jigasi-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: jigasi
+  minReplicas: 2
+  maxReplicas: 10
+  behavior:
+    # We'll allow to scale down 20% of the pods every 30s
+    scaleDown:
+      stabilizationWindowSeconds: 60
+      policies:
+        - type: Percent
+          value: 20
+          periodSeconds: 30
+    # We allow to add 2 pods every 2 minutes.
+    # FIXME: Adjust this value when cluster autoscaler is enabled.
+    # It should give enough time to provision new nodes, but not too much
+    # to be able to scale-up in case of high demand.
+    scaleUp:
+      policies:
+        - type: Pods
+          value: 2
+          periodSeconds: 120
+  metrics:
+    - type: Object
+      object:
+        metric:
+          name: jigasi_busy
+        describedObject:
+          apiVersion: v1
+          kind: Namespace
+          name: jitsi
+        target:
+          type: Value
+          # We want to always have at least 20% of available jigasi instances.
+          value: 0.8