Slow worker restart

Author: aequitas

docker/docker-compose.yml

@@ -728,6 +728,9 @@ services:
   # cron daemon with access to Docker socket but no networking
   cron-docker:
     image: alpinelinux/docker-cli:latest
+    build:
+      context: ..
+      dockerfile: docker/util.Dockerfile
     command: crond -f -d7
     configs:
       # restart worker to prevent memory issues
@@ -738,10 +741,18 @@ services:
       - source: restart_nassl_worker_cron
         target: /etc/periodic/15min/restart_nassl_worker
         mode: 0755
+      - source: restart_slow_worker_cron
+        target: /etc/periodic/daily/restart_slow_worker
+        mode: 0755
       # remove unused old docker images to reduce disk space usage
       - source: docker_image_prune
         target: /etc/periodic/weekly/docker-image-prune
         mode: 0755
+    environment:
+      - AUTO_UPDATE_TO
+      - WORKER_REPLICAS
+      - WORKER_SLOW_REPLICAS
+
     restart: unless-stopped
     logging:
       driver: $LOGGING_DRIVER
@@ -753,6 +764,7 @@ services:
     network_mode: "none"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - /opt/Internet.nl:/opt/Internet.nl
 
     profiles:
       - cron
@@ -1128,24 +1140,83 @@ configs:
   restart_worker_cron:
     content: |
       #!/bin/sh
+
+      # restart slow worker every day to prevent slow memory leaks
+      # as the slow worker can run very long tasks (eg: report generation)
+      # we first start  a new container before stopping the previous one
+
       set -e
-      # stop and start worker one at a time to ensure (batch) tasks are still being picked up
-      # workers are sent a TERM signal with which a 10 minute grace period before QUIT is sent
-      for worker in $(docker ps --filter label=com.docker.compose.service=worker --quiet); do
-        docker stop "$$worker"
-        docker start "$$worker"
-      done
 
+      cd /opt/Internet.nl
+
+      SERVICE=worker
+      REPLICAS=$$WORKER_REPLICAS
+      COMPOSE_CMD="docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env"
+
+      OLD_CONTAINERS=$$($$COMPOSE_CMD ps --format "{{ .Name }}"|grep "$$SERVICE")
+
+      # bring up new containers, wait until healthy
+      $$COMPOSE_CMD up --no-deps --no-recreate --wait --scale="$$SERVICE=$$(($$REPLICAS*2))" "$$SERVICE"
+
+      # graceful shutdown and remove old containers
+      docker rm --force "$$OLD_CONTAINERS"
+
+      # restore replica number to original
+      $$COMPOSE_CMD scale $$SERVICE=$$REPLICAS
   restart_nassl_worker_cron:
     content: |
       #!/bin/sh
+
+      # restart slow worker every day to prevent slow memory leaks
+      # as the slow worker can run very long tasks (eg: report generation)
+      # we first start  a new container before stopping the previous one
+
       set -e
-      # stop and start worker one at a time to ensure (batch) tasks are still being picked up
-      # workers are sent a TERM signal with which a 10 minute grace period before QUIT is sent
-      for worker in $(docker ps --filter label=com.docker.compose.service=worker-nassl --quiet); do
-        docker stop "$$worker"
-        docker start "$$worker"
-      done
+
+      cd /opt/Internet.nl
+
+      SERVICE=worker-nassl
+      REPLICAS=$$WORKER_REPLICAS
+      COMPOSE_CMD="docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env"
+
+      OLD_CONTAINERS=$$($$COMPOSE_CMD ps --format "{{ .Name }}"|grep "$$SERVICE")
+
+      # bring up new containers, wait until healthy
+      $$COMPOSE_CMD up --no-deps --no-recreate --wait --scale="$$SERVICE=$$(($$REPLICAS*2))" "$$SERVICE"
+
+      # graceful shutdown and remove old containers
+      docker rm --force "$$OLD_CONTAINERS"
+
+      # restore replica number to original
+      $$COMPOSE_CMD scale $$SERVICE=$$REPLICAS
+
+  restart_slow_worker_cron:
+    content: |
+      #!/bin/sh
+
+      # restart slow worker every day to prevent slow memory leaks
+      # as the slow worker can run very long tasks (eg: report generation)
+      # we first start  a new container before stopping the previous one
+
+      set -e
+
+      cd /opt/Internet.nl
+
+      SERVICE=worker-slow
+      REPLICAS=$$WORKER_SLOW_REPLICAS
+      COMPOSE_CMD="docker compose --env-file=docker/defaults.env --env-file=docker/host.env --env-file=docker/local.env"
+
+      OLD_CONTAINERS=$$($$COMPOSE_CMD ps --format "{{ .Name }}"|grep "$$SERVICE")
+
+      # bring up new containers, wait until healthy
+      $$COMPOSE_CMD up --no-deps --no-recreate --wait --scale="$$SERVICE=$$(($$REPLICAS*2))" "$$SERVICE"
+
+      # graceful shutdown and remove old containers
+      docker rm --force "$$OLD_CONTAINERS"
+
+      # restore replica number to original
+      $$COMPOSE_CMD scale $$SERVICE=$$REPLICAS
+
 
   docker_image_prune:
     content: |