feat: systemd timer unit to restart network on NDisc failure (#971)

Author: tomashley

ansible/files/systemd-networkd/systemd-networkd-check-and-fix.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Check if systemd-networkd has broken NDisc routes and fix
+
+[Service]
+Type=oneshot
+# This needs to be root for the service restart to work
+User=root
+Group=root
+ExecStart=/usr/local/bin/systemd-networkd-check-and-fix.sh
+
+Requisite=systemd-networkd.service

ansible/files/systemd-networkd/systemd-networkd-check-and-fix.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Check for occurrences of an NDisc log error
+# NOTE: --since timer flag must match the cadence of systemd timer unit. Risk of repeat matches and restart loop
+journalctl --no-pager --unit systemd-networkd --since "1 minutes ago" --grep "Could not set NDisc route" >/dev/null
+NDISC_ERROR=$?
+
+if systemctl is-active --quiet systemd-networkd.service && [ "${NDISC_ERROR}" == 0 ]; then
+  echo "$(date) systemd-network running but NDisc routes are broken. Restarting systemd.networkd.system"
+  /usr/bin/systemctl restart systemd-networkd.service
+fi

ansible/files/systemd-networkd/systemd-networkd-check-and-fix.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Check if systemd-networkd has broken NDisc routes and fix
+
+[Timer]
+# NOTE: cadence must match that of the journalctl search (--since). Risk of repeat matches and restart loop
+OnCalendar=minutely
+
+[Install]
+WantedBy=timers.target

ansible/playbook.yml

@@ -71,6 +71,11 @@
       tags:
         - install-supabase-internal
 
+    - name: Fix IPv6 NDisc issues
+      import_tasks: tasks/fix_ipv6_ndisc.yml
+      tags:
+        - install-supabase-internal
+
     - name: Start Postgres Database
       systemd:
         name: postgresql

ansible/tasks/fix_ipv6_ndisc.yml

@@ -0,0 +1,33 @@
+---
+- name: fix Network - systemd timer file
+  copy:
+    dest: /etc/systemd/system/systemd-networkd-check-and-fix.timer
+    src: "files/systemd-networkd/systemd-networkd-check-and-fix.timer"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: fix Network - systemd service file
+  copy:
+    dest: /etc/systemd/system/systemd-networkd-check-and-fix.service
+    src: "files/systemd-networkd/systemd-networkd-check-and-fix.service"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: fix Network - detect script
+  copy:
+    dest: /usr/local/bin/systemd-networkd-check-and-fix.sh
+    src: "files/systemd-networkd/systemd-networkd-check-and-fix.sh"
+    owner: root
+    group: root
+    mode: 0700
+
+- name: fix Network - reload systemd
+  systemd:
+    daemon_reload: yes
+
+- name: fix Network - enable systemd timer
+  systemd:
+    name: systemd-networkd-check-and-fix.timer
+    enabled: true