systemd: retry when the dbus connection returns EAGAIN

Author: jianghao65536

systemd/dbus.go

@@ -4,10 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand/v2"
 	"sync"
+	"time"
 
 	systemdDbus "github.com/coreos/go-systemd/v22/dbus"
 	dbus "github.com/godbus/dbus/v5"
+	"golang.org/x/sys/unix"
 )
 
 var (
@@ -64,10 +67,34 @@ func (d *dbusConnManager) getConnection() (*systemdDbus.Conn, error) {
 }
 
 func (d *dbusConnManager) newConnection() (*systemdDbus.Conn, error) {
-	if dbusRootless {
-		return newUserSystemdDbus()
+	newDbusConn := func() (*systemdDbus.Conn, error) {
+		if dbusRootless {
+			return newUserSystemdDbus()
+		}
+		return systemdDbus.NewWithContext(context.TODO())
+	}
+
+	var conn *systemdDbus.Conn
+	var err error
+	for retry := range 5 {
+		conn, err = newDbusConn()
+		if err == nil {
+			break
+		}
+		if !errors.Is(err, unix.EAGAIN) {
+			return nil, err
+		}
+		if retry < 4 {
+			// Exponential backoff with jitter: base delay doubles each retry, plus random jitter up to 1 second
+			baseDelay := time.Second << retry // 1s, 2s, 4s, 8s
+			jitter := time.Duration(rand.IntN(1000)) * time.Millisecond
+			time.Sleep(baseDelay + jitter)
+		} else {
+			// despite retries, still EAGAIN
+			return nil, fmt.Errorf("dbus connection failed after 5 retries: %w", err)
+		}
 	}
-	return systemdDbus.NewWithContext(context.TODO())
+	return conn, nil
 }
 
 // resetConnection resets the connection to its initial state

systemd/dbus_test.go

@@ -0,0 +1,25 @@
+package systemd
+
+import (
+	"sync"
+	"testing"
+)
+
+func Test200ParallelConnection(t *testing.T) {
+	var dms []*dbusConnManager
+	for range 400 {
+		dms = append(dms, newDbusConnManager(false))
+	}
+
+	var wg sync.WaitGroup
+	for _, dm := range dms {
+		wg.Add(1)
+		go func(dm *dbusConnManager) {
+			defer wg.Done()
+			_, err := dm.newConnection()
+			if err != nil {
+				t.Error(err)
+			}
+		}(dm)
+	}
+}