vz: add SSH over AF_VSOCK

Since systemd v256 (Ubuntu 24.10), SSH is bound to AF_VSOCK port 22.

https://github.com/systemd/systemd/releases/tag/v256
> - If the system is run in a VM providing AF_VSOCK support, it automatically
binds sshd to AF_VSOCK port 22.

https://discourse.ubuntu.com/t/oracular-oriole-release-notes/44878
> - When sshd is installed on a system, a new systemd generator, systemd-ssh-generator
binds a socket-activated SSH server to local AF_VSOCK and AF_UNIX sockets under certain conditions.

This changes to delay starting SSH port forwarding until the SSH server on the VM becomes ready.
If AF_VSOCK port 22 can be connected, start a local SSH port as a proxy for AF_VSOCK port 22,
instead of starting gvisor's port forwarder.

SSH over VSOCK is faster than SSH over gvisor's port forwarder.

This change is opt-out because it requires VZ and VM with systemd v256+,
setting `LIMA_SSH_OVER_VSOCK=true` does not mean it works.
To disable, set `LIMA_SSH_OVER_VSOCK=false`.

Signed-off-by: Norio Nomura <[email protected]>

Author: norio-nomura

pkg/driver/vz/vm_darwin.go

@@ -108,10 +108,36 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (*v
 					filesToRemove[pidFile] = struct{}{}
 					logrus.Info("[VZ] - vm state change: running")
 
-					err := usernetClient.ConfigureDriver(ctx, inst, sshLocalPort)
-					if err != nil {
-						errCh <- err
-					}
+					go func() {
+						sshLocalPort := sshLocalPort
+						useSSHOverVsock := true
+						if envVar := os.Getenv("LIMA_SSH_OVER_VSOCK"); envVar != "" {
+							b, err := strconv.ParseBool(envVar)
+							if err != nil {
+								logrus.WithError(err).Warnf("invalid LIMA_SSH_OVER_VSOCK value %q", envVar)
+							} else {
+								useSSHOverVsock = b
+							}
+						}
+						if !useSSHOverVsock {
+							logrus.Info("LIMA_SSH_OVER_VSOCK is false, skipping detection of SSH server on vsock port")
+						} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst); err == nil {
+							hostAddress := net.JoinHostPort(inst.SSHAddress, strconv.Itoa(sshLocalPort))
+							if err := wrapper.startVsockForwarder(ctx, 22, hostAddress); err == nil {
+								logrus.Infof("Detected SSH server is listening on the vsock port; changed %s to proxy for the vsock port", hostAddress)
+								sshLocalPort = 0 // disable gvisor ssh port forwarding
+							} else {
+								logrus.WithError(err).Warn("Failed to detect SSH server on vsock port, falling back to gvisor's forwarder")
+							}
+						} else {
+							logrus.WithError(err).Warn("Failed to wait for the guest SSH server to become available, falling back to gvisor's forwarder")
+						}
+						err := usernetClient.ConfigureDriver(ctx, inst, sshLocalPort)
+						if err != nil {
+							errCh <- err
+						}
+					}()
+
 				case vz.VirtualMachineStateStopped:
 					logrus.Info("[VZ] - vm state change: stopped")
 					wrapper.mu.Lock()

pkg/driver/vz/vsock_forwarder.go

@@ -0,0 +1,75 @@
+//go:build darwin && !no_vz
+
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package vz
+
+import (
+	"context"
+	"errors"
+	"net"
+
+	"github.com/containers/gvisor-tap-vsock/pkg/tcpproxy"
+	"github.com/sirupsen/logrus"
+)
+
+func (m *virtualMachineWrapper) startVsockForwarder(ctx context.Context, vsockPort uint32, hostAddress string) error {
+	// Test if the vsock port is open
+	conn, err := m.dialVsock(ctx, vsockPort)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	// Start listening on localhost:hostPort and forward to vsock:vsockPort
+	_, _, err = net.SplitHostPort(hostAddress)
+	if err != nil {
+		return err
+	}
+	var lc net.ListenConfig
+	l, err := lc.Listen(ctx, "tcp", hostAddress)
+	if err != nil {
+		return err
+	}
+	go func() {
+		<-ctx.Done()
+		l.Close()
+	}()
+	logrus.Infof("Started vsock forwarder: %s -> vsock:%d on VM", hostAddress, vsockPort)
+	go func() {
+		defer l.Close()
+		for {
+			conn, err := l.Accept()
+			if err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					return
+				}
+				logrus.WithError(err).Errorf("vsock forwarder accept error: %v", err)
+			} else {
+				p := tcpproxy.DialProxy{
+					DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+						return m.dialVsock(ctx, vsockPort)
+					},
+				}
+				go p.HandleConn(conn)
+			}
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				continue
+			}
+		}
+	}()
+	return nil
+}
+
+func (m *virtualMachineWrapper) dialVsock(_ context.Context, port uint32) (conn net.Conn, err error) {
+	for _, socket := range m.SocketDevices() {
+		conn, err = socket.Connect(port)
+		if err == nil {
+			return conn, nil
+		}
+	}
+	return nil, err
+}

pkg/networks/usernet/client.go

@@ -38,9 +38,11 @@ func (c *Client) ConfigureDriver(ctx context.Context, inst *limatype.Instance, s
 	if err != nil {
 		return err
 	}
-	err = c.ResolveAndForwardSSH(ipAddress, sshLocalPort)
-	if err != nil {
-		return err
+	if sshLocalPort != 0 {
+		err = c.ResolveAndForwardSSH(ipAddress, sshLocalPort)
+		if err != nil {
+			return err
+		}
 	}
 	hosts := inst.Config.HostResolver.Hosts
 	if hosts == nil {
@@ -127,6 +129,27 @@ func (c *Client) Leases(ctx context.Context) (map[string]string, error) {
 	return leases, nil
 }
 
+// WaitOpeningSSHPort Wait until the guest ssh server is available.
+func (c *Client) WaitOpeningSSHPort(ctx context.Context, inst *limatype.Instance) error {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	macAddress := limayaml.MACAddress(inst.Dir)
+	ipAddr, err := c.ResolveIPAddress(ctx, macAddress)
+	if err != nil {
+		return err
+	}
+	u := fmt.Sprintf("%s/extension/wait_port?ip=%s&port=22", c.base, ipAddr)
+	res, err := httpclientutil.Get(ctx, c.client, u)
+	if err != nil {
+		return err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return errors.New("failed to wait for SSH port")
+	}
+	return nil
+}
+
 func NewClientByName(nwName string) *Client {
 	endpointSock, err := Sock(nwName, EndpointSock)
 	if err != nil {

pkg/networks/usernet/gvproxy.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -103,7 +104,8 @@ func run(ctx context.Context, g *errgroup.Group, configuration *types.Configurat
 	if err != nil {
 		return err
 	}
-	httpServe(ctx, g, ln, vn.Mux())
+
+	httpServe(ctx, g, ln, muxWithExtension(vn))
 
 	if opts.QemuSocket != "" {
 		err = listenQEMU(ctx, vn)
@@ -239,6 +241,48 @@ func httpServe(ctx context.Context, g *errgroup.Group, ln net.Listener, mux http
 	})
 }
 
+func muxWithExtension(n *virtualnetwork.VirtualNetwork) *http.ServeMux {
+	m := n.Mux()
+	m.HandleFunc("/extension/wait_port", func(w http.ResponseWriter, r *http.Request) {
+		ip := r.URL.Query().Get("ip")
+		if net.ParseIP(ip) == nil {
+			msg := fmt.Sprintf("invalid ip address: %s", ip)
+			http.Error(w, msg, http.StatusBadRequest)
+			return
+		}
+		port16, err := strconv.ParseUint(r.URL.Query().Get("port"), 10, 16)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		port := uint16(port16)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		addr := fmt.Sprintf("%s:%d", ip, port)
+		// Wait until the port is available.
+		for {
+			conn, err := n.DialContextTCP(ctx, addr)
+			if err == nil {
+				conn.Close()
+				logrus.Infof("Port is available on %s", addr)
+				w.WriteHeader(http.StatusOK)
+				break
+			}
+			select {
+			case <-ctx.Done():
+				msg := fmt.Sprintf("timed out waiting for port to become available on %s", addr)
+				logrus.Warn(msg)
+				http.Error(w, msg, http.StatusRequestTimeout)
+				return
+			default:
+			}
+			logrus.Infof("Waiting for port to become available on %s", addr)
+			time.Sleep(500 * time.Millisecond)
+		}
+	})
+	return m
+}
+
 func searchDomains() []string {
 	if runtime.GOOS != "windows" {
 		return resolveSearchDomain("/etc/resolv.conf")

website/content/en/docs/config/environment-variables.md

@@ -106,6 +106,15 @@ This page documents the environment variables used in Lima.
   lima
   ```
 
+### `LIMA_SSH_OVER_VSOCK`
+- **Description**: Specifies to use vsock for SSH connection instead of port forwarding.
+- **Default**: `true` (since v2.0.0)
+- **Usage**: 
+  ```sh
+  export LIMA_SSH_OVER_VSOCK=true
+  ```
+- **Note**: This variable is effective only if the VM is VZ based and systemd is v256 or later (e.g. Ubuntu 24.10+).
+
 ### `LIMA_SSH_PORT_FORWARDER`
 
 - **Description**: Specifies to use the SSH port forwarder (slow) instead of gRPC (fast, previously unstable)

website/content/en/docs/config/port.md

@@ -36,6 +36,20 @@ LIMA_SSH_PORT_FORWARDER=true limactl start
 - Doesn't support UDP based port forwarding
 - Spawns child process on host for running SSH master.
 
+#### SSH over AF_VSOCK
+
+| ⚡ Requirement | Lima >= 2.0 |
+|---------------|-------------|
+
+If VM is VZ based and systemd is v256 or later (e.g. Ubuntu 24.10+), Lima uses AF_VSOCK for communication between host and guest.
+SSH based port forwarding is much faster when using AF_VSOCK compared to traditional virtual network based port forwarding.
+
+To disable this feature, set `LIMA_SSH_OVER_VSOCK` to `false`:
+
+```bash
+export LIMA_SSH_OVER_VSOCK=false
+```
+
 ### Using GRPC
 
 | ⚡ Requirement | Lima >= 1.0 |