Support tmpfs-backed sandbox mount for LCOW containers

Signed-off-by: Jie Chen <[email protected]>

Author: jiechen0826

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -988,8 +988,9 @@ func isMountTypeSupported(hostPath, mountType string) bool {
 		hcsoci.MountTypeVirtualDisk, hcsoci.MountTypeExtensibleVirtualDisk:
 		return false
 	default:
-		// Ensure that host path is not sandbox://, hugepages://, \\.\pipe, uvm://
+		// Ensure that host path is not sandbox://, sandbox-tmp://, hugepages://, \\.\pipe, uvm://
 		if strings.HasPrefix(hostPath, guestpath.SandboxMountPrefix) ||
+			strings.HasPrefix(hostPath, guestpath.SandboxTmpfsMountPrefix) ||
 			strings.HasPrefix(hostPath, guestpath.HugePagesMountPrefix) ||
 			strings.HasPrefix(hostPath, guestpath.PipePrefix) ||
 			strings.HasPrefix(hostPath, guestpath.UVMMountPrefix) {

internal/guest/runtime/hcsv2/uvm.go

@@ -290,6 +290,31 @@ func setupSandboxMountsPath(id string) (err error) {
 	return storage.MountRShared(mountPath)
 }
 
+func setupSandboxTmpfsMountsPath(id string) error {
+	var err error
+	tmpfsDir := specGuest.SandboxTmpfsMountsDir(id)
+	if err := os.MkdirAll(tmpfsDir, 0755); err != nil {
+		return errors.Wrapf(err, "failed to create sandbox tmpfs mounts dir in sandbox %v", id)
+	}
+
+	defer func() {
+		if err != nil {
+			_ = os.RemoveAll(tmpfsDir)
+		}
+	}()
+
+	// mount a tmpfs at the tmpfsDir
+	// this ensures that the tmpfsDir is a mount point and not just a directory
+	// we don't care if it is already mounted, so ignore EBUSY
+	if err := unix.Mount("tmpfs", tmpfsDir, "tmpfs", 0, ""); err != nil && !errors.Is(err, unix.EBUSY) {
+		return errors.Wrapf(err, "failed to mount tmpfs at %s", tmpfsDir)
+	}
+
+	//TODO: should tmpfs be mounted as noexec?
+
+	return storage.MountRShared(tmpfsDir)
+}
+
 func setupSandboxHugePageMountsPath(id string) error {
 	mountPath := specGuest.HugePagesMountsDir(id)
 	if err := os.MkdirAll(mountPath, 0755); err != nil {
@@ -351,6 +376,10 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 				return nil, err
 			}
 
+			if err = setupSandboxTmpfsMountsPath(id); err != nil {
+				return nil, err
+			}
+
 			if err = setupSandboxHugePageMountsPath(id); err != nil {
 				return nil, err
 			}

internal/guest/runtime/hcsv2/workload_container.go

@@ -34,15 +34,28 @@ func mkdirAllModePerm(target string) error {
 
 func updateSandboxMounts(sbid string, spec *oci.Spec) error {
 	for i, m := range spec.Mounts {
-		if !strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) {
+		if !strings.HasPrefix(m.Source, guestpath.SandboxMountPrefix) &&
+			!strings.HasPrefix(m.Source, guestpath.SandboxTmpfsMountPrefix) {
 			continue
 		}
-		sandboxSource := specGuest.SandboxMountSource(sbid, m.Source)
 
-		// filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given.
-		// Hence, we need to ensure that the resolved path is still under the correct directory
-		if !strings.HasPrefix(sandboxSource, specGuest.SandboxMountsDir(sbid)) {
-			return errors.Errorf("mount path %v for mount %v is not within sandbox's mounts dir", sandboxSource, m.Source)
+		var sandboxSource string
+		// if using `sandbox-tmp://` prefix, we mount a tmpfs in sandboxTmpfsMountsDir
+		if strings.HasPrefix(m.Source, guestpath.SandboxTmpfsMountPrefix) {
+			sandboxSource = specGuest.SandboxTmpfsMountSource(sbid, m.Source)
+			// filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given.
+			// Hence, we need to ensure that the resolved path is still under the correct directory
+			if !strings.HasPrefix(sandboxSource, specGuest.SandboxTmpfsMountsDir(sbid)) {
+				return errors.Errorf("mount path %v for mount %v is not within sandboxTmpfsMountsDir", sandboxSource, m.Source)
+			}
+
+		} else {
+			sandboxSource = specGuest.SandboxMountSource(sbid, m.Source)
+			// filepath.Join cleans the resulting path before returning, so it would resolve the relative path if one was given.
+			// Hence, we need to ensure that the resolved path is still under the correct directory
+			if !strings.HasPrefix(sandboxSource, specGuest.SandboxMountsDir(sbid)) {
+				return errors.Errorf("mount path %v for mount %v is not within sandbox's mounts dir", sandboxSource, m.Source)
+			}
 		}
 
 		spec.Mounts[i].Source = sandboxSource

internal/guest/spec/spec.go

@@ -84,6 +84,11 @@ func SandboxMountsDir(sandboxID string) string {
 	return filepath.Join(SandboxRootDir(sandboxID), "sandboxMounts")
 }
 
+// SandboxTmpfsMountsDir returns sandbox tmpfs mounts directory inside UVM.
+func SandboxTmpfsMountsDir(sandboxID string) string {
+	return filepath.Join(SandboxRootDir(sandboxID), "sandboxTmpfsMounts")
+}
+
 // HugePagesMountsDir returns hugepages mounts directory inside UVM.
 func HugePagesMountsDir(sandboxID string) string {
 	return filepath.Join(SandboxRootDir(sandboxID), "hugepages")
@@ -96,6 +101,13 @@ func SandboxMountSource(sandboxID, path string) string {
 	return filepath.Join(mountsDir, subPath)
 }
 
+// SandboxTmpfsMountSource returns sandbox tmpfs mount path inside UVM
+func SandboxTmpfsMountSource(sandboxID, path string) string {
+	tmpfsMountDir := SandboxTmpfsMountsDir(sandboxID)
+	subPath := strings.TrimPrefix(path, guestpath.SandboxTmpfsMountPrefix)
+	return filepath.Join(tmpfsMountDir, subPath)
+}
+
 // HugePagesMountSource returns hugepages mount path inside UVM
 func HugePagesMountSource(sandboxID, path string) string {
 	mountsDir := HugePagesMountsDir(sandboxID)

internal/guestpath/paths.go

@@ -10,6 +10,9 @@ const (
 	// SandboxMountPrefix is mount prefix used in container spec to mark a
 	// sandbox-mount
 	SandboxMountPrefix = "sandbox://"
+	// SandboxTmpfsMountPrefix is mount prefix used in container spec to mark a
+	// sandbox-tmp mount
+	SandboxTmpfsMountPrefix = "sandbox-tmp://"
 	// HugePagesMountPrefix is mount prefix used in container spec to mark a
 	// huge-pages mount
 	HugePagesMountPrefix = "hugepages://"

internal/hcsoci/resources_lcow.go

@@ -129,10 +129,13 @@ func allocateLinuxResources(ctx context.Context, coi *createOptionsInternal, r *
 					mt = "bind"
 				}
 				coi.Spec.Mounts[i].Type = mt
-			} else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) || strings.HasPrefix(mount.Source, guestpath.UVMMountPrefix) {
-				// Mounts that map to a path in UVM are specified with a 'sandbox://' or 'uvm://' prefix.
+			} else if strings.HasPrefix(mount.Source, guestpath.SandboxMountPrefix) ||
+				strings.HasPrefix(mount.Source, guestpath.SandboxTmpfsMountPrefix) ||
+				strings.HasPrefix(mount.Source, guestpath.UVMMountPrefix) {
+				// Mounts that map to a path in UVM are specified with a 'sandbox://', 'sandbox-tmp://', or 'uvm://' prefix.
 				// examples:
 				//  - sandbox:///a/dirInUvm destination:/b/dirInContainer
+				//  - sandbox-tmp:///a/dirInUvm destination:/b/dirInContainer
 				//  - uvm:///a/dirInUvm destination:/b/dirInContainer
 				uvmPathForFile = mount.Source
 			} else if strings.HasPrefix(mount.Source, guestpath.HugePagesMountPrefix) {