Skip to content

Commit a6db1e9

Browse files
andre-j3susandre-j3sus
committed
Add support for remote Firecracker snapshots
- When remote snapshots are enabled, after committing the snapshot, it is uploaded to a MinIO instance. When loading from a snapshot, if it is not available locally, it checks if it is available in MinIO and fetches it. - Remote Firecracker snapshots are currently only supported using the Stargz snapshotter (there are some container corruption issues when using devmapper). Signed-off-by: André Jesus <[email protected]>
1 parent 03331f5 commit a6db1e9

27 files changed

+1382
-170
lines changed

‎.github/workflows/unit_tests.yml‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,3 +170,71 @@ jobs:
170170
- name: Cleaning
171171
if: ${{ always() }}
172172
run: ./scripts/setup_tool clean_fcctr
173+
174+
remote-firecracker-snapshots-test:
175+
name: "Unit tests: Remote Firecracker snapshots"
176+
runs-on: ubuntu-24.04
177+
strategy:
178+
fail-fast: false
179+
matrix:
180+
module: [storage, snapshotting, ctriface]
181+
services:
182+
minio: # MinIO service for testing remote snapshots
183+
image: lazybit/minio # Can't use minio/minio because there's still no support for the jobs.<job_id>.services.<service_id>.command option in GH Actions
184+
ports:
185+
- 9000:9000
186+
options: >-
187+
--health-cmd "curl -f http://localhost:9000/minio/health/live || exit 1"
188+
--health-interval 5s
189+
--health-timeout 5s
190+
--health-retries 5
191+
volumes:
192+
- ${{ github.workspace }}/data:/data
193+
env:
194+
MINIO_ROOT_USER: minio
195+
MINIO_ROOT_PASSWORD: minio123
196+
# command: server /data
197+
steps:
198+
- name: Check out code into the Go module directory
199+
uses: actions/checkout@v4
200+
with:
201+
lfs: true
202+
path: vhive # Use a specific path to avoid conflicts with the MinIO volume
203+
204+
- name: Set up Go version in go.mod file
205+
uses: actions/setup-go@v5
206+
with:
207+
go-version-file: ${{ github.workspace }}/vhive/go.mod
208+
cache-dependency-path: |
209+
**/go.sum
210+
**/go.mod
211+
212+
- name: Add rsync
213+
run: |
214+
sudo apt update
215+
sudo apt install rsync -y
216+
217+
- name: Cleanup runner disk space
218+
run: ./vhive/scripts/github_runner/gh-actions-disk-cleanup.sh
219+
220+
- name: Build setup scripts
221+
run: pushd ./vhive/scripts && go build -o setup_tool && popd
222+
223+
- name: Pull binaries
224+
run: ./vhive/scripts/setup_tool setup_firecracker_containerd
225+
226+
- name: Setup Stargz
227+
run: ./vhive/scripts/setup_tool setup_stargz firecracker
228+
229+
- name: Build
230+
run: cd ./vhive && go build -race -v -a ./...
231+
232+
- name: Run tests in submodules
233+
env:
234+
MODULE: ${{ matrix.module }}
235+
run: |
236+
make -C vhive/$MODULE test-remote-snaps
237+
238+
- name: Cleaning
239+
if: ${{ always() }}
240+
run: ./vhive/scripts/setup_tool clean_fcctr

‎Makefile‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,9 @@ EXTRATESTFILES:=vhive_test.go stats.go vhive.go functions.go
2929
# WITHLAZY:=-lazyTest
3030
WITHUPF:=
3131
WITHLAZY:=
32-
WITHSNAPSHOTS:=-snapshotsTest
32+
STARGZ:=-ss 'proxy' -img 'ghcr.io/vhive-serverless/helloworld:var_workload-esgz'
33+
DOCKER_CREDENTIALS:=-dockerCredentials '{"docker-credentials":{"ghcr.io":{"username":"","password":""}}}'
34+
WITH_LOCAL_SNAPSHOTS:=-snapshotsTest 'local'
3335
CTRDLOGDIR:=/tmp/ctrd-logs
3436

3537
vhive: proto
@@ -51,13 +53,13 @@ test:
5153
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS)
5254
./scripts/clean_fcctr.sh
5355
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log.err &
54-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS)
56+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITH_LOCAL_SNAPSHOTS)
5557
./scripts/clean_fcctr.sh
5658
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_upf_log.out 2>$(CTRDLOGDIR)/fccd_orch_upf_log.err &
57-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF)
59+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF)
5860
./scripts/clean_fcctr.sh
5961
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.out 2>$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.err &
60-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY)
62+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -short $(EXTRAGOARGS) -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF) $(WITHLAZY)
6163
./scripts/clean_fcctr.sh
6264
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
6365
sudo env "PATH=$(PATH)" go test -short $(EXTRAGOARGS) -run TestProfileSingleConfiguration -args -test -loadStep 100 && sudo rm -rf bench_results
@@ -70,46 +72,46 @@ test-man:
7072
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_travis.err &
7173
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe
7274
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree
73-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS)
75+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITH_LOCAL_SNAPSHOTS)
7476
./scripts/clean_fcctr.sh
7577
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_both_log_man_travis.out 2>$(CTRDLOGDIR)/fccd_orch_both_log_man_travis.err &
76-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) $(WITHUPF)
78+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF)
7779
./scripts/clean_fcctr.sh
7880
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_travis.out 2>$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_travis.err &
79-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY)
81+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestServeThree -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF) $(WITHLAZY)
8082
./scripts/clean_fcctr.sh
8183

8284
test-skip:
8385
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_skip.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_man_skip.err &
84-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe -args $(WITHSNAPSHOTS)
86+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe -args $(WITH_LOCAL_SNAPSHOTS)
8587
./scripts/clean_fcctr.sh
8688
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_both_log_man_skip.out 2>$(CTRDLOGDIR)/fccd_orch_both_log_man_skip.err &
87-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe -args $(WITHSNAPSHOTS) $(WITHUPF)
89+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF)
8890
./scripts/clean_fcctr.sh
8991
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_skip.out 2>$(CTRDLOGDIR)/fccd_orch_both_lazy_log_man_skip.err &
90-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY)
92+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS_NORACE) -run TestParallelServe -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF) $(WITHLAZY)
9193
./scripts/clean_fcctr.sh
9294

9395
bench:
9496
./scripts/clean_fcctr.sh
9597
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
96-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase
98+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITH_LOCAL_SNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase
9799
./scripts/clean_fcctr.sh
98100
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
99-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP
101+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP
100102
./scripts/clean_fcctr.sh
101103
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
102-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy
104+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchServe -args -iter 1 $(WITH_LOCAL_SNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy
103105
./scripts/clean_fcctr.sh
104106

105107
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
106-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase
108+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITH_LOCAL_SNAPSHOTS) -benchDirTest configBase -metricsTest -funcName helloworld && sudo rm -rf configBase
107109
./scripts/clean_fcctr.sh
108110
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
109-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP
111+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF) -benchDirTest configREAP -metricsTest -funcName helloworld && sudo rm -rf configREAP
110112
./scripts/clean_fcctr.sh
111113
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
112-
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy
114+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITH_LOCAL_SNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest -funcName helloworld && sudo rm -rf configLazy
113115
./scripts/clean_fcctr.sh
114116

115117
test-man-bench:
@@ -119,13 +121,13 @@ test-man-bench:
119121
nightly-test:
120122
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log.err &
121123
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS)
122-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS) -args $(WITHSNAPSHOTS)
124+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS) -args $(WITH_LOCAL_SNAPSHOTS)
123125
./scripts/clean_fcctr.sh
124126
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_upf_log.out 2>$(CTRDLOGDIR)/fccd_orch_upf_log.err &
125-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF)
127+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS) -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF)
126128
./scripts/clean_fcctr.sh
127129
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.out 2>$(CTRDLOGDIR)/fccd_orch_upf_lazy_log.err &
128-
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS) -args $(WITHSNAPSHOTS) $(WITHUPF) $(WITHLAZY)
130+
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) -run TestAllFunctions $(EXTRAGOARGS) -args $(WITH_LOCAL_SNAPSHOTS) $(WITHUPF) $(WITHLAZY)
129131
./scripts/clean_fcctr.sh
130132

131133
test-skip-all:

‎bench_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func TestBenchParallelServe(t *testing.T) {
6363
imageName, isPresent := images[*funcName]
6464
require.True(t, isPresent, "Function is not supported")
6565

66-
funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
66+
funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, *snapshotTestMode)
6767

6868
createResultsDir()
6969

@@ -136,7 +136,7 @@ func TestBenchWarmServe(t *testing.T) {
136136
imageName, isPresent := images[*funcName]
137137
require.True(t, isPresent, "Function is not supported")
138138

139-
funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
139+
funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, *snapshotTestMode)
140140

141141
createResultsDir()
142142

@@ -201,7 +201,7 @@ func TestBenchServe(t *testing.T) {
201201
imageName, isPresent := images[*funcName]
202202
require.True(t, isPresent, "Function is not supported")
203203

204-
funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
204+
funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst, *snapshotTestMode)
205205

206206
createResultsDir()
207207

‎configs/.wordlist.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,7 @@ SMI
465465
sms
466466
SMT
467467
snapshotted
468+
snapshotters
468469
snapshotting
469470
SoC
470471
SOCACHE

‎cri/firecracker/coordinator.go‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ import (
3535

3636
log "github.com/sirupsen/logrus"
3737
"github.com/vhive-serverless/vhive/ctriface"
38+
39+
"github.com/vhive-serverless/vhive/storage"
40+
41+
"github.com/minio/minio-go/v7"
42+
"github.com/minio/minio-go/v7/pkg/credentials"
3843
)
3944

4045
type coordinator struct {
@@ -67,10 +72,27 @@ func newFirecrackerCoordinator(orch *ctriface.Orchestrator, opts ...coordinatorO
6772
}
6873

6974
snapshotsDir := "/fccd/test/snapshots"
75+
var objectStore storage.ObjectStorage
76+
7077
if !c.withoutOrchestrator {
7178
snapshotsDir = orch.GetSnapshotsDir()
79+
snapshotsBucket := orch.GetSnapshotsBucket()
80+
81+
if orch.GetSnapshotMode() == "remote" {
82+
minioClient, _ := minio.New(orch.GetMinioAddr(), &minio.Options{
83+
Creds: credentials.NewStaticV4(orch.GetMinioAccessKey(), orch.GetMinioSecretKey(), ""),
84+
Secure: false,
85+
})
86+
87+
var err error
88+
objectStore, err = storage.NewMinioStorage(minioClient, snapshotsBucket)
89+
if err != nil {
90+
log.WithError(err).Fatalf("failed to create MinIO storage for snapshots in bucket %s", snapshotsBucket)
91+
}
92+
}
7293
}
73-
c.snapshotManager = snapshotting.NewSnapshotManager(snapshotsDir)
94+
95+
c.snapshotManager = snapshotting.NewSnapshotManager(snapshotsDir, objectStore)
7496

7597
return c
7698
}
@@ -80,9 +102,19 @@ func (c *coordinator) startVM(ctx context.Context, image, revision string) (*fun
80102
}
81103

82104
func (c *coordinator) startVMWithEnvironment(ctx context.Context, image, revision string, environment []string) (*funcInstance, error) {
83-
if c.orch != nil && c.orch.GetSnapshotsEnabled() {
105+
if c.orch != nil && c.orch.GetSnapshotMode() != "disabled" {
84106
// Check if snapshot is available
85-
if snap, err := c.snapshotManager.AcquireSnapshot(revision); err == nil {
107+
if snap, _ := c.snapshotManager.AcquireSnapshot(revision); snap == nil {
108+
if c.orch.GetSnapshotMode() == "remote" {
109+
if _, err := c.snapshotManager.DownloadSnapshot(revision); err != nil {
110+
log.WithError(err).Errorf("failed to download snapshot %s from remote storage", revision)
111+
} else {
112+
log.Printf("downloaded snapshot %s from remote storage", revision)
113+
}
114+
}
115+
}
116+
117+
if snap, _ := c.snapshotManager.AcquireSnapshot(revision); snap != nil {
86118
return c.orchLoadInstance(ctx, snap)
87119
}
88120
}
@@ -102,7 +134,7 @@ func (c *coordinator) stopVM(ctx context.Context, containerID string) error {
102134
return nil
103135
}
104136

105-
if c.orch != nil && c.orch.GetSnapshotsEnabled() && !fi.SnapBooted {
137+
if c.orch != nil && c.orch.GetSnapshotMode() != "disabled" && !fi.SnapBooted {
106138
err := c.orchCreateSnapshot(ctx, fi)
107139
if err != nil {
108140
log.Printf("Err creating snapshot %s\n", err)
@@ -230,11 +262,22 @@ func (c *coordinator) orchCreateSnapshot(ctx context.Context, fi *funcInstance)
230262
}
231263
}
232264

265+
if err := snap.SerializeSnapInfo(); err != nil {
266+
fi.Logger.WithError(err).Error("failed to serialize snapshot info")
267+
return err
268+
}
269+
233270
if err := c.snapshotManager.CommitSnapshot(fi.Revision); err != nil {
234271
fi.Logger.WithError(err).Error("failed to commit snapshot")
235272
return err
236273
}
237274

275+
if !c.withoutOrchestrator && c.orch.GetSnapshotMode() == "remote" {
276+
if err := c.snapshotManager.UploadSnapshot(fi.Revision); err != nil {
277+
fi.Logger.WithError(err).Error("failed to upload snapshot")
278+
}
279+
}
280+
238281
return nil
239282
}
240283

‎ctriface/Makefile‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ STARGZ:=-ss 'proxy' -img 'ghcr.io/vhive-serverless/helloworld:var_workload-esgz'
3232
DOCKER_CREDENTIALS:=-dockerCredentials '{"docker-credentials":{"ghcr.io":{"username":"","password":""}}}'
3333
GOBENCH:=-v -timeout 1500s
3434
CTRDLOGDIR:=/tmp/ctrd-logs
35+
REMOTE_SNAPS:=-remote-snaps
3536

3637
test:
3738
./../scripts/clean_fcctr.sh
@@ -83,6 +84,22 @@ test-skip:
8384
./../scripts/clean_fcctr.sh
8485

8586

87+
test-remote-snaps:
88+
# Creates a remote snapshot.
89+
sudo env "PATH=$(PATH)" /usr/local/bin/http-address-resolver &
90+
sudo env "PATH=$(PATH)" /bin/bash -c 'while true; do /usr/local/bin/demux-snapshotter; done' &
91+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_remote_snap_create.out 2>$(CTRDLOGDIR)/ctriface_log_remote_snap_create.err &
92+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestRemoteSnapCreate -args $(STARGZ) $(DOCKER_CREDENTIALS) $(REMOTE_SNAPS)
93+
# Cleans up the node and remove the local snapshot.
94+
./../scripts/clean_fcctr.sh && sudo rm -r /tmp/vhive/remote-snapshots
95+
# Loads the remote snapshot.
96+
sudo env "PATH=$(PATH)" /usr/local/bin/http-address-resolver &
97+
sudo env "PATH=$(PATH)" /bin/bash -c 'while true; do /usr/local/bin/demux-snapshotter; done' &
98+
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log_remote_snap_load.out 2>$(CTRDLOGDIR)/ctriface_log_remote_snap_load.err &
99+
sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestRemoteSnapLoad -args $(STARGZ) $(DOCKER_CREDENTIALS) $(REMOTE_SNAPS)
100+
./../scripts/clean_fcctr.sh
101+
102+
86103
bench:
87104
sudo env "PATH=$(PATH)" go test $(BENCHFILES) $(GOBENCH)
88105
./../scripts/clean_fcctr.sh

0 commit comments

Comments
 (0)