Implements CVM disk backup

certbot/cli/src/main.rs

@@ -121,7 +121,7 @@ fn load_config(config: &PathBuf) -> Result<CertBotConfig> {
     let renew_timeout = Duration::from_secs(config.renew_timeout);
     let bot_config = CertBotConfig::builder()
         .acme_url(config.acme_url)
-        .cert_dir(workdir.backup_dir())
+        .cert_dir(workdir.cert_backup_dir())
         .cert_file(workdir.cert_path())
         .key_file(workdir.key_path())
         .auto_create_account(true)

certbot/src/workdir.rs

@@ -27,24 +27,24 @@ impl WorkDir {
         self.workdir.join("credentials.json")
     }
 
-    pub fn backup_dir(&self) -> PathBuf {
+    pub fn cert_backup_dir(&self) -> PathBuf {
         self.workdir.join("backup")
     }
 
-    pub fn live_dir(&self) -> PathBuf {
+    pub fn cert_live_dir(&self) -> PathBuf {
         self.workdir.join("live")
     }
 
     pub fn cert_path(&self) -> PathBuf {
-        self.live_dir().join("cert.pem")
+        self.cert_live_dir().join("cert.pem")
     }
 
     pub fn key_path(&self) -> PathBuf {
-        self.live_dir().join("key.pem")
+        self.cert_live_dir().join("key.pem")
     }
 
     pub fn list_certs(&self) -> Result<Vec<PathBuf>> {
-        crate::bot::list_certs(self.backup_dir())
+        crate::bot::list_certs(self.cert_backup_dir())
     }
 
     pub fn acme_account_uri(&self) -> Result<String> {
@@ -58,6 +58,6 @@ impl WorkDir {
     }
 
     pub fn list_cert_public_keys(&self) -> Result<BTreeSet<Vec<u8>>> {
-        crate::bot::list_cert_public_keys(self.backup_dir())
+        crate::bot::list_cert_public_keys(self.cert_backup_dir())
     }
 }

gateway/src/config.rs

@@ -210,7 +210,7 @@ impl CertbotConfig {
         let workdir = certbot::WorkDir::new(&self.workdir);
         certbot::CertBotConfig::builder()
             .auto_create_account(true)
-            .cert_dir(workdir.backup_dir())
+            .cert_dir(workdir.cert_backup_dir())
             .cert_file(workdir.cert_path())
             .key_file(workdir.key_path())
             .credentials_file(workdir.account_credentials_path())

guest-agent/Cargo.toml

@@ -15,7 +15,7 @@ fs-err.workspace = true
 rcgen.workspace = true
 sha2.workspace = true
 clap.workspace = true
-tokio.workspace = true
+tokio = { workspace = true, features = ["full"] }
 hex.workspace = true
 serde_json.workspace = true
 bollard.workspace = true

guest-agent/src/guest_api_service.rs

@@ -1,6 +1,6 @@
 use std::fmt::Debug;
 
-use anyhow::{Context, Result};
+use anyhow::{bail, Context, Result};
 use bollard::{container::ListContainersOptions, Docker};
 use cmd_lib::{run_cmd as cmd, run_fun};
 use dstack_guest_agent_rpc::worker_server::WorkerRpc as _;
@@ -18,6 +18,8 @@ use tracing::error;
 
 use crate::{rpc_service::ExternalRpcHandler, AppState};
 
+const BACKUP_LOCK_FILE: &str = "/run/dstack-backup.lock";
+
 pub struct GuestApiHandler {
     state: AppState,
 }
@@ -43,6 +45,7 @@ impl GuestApiRpc for GuestApiHandler {
             device_id: info.device_id,
             app_cert: info.app_cert,
             tcb_info: info.tcb_info,
+            backup_in_progress: fs::metadata(BACKUP_LOCK_FILE).is_ok(),
         })
     }
 
@@ -112,6 +115,53 @@ impl GuestApiRpc for GuestApiHandler {
     async fn list_containers(self) -> Result<ListContainersResponse> {
         list_containers().await
     }
+
+    async fn pre_backup(self) -> Result<()> {
+        fs::OpenOptions::new()
+            .create_new(true)
+            .write(true)
+            .open(BACKUP_LOCK_FILE)
+            .context("Failed to create backup lock file, there is another backup in progress")?;
+        // Run /dstack/hooks/pre-backup if it exists
+        let pre_backup_hook = "/dstack/hooks/pre-backup";
+        if is_exe(pre_backup_hook) {
+            let status = tokio::process::Command::new(pre_backup_hook)
+                .spawn()
+                .context("Failed to run pre-backup hook")?
+                .wait()
+                .await
+                .context("Failed to run pre-backup hook")?;
+            if !status.success() {
+                bail!("Failed to run pre-backup hook");
+            }
+        }
+        Ok(())
+    }
+
+    async fn post_backup(self) -> Result<()> {
+        fs::remove_file(BACKUP_LOCK_FILE).context("Failed to remove backup lock file")?;
+        let post_backup_hook = "/dstack/hooks/post-backup";
+        if is_exe(post_backup_hook) {
+            let status = tokio::process::Command::new(post_backup_hook)
+                .spawn()
+                .context("Failed to run post-backup hook")?
+                .wait()
+                .await
+                .context("Failed to run post-backup hook")?;
+            if !status.success() {
+                bail!("Failed to run post-backup hook");
+            }
+        }
+        Ok(())
+    }
+}
+
+fn is_exe(path: &str) -> bool {
+    use std::os::unix::fs::PermissionsExt;
+    let Ok(metadata) = fs::metadata(path) else {
+        return false;
+    };
+    metadata.is_file() && metadata.permissions().mode() & 0o111 != 0
 }
 
 pub(crate) async fn list_containers() -> Result<ListContainersResponse> {

guest-api/proto/guest_api.proto

@@ -22,6 +22,8 @@ message GuestInfo {
   string tcb_info = 5;
   // Device ID
   bytes device_id = 6;
+  // true if backup is in progress
+  bool backup_in_progress = 7;
 }
 
 message IpAddress {
@@ -123,6 +125,8 @@ service GuestApi {
   rpc NetworkInfo(google.protobuf.Empty) returns (NetworkInformation);
   rpc ListContainers(google.protobuf.Empty) returns (ListContainersResponse);
   rpc Shutdown(google.protobuf.Empty) returns (google.protobuf.Empty);
+  rpc PreBackup(google.protobuf.Empty) returns (google.protobuf.Empty);
+  rpc PostBackup(google.protobuf.Empty) returns (google.protobuf.Empty);
 }
 
 service ProxiedGuestApi {
@@ -131,4 +135,6 @@ service ProxiedGuestApi {
   rpc NetworkInfo(Id) returns (NetworkInformation);
   rpc ListContainers(Id) returns (ListContainersResponse);
   rpc Shutdown(Id) returns (google.protobuf.Empty);
+  rpc PreBackup(Id) returns (google.protobuf.Empty);
+  rpc PostBackup(Id) returns (google.protobuf.Empty);
 }