fsync completed files, and directories before renames

src/host/ledger.h

@@ -226,7 +226,6 @@ namespace asynchost
       const auto mode = committed ? "rb" : "r+b";
 
       file = fopen(file_path.c_str(), mode);
-
       if (!file)
       {
         throw std::logic_error(fmt::format(
@@ -630,6 +629,26 @@ namespace asynchost
           fmt::format("Failed to flush ledger file: {}", strerror(errno)));
       }
 
+      if (fsync(fileno(file)) != 0)
+      {
+        throw std::logic_error(fmt::format(
+          "Failed to sync completed ledger file: {}", strerror(errno)));
+      }
+
+      auto parent_dir = fopen(dir.c_str(), "r");
+      if (!parent_dir)
+      {
+        throw std::logic_error(fmt::format(
+          "Unable to open ledger directory {}: {}", dir, strerror(errno)));
+      }
+      if (fsync(fileno(parent_dir)) != 0)
+      {
+        throw std::logic_error(fmt::format(
+          "Failed to sync ledger directory after completing file: {}",
+          strerror(errno)));
+      }
+      fclose(parent_dir);
+
       LOG_TRACE_FMT("Completed ledger file {}", file_name);
 
       completed = true;
@@ -643,6 +662,20 @@ namespace asynchost
       try
       {
         files::rename(file_path, new_file_path);
+
+        auto parent_dir = fopen(dir.c_str(), "r");
+        if (!parent_dir)
+        {
+          throw std::logic_error(fmt::format(
+            "Unable to open ledger directory {}: {}", dir, strerror(errno)));
+        }
+        if (fsync(fileno(parent_dir)) != 0)
+        {
+          throw std::logic_error(fmt::format(
+            "Failed to sync ledger directory after renaming file: {}",
+            strerror(errno)));
+        }
+        fclose(parent_dir);
       }
       catch (const std::exception& e)
       {

src/snapshots/snapshot_manager.h

@@ -110,18 +110,50 @@ namespace snapshots
       {
         asynchost::TimeBoundLogger log_if_slow(
           fmt::format("Committing snapshot - fsync({})", data->tmp_file_name));
-        fsync(data->snapshot_fd);
+        if (fsync(data->snapshot_fd) != 0)
+        {
+          throw std::logic_error(
+            fmt::format("Failed to sync snapshot file: {}", strerror(errno)));
+        }
       }
 
       close(data->snapshot_fd);
 
+      auto parent_dir = fopen(data->dir.c_str(), "r");
+      if (parent_dir)
+      {
+        asynchost::TimeBoundLogger log_if_slow(fmt::format(
+          "Committing snapshot - fsync({}) (after file close)", data->dir));
+        if (fsync(fileno(parent_dir)) != 0)
+        {
+          throw std::logic_error(fmt::format(
+            "Failed to sync snapshot directory after closing committed "
+            "snapshot file: {}",
+            strerror(errno)));
+        }
+      }
+
       // e.g. snapshot_100_105.committed
       data->committed_file_name =
         fmt::format("{}{}", data->tmp_file_name, snapshot_committed_suffix);
       const auto full_committed_path = data->dir / data->committed_file_name;
 
       const auto full_tmp_path = data->dir / data->tmp_file_name;
       files::rename(full_tmp_path, full_committed_path);
+
+      if (parent_dir)
+      {
+        asynchost::TimeBoundLogger log_if_slow(fmt::format(
+          "Committing snapshot - fsync({}) (after rename)", data->dir));
+        if (fsync(fileno(parent_dir)) != 0)
+        {
+          throw std::logic_error(fmt::format(
+            "Failed to sync snapshot directory after renaming file: {}",
+            strerror(errno)));
+        }
+      }
+
+      fclose(parent_dir);
     }
 
     static void on_snapshot_sync_and_rename_complete(uv_work_t* req, int status)
@@ -162,6 +194,26 @@ namespace snapshots
               it->second.evidence_idx);
             auto full_snapshot_path = snapshot_dir / file_name;
 
+#define THROW_ON_ERROR(x) \
+  do \
+  { \
+    auto rc = x; \
+    if (rc == -1) \
+    { \
+      throw std::runtime_error(fmt::format( \
+        "Error ({}) writing snapshot {} in " #x, errno, file_name)); \
+    } \
+  } while (0)
+
+            int dir_fd = open(snapshot_dir.c_str(), O_DIRECTORY);
+            if (dir_fd == -1)
+            {
+              throw std::runtime_error(fmt::format(
+                "Error ({}) opening snapshots directory {}",
+                errno,
+                snapshot_dir));
+            }
+
             int snapshot_fd = open(
               full_snapshot_path.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0664);
             if (snapshot_fd == -1)
@@ -222,6 +274,8 @@ namespace snapshots
 #endif
             }
 
+#undef THROW_ON_ERROR
+
             pending_snapshots.erase(it);
 
             return;