cmd/photocamera-archiver: make archives self-contained

The IA deals poorly with more than a handful of files.

Author: FiloSottile

cmd/photocamera-archiver/photocamera.go

@@ -1,22 +1,12 @@
 // Command photocamera-archiver creates an archive of a Static Certificate
 // Transparency log, by compressing tiles into zip files, each containing a
-// subtree 16,777,216 entries wide (65,536 level -1 and 0 tiles, 256 level 1
-// tiles, and 1 level 2 tile). The checkpoint, JSON metadata, and level 3+ tiles
-// are left uncompressed. The zip files are stored at tile/zip/<N>.zip.
-// Unnecessary partial tiles at levels 3+ are also removed.
-//
-// After running this tool, archive the following files and directories:
-//
-//   - checkpoint
-//   - log.v3.json
-//   - tile/zip/
-//   - tile/3/
-//   - tile/4/ (if present)
-//   - issuer/
+// subtree of up to 16,777,216 entries (65,536 level -1 and 0 tiles, 256 level 1
+// tiles, and 1 level 2 tile). The checkpoint, JSON metadata, issuers, level 3+
+// tiles, and partial tiles on the right edge are included in every zip file.
+// The zip files are stored at archive/<N>.zip.
 package main
 
 import (
-	"bytes"
 	"context"
 	"crypto/x509"
 	"encoding/json"
@@ -37,6 +27,39 @@ import (
 	"golang.org/x/mod/sumdb/tlog"
 )
 
+const README = `This is an archive of a Certificate Transparency log, stored in the
+c2sp.org/[email protected] format, although if this log was originally served
+through RFC 6962 APIs, leaves might miss the LeafIndex extension.
+
+The log is split over multiple zip files. Archive <N>.zip contains the tiles
+at levels -1 (the data entries), 0, 1, and 2 belonging to subtree
+
+    [ 256×256×256×N, min(256×256×256×(N+1), TreeSize) )
+
+In other words, each zip file contains one level 2 tile, and all those below it.
+
+Every zip file also contains the following files:
+
+  - README.txt — this file
+
+  - checkpoint — a Signed Tree Head in c2sp.org/tlog-checkpoint format
+
+  - log.v3.json — a JSON specification of the log, including its log ID, public
+                  key, and original URL
+
+  - issuers/* — the X.509 chain issuers for the whole log
+
+  - tile/{3,4}/* — higher-level tiles, including the hash of other level 2
+                   tiles not included in this archive
+
+  - tile/{0,1,2}/*.p/* — partial tiles on the right edge of the tree, if any,
+                         necessary to compute the tree head
+
+This archive was generated by filippo.io/sunlight/cmd/photocamera-archiver.
+`
+
+const archiveWidth = 256 * 256 * 256
+
 func main() {
 	logger := slog.New(stdlog.Handler)
 
@@ -47,8 +70,8 @@ func main() {
 	if err != nil {
 		fatalError(logger, "failed to open local directory", "err", err)
 	}
-	if err := root.MkdirAll("tile/zip", 0o755); err != nil {
-		fatalError(logger, "failed to create zip directory", "err", err)
+	if err := root.MkdirAll("archive", 0o755); err != nil {
+		fatalError(logger, "failed to create archive directory", "err", err)
 	}
 	tr := localTileReader{root.FS()}
 
@@ -88,28 +111,68 @@ func main() {
 	}
 	logger.Info("loaded checkpoint", "tree_size", c.N, "root_hash", c.Hash)
 
+	var globalTiles []tlog.Tile
+	for L := 5; L >= 0; L-- {
+		levelHashes := c.N / int64(1<<(sunlight.TileHeight*L))
+		if L >= 3 {
+			// All level 3+ tiles.
+			for N := int64(0); (N * 256) < levelHashes; N++ {
+				W := int(min(256, levelHashes-(N*256)))
+				tile := tlog.Tile{H: sunlight.TileHeight, L: L, N: N, W: W}
+				globalTiles = append(globalTiles, tile)
+			}
+		} else {
+			// Only the partial tile on the right edge, if any.
+			if W := int(levelHashes % 256); W != 0 {
+				N := levelHashes / 256
+				tile := tlog.Tile{H: sunlight.TileHeight, L: L, N: N, W: W}
+				globalTiles = append(globalTiles, tile)
+			}
+		}
+	}
+
 	hr := torchwood.TileHashReaderWithContext(ctx, c.Tree, tr)
 
-	for n := int64(0); n < c.N; n += 256 * 256 * 256 {
-		i := n / (256 * 256 * 256)
+	for n := int64(0); n < c.N; n += archiveWidth {
+		i := n / archiveWidth
 		if i >= 1000 {
 			fatalError(logger, "cannot archive more than 1000 zip files")
 		}
-		name := fmt.Sprintf("tile/zip/%03d.zip", i)
-		subtree := min(256*256*256, c.N-n)
+		name := fmt.Sprintf("archive/%03d.zip", i)
+		subtree := min(archiveWidth, c.N-n)
 		logger.Info("processing subtree", "name", name, "start", n, "end", n+subtree)
 		f, err := root.Create(name)
 		if err != nil {
 			fatalError(logger, "failed to create zip file", "name", name, "err", err)
 		}
 		w := zip.NewWriter(f)
-		comment := fmt.Sprintf("%s %03d", c.Origin, i)
+		comment := fmt.Sprintf("%s archive (%d of %d, generated by photocamera-archiver)",
+			c.Origin, i+1, (c.N+archiveWidth-1)/archiveWidth)
 		if err := w.SetComment(comment); err != nil {
 			fatalError(logger, "failed to set zip comment", "name", name, "err", err)
 		}
+
+		// Store README, checkpoint, and log info. Uncompressed, since they're small.
+		if err := storeMetadataFile(w, "README.txt", []byte(README)); err != nil {
+			fatalError(logger, "failed to write README", "name", name, "err", err)
+		}
+		if err := storeMetadataFile(w, "checkpoint", checkpointBytes); err != nil {
+			fatalError(logger, "failed to write checkpoint", "name", name, "err", err)
+		}
+		if err := storeMetadataFile(w, "log.v3.json", logBytes); err != nil {
+			fatalError(logger, "failed to write log info", "name", name, "err", err)
+		}
+
+		// Store high-level and partial tiles.
+		for _, tile := range globalTiles {
+			if err := storeTile(w, tile, hr); err != nil {
+				fatalError(logger, "failed to store tile", "tile", sunlight.TilePath(tile), "err", err)
+			}
+		}
+
+		// Store sub-tree tiles, starting from higher-level ones.
 		tiles := tlog.NewTiles(torchwood.TileHeight, n, n+subtree)
 		pb := progressbar.Default(int64(len(tiles)), name)
-		// Sort tiles in the zip files so that higher-level tiles come first.
 		slices.SortStableFunc(tiles, func(a, b tlog.Tile) int {
 			switch {
 			case a.L < b.L:
@@ -121,26 +184,13 @@ func main() {
 			}
 		})
 		for _, tile := range tiles {
-			if tile.L >= 3 {
+			if tile.L >= 3 || tile.W < sunlight.TileWidth {
+				// Already stored as part of globalTiles.
 				pb.Add(1)
 				continue
 			}
-			path := sunlight.TilePath(tile)
-			// Pull the hashes through TileHashReader instead of reading them
-			// directly, so that their inclusion in the tree is verified.
-			data, err := tlog.ReadTileData(tile, hr)
-			if err != nil {
-				fatalError(logger, "failed to read tile data", "tile", path, "err", err)
-			}
-			zf, err := w.CreateHeader(&zip.FileHeader{
-				Name:   path,
-				Method: zip.Store, // hashes don't compress!
-			})
-			if err != nil {
-				fatalError(logger, "failed to create zip entry", "tile", path, "err", err)
-			}
-			if _, err := zf.Write(data); err != nil {
-				fatalError(logger, "failed to write zip entry", "tile", path, "err", err)
+			if err := storeTile(w, tile, hr); err != nil {
+				fatalError(logger, "failed to store tile", "tile", sunlight.TilePath(tile), "err", err)
 			}
 			pb.Add(1)
 			if err := ctx.Err(); err != nil {
@@ -149,7 +199,8 @@ func main() {
 		}
 		pb.Reset()
 		pb.ChangeMax64((subtree + 255) / 256)
-		// Store data tiles after the Merkle tree tiles.
+
+		// Verify and store data tiles after the Merkle tree tiles.
 		for _, tile := range tiles {
 			if tile.L != 0 {
 				continue
@@ -188,56 +239,42 @@ func main() {
 		logger.Info("wrote zip file", "name", name)
 	}
 
-	// Delete unnecessary tiles at level 3+, and verify the rest of them.
-	for L := 3; L <= 5; L++ {
-		levelDir := fmt.Sprintf("tile/%d", L)
-		levelMaxSize := c.N >> (sunlight.TileHeight * L)
-		if levelMaxSize == 0 {
-			break
-		}
-		if err := fs.WalkDir(root.FS(), levelDir, func(path string, d fs.DirEntry, err error) error {
-			if err != nil {
-				return err
-			}
-			if d.IsDir() {
-				return nil
-			}
-			t, err := sunlight.ParseTilePath(strings.TrimSuffix(path, ".p"))
-			if err != nil {
-				return fmt.Errorf("failed to parse tile path %s: %w", path, err)
-			}
-			if t.L != L {
-				return fmt.Errorf("unexpected tile level %d, want %d", t.L, L)
-			}
-			size := t.N*sunlight.TileWidth + int64(t.W)
-			if t.W != sunlight.TileWidth && size < levelMaxSize {
-				// Partial tile, can be deleted.
-				logger.Info("removing unnecessary partial tile", "tile", path,
-					"size", size, "max", levelMaxSize)
-				if err := os.Remove(path); err != nil {
-					return fmt.Errorf("failed to remove tile %s: %w", path, err)
-				}
-				return nil
-			}
-			data, err := root.ReadFile(path)
-			if err != nil {
-				return fmt.Errorf("failed to read tile data %s: %w", path, err)
-			}
-			exp, err := tlog.ReadTileData(t, hr)
-			if err != nil {
-				return fmt.Errorf("failed to read tile data %s: %w", path, err)
-			}
-			if !bytes.Equal(data, exp) {
-				return fmt.Errorf("tile data mismatch for %s", path)
-			}
-			logger.Info("verified tile", "tile", path)
-			return nil
-		}); err != nil {
-			fatalError(logger, "failed to walk tile directory", "level", L, "err", err)
-		}
+	logger.Info("done")
+}
+
+func storeMetadataFile(w *zip.Writer, name string, data []byte) error {
+	zf, err := w.CreateHeader(&zip.FileHeader{
+		Name:   name,
+		Method: zip.Store,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create zip entry %q: %w", name, err)
 	}
+	if _, err := zf.Write(data); err != nil {
+		return fmt.Errorf("failed to write zip entry %q: %w", name, err)
+	}
+	return nil
+}
 
-	logger.Info("done")
+func storeTile(w *zip.Writer, tile tlog.Tile, hr tlog.HashReader) error {
+	path := sunlight.TilePath(tile)
+	// Pull the hashes through TileHashReader instead of reading them
+	// directly, so that their inclusion in the tree is verified.
+	data, err := tlog.ReadTileData(tile, hr)
+	if err != nil {
+		return fmt.Errorf("failed to read tile data %q: %w", path, err)
+	}
+	zf, err := w.CreateHeader(&zip.FileHeader{
+		Name:   path,
+		Method: zip.Store, // hashes don't compress!
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create zip entry %q: %w", path, err)
+	}
+	if _, err := zf.Write(data); err != nil {
+		return fmt.Errorf("failed to write zip entry %q: %w", path, err)
+	}
+	return nil
 }
 
 func verifyTileData(tile tlog.Tile, data []byte, hr tlog.HashReader) error {