feat(goldfish): Add support for query results in object storage

pkg/goldfish/migrations/20251007000001_add_result_storage_columns.sql

@@ -0,0 +1,21 @@
+-- +goose Up
+-- Add columns to persist result storage metadata for Goldfish
+
+ALTER TABLE sampled_queries
+  ADD COLUMN cell_a_result_uri TEXT NULL AFTER cell_a_status_code,
+  ADD COLUMN cell_b_result_uri TEXT NULL AFTER cell_a_result_uri,
+  ADD COLUMN cell_a_result_size_bytes BIGINT NULL AFTER cell_b_result_uri,
+  ADD COLUMN cell_b_result_size_bytes BIGINT NULL AFTER cell_a_result_size_bytes,
+  ADD COLUMN cell_a_result_compression VARCHAR(32) NULL AFTER cell_b_result_size_bytes,
+  ADD COLUMN cell_b_result_compression VARCHAR(32) NULL AFTER cell_a_result_compression;
+
+-- +goose Down
+-- Remove result storage metadata columns
+
+ALTER TABLE sampled_queries
+  DROP COLUMN cell_b_result_compression,
+  DROP COLUMN cell_a_result_compression,
+  DROP COLUMN cell_b_result_size_bytes,
+  DROP COLUMN cell_a_result_size_bytes,
+  DROP COLUMN cell_b_result_uri,
+  DROP COLUMN cell_a_result_uri;

pkg/goldfish/storage_mysql.go

@@ -92,11 +92,14 @@ func (s *MySQLStorage) StoreQuerySample(ctx context.Context, sample *QuerySample
 			cell_a_response_hash, cell_b_response_hash,
 			cell_a_response_size, cell_b_response_size,
 			cell_a_status_code, cell_b_status_code,
+			cell_a_result_uri, cell_b_result_uri,
+			cell_a_result_size_bytes, cell_b_result_size_bytes,
+			cell_a_result_compression, cell_b_result_compression,
 			cell_a_trace_id, cell_b_trace_id,
 			cell_a_span_id, cell_b_span_id,
 			cell_a_used_new_engine, cell_b_used_new_engine,
 			sampled_at
-		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
 	`
 
 	// Convert empty span IDs to NULL for database storage
@@ -108,6 +111,25 @@ func (s *MySQLStorage) StoreQuerySample(ctx context.Context, sample *QuerySample
 		cellBSpanID = sample.CellBSpanID
 	}
 
+	// Prepare nullable result storage metadata
+	var cellAResultURI, cellBResultURI any
+	var cellAResultSize, cellBResultSize any
+	var cellAResultCompression, cellBResultCompression any
+	if sample.CellAResultURI != "" {
+		cellAResultURI = sample.CellAResultURI
+		cellAResultSize = sample.CellAResultSize
+		if sample.CellAResultCompression != "" {
+			cellAResultCompression = sample.CellAResultCompression
+		}
+	}
+	if sample.CellBResultURI != "" {
+		cellBResultURI = sample.CellBResultURI
+		cellBResultSize = sample.CellBResultSize
+		if sample.CellBResultCompression != "" {
+			cellBResultCompression = sample.CellBResultCompression
+		}
+	}
+
 	_, err := s.db.ExecContext(ctx, query,
 		sample.CorrelationID,
 		sample.TenantID,
@@ -142,6 +164,12 @@ func (s *MySQLStorage) StoreQuerySample(ctx context.Context, sample *QuerySample
 		sample.CellBResponseSize,
 		sample.CellAStatusCode,
 		sample.CellBStatusCode,
+		cellAResultURI,
+		cellBResultURI,
+		cellAResultSize,
+		cellBResultSize,
+		cellAResultCompression,
+		cellBResultCompression,
 		sample.CellATraceID,
 		sample.CellBTraceID,
 		cellASpanID,
@@ -214,6 +242,9 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 			cell_a_entries_returned, cell_b_entries_returned, cell_a_splits, cell_b_splits,
 			cell_a_shards, cell_b_shards, cell_a_response_hash, cell_b_response_hash,
 			cell_a_response_size, cell_b_response_size, cell_a_status_code, cell_b_status_code,
+			cell_a_result_uri, cell_b_result_uri,
+			cell_a_result_size_bytes, cell_b_result_size_bytes,
+			cell_a_result_compression, cell_b_result_compression,
 			cell_a_trace_id, cell_b_trace_id,
 			cell_a_span_id, cell_b_span_id,
 			cell_a_used_new_engine, cell_b_used_new_engine,
@@ -245,6 +276,9 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 		var createdAt time.Time
 		// Use sql.NullString for nullable span ID columns
 		var cellASpanID, cellBSpanID sql.NullString
+		var cellAResultURI, cellBResultURI sql.NullString
+		var cellAResultCompression, cellBResultCompression sql.NullString
+		var cellAResultSize, cellBResultSize sql.NullInt64
 
 		err := rows.Scan(
 			&q.CorrelationID, &q.TenantID, &q.User, &q.Query, &q.QueryType, &q.StartTime, &q.EndTime, &stepDurationMs,
@@ -254,6 +288,9 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 			&q.CellAStats.TotalEntriesReturned, &q.CellBStats.TotalEntriesReturned, &q.CellAStats.Splits, &q.CellBStats.Splits,
 			&q.CellAStats.Shards, &q.CellBStats.Shards, &q.CellAResponseHash, &q.CellBResponseHash,
 			&q.CellAResponseSize, &q.CellBResponseSize, &q.CellAStatusCode, &q.CellBStatusCode,
+			&cellAResultURI, &cellBResultURI,
+			&cellAResultSize, &cellBResultSize,
+			&cellAResultCompression, &cellBResultCompression,
 			&q.CellATraceID, &q.CellBTraceID,
 			&cellASpanID, &cellBSpanID,
 			&q.CellAUsedNewEngine, &q.CellBUsedNewEngine,
@@ -270,6 +307,24 @@ func (s *MySQLStorage) GetSampledQueries(ctx context.Context, page, pageSize int
 		if cellBSpanID.Valid {
 			q.CellBSpanID = cellBSpanID.String
 		}
+		if cellAResultURI.Valid {
+			q.CellAResultURI = cellAResultURI.String
+		}
+		if cellBResultURI.Valid {
+			q.CellBResultURI = cellBResultURI.String
+		}
+		if cellAResultSize.Valid {
+			q.CellAResultSize = cellAResultSize.Int64
+		}
+		if cellBResultSize.Valid {
+			q.CellBResultSize = cellBResultSize.Int64
+		}
+		if cellAResultCompression.Valid {
+			q.CellAResultCompression = cellAResultCompression.String
+		}
+		if cellBResultCompression.Valid {
+			q.CellBResultCompression = cellBResultCompression.String
+		}
 
 		// Convert step duration from milliseconds to Duration
 		q.Step = time.Duration(stepDurationMs) * time.Millisecond

pkg/goldfish/types.go

@@ -32,6 +32,14 @@ type QuerySample struct {
 	CellASpanID       string `json:"cellASpanID"`
 	CellBSpanID       string `json:"cellBSpanID"`
 
+	// Result storage metadata
+	CellAResultURI         string `json:"cellAResultURI"`
+	CellBResultURI         string `json:"cellBResultURI"`
+	CellAResultSize        int64  `json:"cellAResultSize"`
+	CellBResultSize        int64  `json:"cellBResultSize"`
+	CellAResultCompression string `json:"cellAResultCompression"`
+	CellBResultCompression string `json:"cellBResultCompression"`
+
 	// Query engine version tracking
 	CellAUsedNewEngine bool `json:"cellAUsedNewEngine"`
 	CellBUsedNewEngine bool `json:"cellBUsedNewEngine"`

pkg/ui/goldfish.go

@@ -76,6 +76,14 @@ type SampledQuery struct {
 	CellAStatusCode   *int    `json:"cellAStatusCode" db:"cell_a_status_code"`
 	CellBStatusCode   *int    `json:"cellBStatusCode" db:"cell_b_status_code"`
 
+	// Result storage metadata - nullable when persistence is disabled
+	CellAResultURI         *string `json:"cellAResultURI,omitempty" db:"cell_a_result_uri"`
+	CellBResultURI         *string `json:"cellBResultURI,omitempty" db:"cell_b_result_uri"`
+	CellAResultSizeBytes   *int64  `json:"cellAResultSizeBytes,omitempty" db:"cell_a_result_size_bytes"`
+	CellBResultSizeBytes   *int64  `json:"cellBResultSizeBytes,omitempty" db:"cell_b_result_size_bytes"`
+	CellAResultCompression *string `json:"cellAResultCompression,omitempty" db:"cell_a_result_compression"`
+	CellBResultCompression *string `json:"cellBResultCompression,omitempty" db:"cell_b_result_compression"`
+
 	// Trace IDs - nullable as not all requests have traces
 	CellATraceID *string `json:"cellATraceID" db:"cell_a_trace_id"`
 	CellBTraceID *string `json:"cellBTraceID" db:"cell_b_trace_id"`
@@ -242,6 +250,25 @@ func (s *Service) GetSampledQueriesWithContext(ctx context.Context, page, pageSi
 			CellBUsedNewEngine: q.CellBUsedNewEngine,
 		}
 
+		if q.CellAResultURI != "" {
+			uiQuery.CellAResultURI = strPtr(q.CellAResultURI)
+			size := q.CellAResultSize
+			uiQuery.CellAResultSizeBytes = &size
+			if q.CellAResultCompression != "" {
+				comp := q.CellAResultCompression
+				uiQuery.CellAResultCompression = &comp
+			}
+		}
+		if q.CellBResultURI != "" {
+			uiQuery.CellBResultURI = strPtr(q.CellBResultURI)
+			size := q.CellBResultSize
+			uiQuery.CellBResultSizeBytes = &size
+			if q.CellBResultCompression != "" {
+				comp := q.CellBResultCompression
+				uiQuery.CellBResultCompression = &comp
+			}
+		}
+
 		// Determine comparison status based on response codes and hashes
 		if q.CellAStatusCode < 200 || q.CellAStatusCode >= 300 || q.CellBStatusCode < 200 || q.CellBStatusCode >= 300 {
 			uiQuery.ComparisonStatus = string(goldfish.ComparisonStatusError)

tools/querytee/goldfish/README.md

@@ -16,7 +16,8 @@ Goldfish is a feature within QueryTee that enables sampling and comparison of qu
   - Query complexity metrics (splits, shards)
   - Performance variance detection and reporting
 - **Query Engine Version Tracking**: Tracks which queries used the new experimental query engine vs the old engine
-- **Persistent Storage**: MySQL storage via Google Cloud SQL Proxy or Amazon RDS for storing query samples and comparison results
+- **Persistent Metadata Storage**: MySQL storage via Google Cloud SQL Proxy or Amazon RDS for storing query samples and comparison results
+- **Raw Result Persistence**: Optional upload of exact JSON responses to GCS or S3 for deep diffing
 
 ## Configuration
 
@@ -55,6 +56,22 @@ export GOLDFISH_DB_PASSWORD=your-password
 # Performance comparison settings
 -goldfish.performance-tolerance=0.1                             # 10% tolerance for execution time variance
 
+# Result persistence (optional)
+-goldfish.results.enabled=true                                  # store raw responses
+-goldfish.results.mode=mismatch-only                            # or 'all'
+-goldfish.results.backend=gcs                                   # inferred for CloudSQL
+-goldfish.results.bucket.gcs.bucket-name=<bucket>               # required for GCS
+-goldfish.results.bucket.gcs.service-account=<json>             # optional service account JSON
+-goldfish.results.prefix=goldfish/results                       # optional prefix inside the bucket
+-goldfish.results.compression=gzip                              # gzip or none
+
+# S3 example (RDS deployments)
+-goldfish.results.backend=s3
+-goldfish.results.bucket.s3.bucket-name=<bucket>
+-goldfish.results.bucket.s3.region=<region>
+-goldfish.results.bucket.s3.access-key-id=<key>                 # optional when using IAM roles
+-goldfish.results.bucket.s3.secret-access-key=<secret>          # optional when using IAM roles
+
 # Or run without storage (sampling and comparison only, no persistence)
 # Simply omit the storage configuration
 ```
@@ -79,6 +96,17 @@ export GOLDFISH_DB_PASSWORD=your-password
         └──────────┘ └──────────┘ └──────────┘
 ```
 
+## Result Storage Layout
+
+When result persistence is enabled, Goldfish writes two objects per sampled query under the configured prefix using UTC date partitions and the correlation ID:
+
+```
+<bucket>/<prefix>/<YYYY>/<MM>/<DD>/<correlation-id>/cell-a.json.gz
+<bucket>/<prefix>/<YYYY>/<MM>/<DD>/<correlation-id>/cell-b.json.gz
+```
+
+Metadata includes the tenant, backend name, HTTP status, and original fnv32 hash so you can correlate objects with database entries. Payloads are gzip-compressed by default. Apply lifecycle policies and IAM so raw log data is protected.
+
 ## Database Schema
 
 Goldfish uses two main tables: