fix: accept KZG proof mismatches when blob API has valid proofs (#53)

Post-Fulu, beacon nodes may return invalid/zeroed KZG proofs (0xc0...)
for blob sidecars while the blob archiver computes and stores valid
proofs. This caused false validation errors.

Changes:
- Add verifyKZGProofs() to cryptographically validate KZG proofs
- Accept data mismatches when only KZG fields differ and blob API
  proofs are valid
- Reject data when blob API proofs are invalid or non-KZG fields differ
- Add comprehensive tests for KZG validation scenarios

This fixes validator errors for slots around the Fulu fork while
maintaining data integrity checks.

Author: niran

validator/service/service.go

@@ -12,9 +12,11 @@ import (
 	client "github.com/attestantio/go-eth2-client"
 	"github.com/attestantio/go-eth2-client/api"
 	v1 "github.com/attestantio/go-eth2-client/api/v1"
+	"github.com/attestantio/go-eth2-client/spec/deneb"
 	"github.com/attestantio/go-eth2-client/spec/phase0"
 	"github.com/base/blob-archiver/common/storage"
 	"github.com/base/blob-archiver/validator/flags"
+	gokzg4844 "github.com/crate-crypto/go-kzg-4844"
 	"github.com/ethereum-optimism/optimism/op-service/retry"
 	"github.com/ethereum/go-ethereum/log"
 )
@@ -115,6 +117,29 @@ func shouldRetry(status int) bool {
 	}
 }
 
+// verifyKZGProofs validates that the KZG proofs in the blob sidecars are valid for the blob data.
+// Returns nil if all proofs are valid, error otherwise.
+func verifyKZGProofs(sidecars []*deneb.BlobSidecar) error {
+	kzgCtx, err := gokzg4844.NewContext4096Secure()
+	if err != nil {
+		return fmt.Errorf("failed to create KZG context: %w", err)
+	}
+
+	for i, sidecar := range sidecars {
+		kzgBlob := (*gokzg4844.Blob)(&sidecar.Blob)
+		commitment := gokzg4844.KZGCommitment(sidecar.KZGCommitment)
+		proof := gokzg4844.KZGProof(sidecar.KZGProof)
+
+		// Verify the KZG proof - returns error if invalid
+		err := kzgCtx.VerifyBlobKZGProof(kzgBlob, commitment, proof)
+		if err != nil {
+			return fmt.Errorf("KZG proof verification failed for sidecar %d: %w", i, err)
+		}
+	}
+
+	return nil
+}
+
 // fetchWithRetries fetches the sidecar and handles retryable error cases (5xx status codes + 429 + connection errors)
 func fetchWithRetries(ctx context.Context, endpoint BlobSidecarClient, id string, format Format) (int, storage.BlobSidecars, error) {
 	return retry.Do2(ctx, retryAttempts, retry.Exponential(), func() (int, storage.BlobSidecars, error) {
@@ -171,11 +196,43 @@ func (a *ValidatorService) checkBlobs(ctx context.Context, start phase0.Slot, en
 			}
 
 			if !reflect.DeepEqual(beaconResponse, blobResponse) {
-				result.MismatchedData = append(result.MismatchedData, id)
-				l.Error(validationErrorLog, "reason", "response-mismatch")
+				// Data mismatch detected - first check if sidecar counts match
+				if len(beaconResponse.Data) != len(blobResponse.Data) {
+					// Different number of sidecars - this is a real error
+					result.MismatchedData = append(result.MismatchedData, id)
+					l.Error(validationErrorLog, "reason", "response-mismatch", "beaconCount", len(beaconResponse.Data), "blobApiCount", len(blobResponse.Data))
+					continue
+				}
+
+				// Same number of sidecars - verify if the blob API's KZG proofs are valid
+				// This handles cases where the beacon node returns zeroed out KZG proofs post-Fulu
+				if err := verifyKZGProofs(blobResponse.Data); err != nil {
+					// Blob API has invalid KZG proofs - this is a real error
+					result.MismatchedData = append(result.MismatchedData, id)
+					l.Error(validationErrorLog, "reason", "response-mismatch", "kzgVerification", "failed", "error", err)
+				} else {
+					// Blob API's KZG proofs are valid - now check if only KZG fields differ
+					// Create a copy of blobResponse with beacon's KZG proofs
+					normalizedBlobResponse := storage.BlobSidecars{Data: make([]*deneb.BlobSidecar, len(blobResponse.Data))}
+					for i, sidecar := range blobResponse.Data {
+						normalized := *sidecar // shallow copy
+						// Copy KZG fields from beacon response
+						normalized.KZGProof = beaconResponse.Data[i].KZGProof
+						normalized.KZGCommitment = beaconResponse.Data[i].KZGCommitment
+						normalized.KZGCommitmentInclusionProof = beaconResponse.Data[i].KZGCommitmentInclusionProof
+						normalizedBlobResponse.Data[i] = &normalized
+					}
+
+					// Compare again with normalized KZG values
+					if !reflect.DeepEqual(beaconResponse, normalizedBlobResponse) {
+						// Other fields differ - this is a real error
+						result.MismatchedData = append(result.MismatchedData, id)
+						l.Error(validationErrorLog, "reason", "response-mismatch", "kzgVerification", "passed", "note", "blob data differs beyond KZG fields")
+					}
+				}
+			} else {
+				l.Info("completed blob check", "blobs", len(beaconResponse.Data))
 			}
-
-			l.Info("completed blob check", "blobs", len(beaconResponse.Data))
 		}
 
 		// Check if we should stop validation otherwise continue

validator/service/service_test.go

@@ -158,30 +158,12 @@ func TestValidatorService_MistmatchedBlobFields(t *testing.T) {
 				(*i)[0].Blob = deneb.Blob{0, 0, 0}
 			},
 		},
-		{
-			name: "mismatched kzg commitment",
-			modification: func(i *[]*deneb.BlobSidecar) {
-				(*i)[0].KZGCommitment = deneb.KZGCommitment{0, 0, 0}
-			},
-		},
-		{
-			name: "mismatched kzg proof",
-			modification: func(i *[]*deneb.BlobSidecar) {
-				(*i)[0].KZGProof = deneb.KZGProof{0, 0, 0}
-			},
-		},
 		{
 			name: "mismatched signed block header",
 			modification: func(i *[]*deneb.BlobSidecar) {
 				(*i)[0].SignedBlockHeader = nil
 			},
 		},
-		{
-			name: "mismatched kzg commitment inclusion proof",
-			modification: func(i *[]*deneb.BlobSidecar) {
-				(*i)[0].KZGCommitmentInclusionProof = deneb.KZGCommitmentInclusionProof{{1, 2, 9}}
-			},
-		},
 	}
 
 	for _, test := range tests {
@@ -215,3 +197,100 @@ func TestValidatorService_MistmatchedBlobFields(t *testing.T) {
 		})
 	}
 }
+
+// TestValidatorService_KZGProofMismatchWithValidBlobAPI tests the scenario where:
+// - Beacon node returns invalid/zeroed KZG proofs (e.g., post-Fulu blob)
+// - Blob API has valid computed KZG proofs
+// - All other data matches
+// This should be accepted as valid
+func TestValidatorService_KZGProofMismatchWithValidBlobAPI(t *testing.T) {
+	validator, headers, beacon, blob := setup(t)
+
+	beacon.setResponses(headers)
+	blob.setResponses(headers)
+
+	// Deep copy the beacon data and zero out KZG fields to simulate beacon node bug
+	d, err := json.Marshal(headers.SidecarsByBlock[blockOne])
+	require.NoError(t, err)
+	var beaconData []*deneb.BlobSidecar
+	err = json.Unmarshal(d, &beaconData)
+	require.NoError(t, err)
+
+	// Simulate beacon node bug: set KZG proof to point at infinity (0xc0...)
+	for i := range beaconData {
+		beaconData[i].KZGProof = deneb.KZGProof{0xc0}
+		beaconData[i].KZGCommitmentInclusionProof = deneb.KZGCommitmentInclusionProof{}
+	}
+
+	beacon.setResponse(blockOne, 200, storage.BlobSidecars{Data: beaconData}, nil)
+
+	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
+
+	// Should have no errors - KZG mismatch accepted because blob API has valid proofs
+	require.Empty(t, result.MismatchedStatus)
+	require.Empty(t, result.ErrorFetching)
+	require.Empty(t, result.MismatchedData)
+}
+
+// TestValidatorService_KZGProofMismatchWithInvalidBlobAPI tests the scenario where:
+// - Both beacon node and blob API have different KZG proofs
+// - Blob API's KZG proofs are INVALID
+// This should be reported as an error
+func TestValidatorService_KZGProofMismatchWithInvalidBlobAPI(t *testing.T) {
+	validator, headers, beacon, blob := setup(t)
+
+	beacon.setResponses(headers)
+	blob.setResponses(headers)
+
+	// Deep copy the blob data and set invalid KZG proof
+	d, err := json.Marshal(headers.SidecarsByBlock[blockOne])
+	require.NoError(t, err)
+	var blobData []*deneb.BlobSidecar
+	err = json.Unmarshal(d, &blobData)
+	require.NoError(t, err)
+
+	// Set invalid KZG proof on blob API
+	for i := range blobData {
+		blobData[i].KZGProof = deneb.KZGProof{0xde, 0xad, 0xbe, 0xef}
+	}
+
+	blob.setResponse(blockOne, 200, storage.BlobSidecars{Data: blobData}, nil)
+
+	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
+
+	// Should report mismatch because blob API has invalid KZG proofs
+	require.Empty(t, result.MismatchedStatus)
+	require.Empty(t, result.ErrorFetching)
+	require.Len(t, result.MismatchedData, 2)
+	require.Equal(t, result.MismatchedData, []string{blockOne, blockOne})
+}
+
+// TestValidatorService_DifferentSidecarCount tests the scenario where:
+// - Beacon and blob API return different numbers of sidecars
+// This should be reported as an error
+func TestValidatorService_DifferentSidecarCount(t *testing.T) {
+	validator, headers, beacon, blob := setup(t)
+
+	beacon.setResponses(headers)
+	blob.setResponses(headers)
+
+	// Set blob API to return fewer sidecars
+	d, err := json.Marshal(headers.SidecarsByBlock[blockOne])
+	require.NoError(t, err)
+	var blobData []*deneb.BlobSidecar
+	err = json.Unmarshal(d, &blobData)
+	require.NoError(t, err)
+
+	// Remove one sidecar
+	blobData = blobData[:len(blobData)-1]
+
+	blob.setResponse(blockOne, 200, storage.BlobSidecars{Data: blobData}, nil)
+
+	result := validator.checkBlobs(context.Background(), phase0.Slot(blobtest.StartSlot), phase0.Slot(blobtest.EndSlot))
+
+	// Should report mismatch due to different sidecar counts
+	require.Empty(t, result.MismatchedStatus)
+	require.Empty(t, result.ErrorFetching)
+	require.Len(t, result.MismatchedData, 2)
+	require.Equal(t, result.MismatchedData, []string{blockOne, blockOne})
+}