feat: Populate state proofs with real Merkle paths

hapi/hapi/src/main/proto/network/pending_proof.proto

@@ -21,6 +21,8 @@ option java_package = "com.hedera.node.internal.network.legacy";
 // <<<pbj.java_package = "com.hedera.node.internal.network">>> This comment is special code for setting PBJ Compiler java package
 option java_multiple_files = true;
 
+import "services/timestamp.proto";
+
 /**
  * Provides context for a block proof pending a TSS signature.
  */
@@ -41,6 +43,10 @@ message PendingProof {
      * The hash of the previous block.
      */
     bytes previous_block_hash = 4;
+    /**
+     * The timestamp of the pending block
+     */
+    proto.Timestamp block_timestamp = 5;
     /**
      * If set, the sibling hashes that could be used to prove the
      * previous block hash (in case it was also pending, and we
@@ -50,5 +56,5 @@ message PendingProof {
      * already been signed and this block proof will not be used
      * for any indirect proofs.
      */
-    repeated com.hedera.hapi.block.stream.MerkleSiblingHash sibling_hashes_from_prev_block_root = 5;
+    repeated com.hedera.hapi.block.stream.MerkleSiblingHash sibling_hashes_from_prev_block_root = 6;
 }

hapi/hedera-protobuf-java-api/src/main/proto/block/stream/block_proof.proto

@@ -109,46 +109,17 @@ message BlockProof {
      */
     uint64 block = 1;
 
-    /**
-     * A set of hash values along with ordering information.<br/>
-     * This list of hash values form the set of sibling hash values needed to
-     * correctly reconstruct the parent hash, and all hash values "above" that
-     * hash in the merkle tree.
-     * <p>
-     * A Block proof can be constructed by combining the sibling hashes for
-     * a previous block hash and sibling hashes for each entry "above" that
-     * node in the merkle tree of a block proof that incorporates that previous
-     * block hash. This form of block proof may be used to prove a chain of
-     * blocks when one or more older blocks is missing the original block
-     * proof that signed the block's merkle root directly.
-     * <p>
-     * This list MUST be ordered from the sibling of the node that contains
-     * this block's root node hash, and continues up the merkle tree to the
-     * root hash of the signed block proof.
-     * <p>
-     * If this block proof has a "direct" signature, then this list MUST be
-     * empty.<br/>
-     * If this list is not empty, then this block proof MUST be verified by
-     * first constructing the "block" merkle tree and computing the root hash
-     * of that tree, then combining that hash with the values in this list,
-     * paying attention to the first/second sibling ordering, until the root
-     * merkle hash is produced from the last pair of sibling hashes. That
-     * "secondary" root hash MUST then be verified using the value of
-     * `block_signature`.
-     */
-    repeated MerkleSiblingHash sibling_hashes = 2;
-
     /**
      * The hinTS key that this signature verifies under; a stream consumer should
      * only use this key after first checking the chain of trust proof.
      */
-    bytes verification_key = 3;
+    bytes verification_key = 2;
 
     /**
      * Proof the hinTS verification key is in the chain of trust extending
      * from the network's ledger id.
      */
-    ChainOfTrustProof verification_key_proof = 4;
+    ChainOfTrustProof verification_key_proof = 3;
 
     /**
      * The proof contents verifying the block's merkle root hash.<br/>
@@ -164,15 +135,42 @@ message BlockProof {
          * used when the current block is signed directly by the consensus
          * nodes with a TSS signature; otherwise it MUST be empty.
          */
-        TssSignedBlockProof signed_block_proof = 5;
+        TssSignedBlockProof signed_block_proof = 4;
 
         /**
          * A proof of the block merkle tree's contents. This proof SHALL
          * contain the information necessary to validate the previous block's
          * hash, along with any information necessary to validate the current
          * block's hash.
+         * <p>
+         * Of necessity, the state proof MUST contain a set of hash values,
+         * along with ordering information, that allows for reconstruction of the
+         * block's hash. This list of hash values form the set of sibling hash values
+         * needed to correctly reconstruct the parent hash, and all hash values
+         * "above" that hash in the merkle tree.
+         * <p>
+         * A Block proof can be constructed by combining the sibling hashes for
+         * a previous block hash and sibling hashes for each entry "above" that
+         * node in the merkle tree of a block proof that incorporates that previous
+         * block hash. This form of block proof may be used to prove a chain of
+         * blocks when one or more older blocks is missing the original block
+         * proof that signed the block's merkle root directly.
+         * <p>
+         * Such a list MUST be ordered from the sibling of the node that contains
+         * this block's root node hash, and continues up the merkle tree to the
+         * root hash of the signed block proof.
+         * <p>
+         * If this block proof has a "direct" signature, then any associated list
+         * of siblings MUST be empty.<br/>
+         * If said list is not empty, then this block proof MUST be verified by
+         * first constructing the "block" merkle tree and computing the root hash
+         * of that tree, then combining that hash with the values in this list,
+         * paying attention to the first/second sibling ordering, until the root
+         * merkle hash is produced from the last pair of sibling hashes. That
+         * "secondary" root hash MUST then be verified using the value of
+         * `block_signature`.
          */
-        StateProof block_state_proof = 6;
+        StateProof block_state_proof = 5;
 
         /**
          * A proof consisting of RSA signatures from consensus nodes.<br/>
@@ -182,7 +180,7 @@ message BlockProof {
          * individual RSA signatures from consensus nodes; otherwise it MUST be
          * empty.
          */
-        SignedRecordFileProof signed_record_file_proof = 7;
+        SignedRecordFileProof signed_record_file_proof = 6;
     }
 }
 

hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/BlockStreamManager.java

@@ -29,6 +29,7 @@
  */
 public interface BlockStreamManager extends BlockRecordInfo, StateHashedListener {
     Bytes ZERO_BLOCK_HASH = Bytes.wrap(new byte[48]);
+    int NUM_SIBLINGS_PER_BLOCK = 4;
 
     /**
      * The types of work that may be identified as pending within a block.

hedera-node/hedera-app/src/main/java/com/hedera/node/app/blocks/impl/BlockStateProofGenerator.java

@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: Apache-2.0
+package com.hedera.node.app.blocks.impl;
+
+import com.hedera.hapi.block.stream.MerklePath;
+import com.hedera.hapi.block.stream.SiblingNode;
+import com.hedera.hapi.block.stream.StateProof;
+import com.hedera.hapi.block.stream.TssSignedBlockProof;
+import com.hedera.hapi.node.base.Timestamp;
+import com.hedera.hapi.node.state.blockstream.MerkleLeaf;
+import com.hedera.node.app.hapi.utils.CommonUtils;
+import com.hedera.pbj.runtime.io.buffer.Bytes;
+import com.swirlds.state.SiblingHash;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.stream.Streams;
+import org.hiero.base.crypto.Hash;
+
+/**
+ * Generator for state proofs used in indirect block proofs.
+ * This class encapsulates the logic for constructing merkle paths needed to prove
+ * blocks that precede the latest signed block.
+ */
+public class BlockStateProofGenerator {
+
+    /**
+     * The unsigned block sibling count includes the pending/unsigned block's timestamp
+     */
+    public static final int UNSIGNED_BLOCK_SIBLING_COUNT = 5;
+    /**
+     * The signed block sibling count doesn't include the signed block's timestamp
+     */
+    public static final int SIGNED_BLOCK_SIBLING_COUNT = 4;
+
+    /**
+     * Each block's state proof consists of exactly three Merkle paths: the timestamp of the signed block,
+     * previous block's hash + sibling hashes forming the path to the right sibling of the timestamp of the
+     * signed block, and a trivial final parent path for the signed block's root
+     */
+    public static final int EXPECTED_MERKLE_PATH_COUNT = 3;
+
+    /**
+     * Index to the Merkle path containing hashes from the previous block's root to the right sibling of the
+     * block's timestamp
+     */
+    public static final int BLOCK_CONTENTS_PATH_INDEX = 1;
+
+    /**
+     * Index to the final Merkle path representing the root hash of the signed block
+     */
+    public static final int FINAL_MERKLE_PATH_INDEX = 2;
+
+    /**
+     * Index indicating the end of the merkle path chain
+     */
+    public static final int FINAL_NEXT_PATH_INDEX = -1;
+
+    /**
+     * Constructs a state proof for a block that precedes the latest signed block. This involves creating merkle
+     * paths for <b>all</b> pending blocks immediately preceding the latest signed block, and so must read from the
+     * current pending blocks in memory.
+     *
+     * @param currentPendingBlock the pending block to generate a state proof for
+     * @param latestSignedBlockNumber the block number of the latest signed block
+     * @param latestSignedBlockSignature the signature of the latest signed block
+     * @param remainingPendingBlocks stream of remaining pending blocks after the current one. This queue is
+     *                               passed for <b>read-only</b> purposes; don't dequeue from it.
+     * @return the constructed state proof
+     */
+    public static StateProof generateStateProof(
+            @NonNull final PendingBlock currentPendingBlock,
+            final long latestSignedBlockNumber,
+            final Bytes latestSignedBlockSignature,
+            final Timestamp latestSignedBlockTimestamp,
+            @NonNull final Stream<PendingBlock> remainingPendingBlocks) {
+
+        // Construct the necessary merkle paths for all blocks from [current, blockNumber - 1]. This makes it necessary
+        // to read each pending block, but not dequeue them. The current pending block was already polled from the
+        // pending blocks queue, so combine it in a stream with all the other pending blocks still in the queue.
+        final Map<Long, PendingBlock> allPendingBlocks = Streams.of(
+                        Stream.of(currentPendingBlock), remainingPendingBlocks)
+                .flatMap(s -> s)
+                .collect(Collectors.toMap(PendingBlock::number, Function.identity()));
+
+        final Map<Long, PendingBlock> indirectProofBlocks = allPendingBlocks.entrySet().stream()
+                .filter(e -> e.getKey() < latestSignedBlockNumber)
+                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        // Construct all merkle paths for each pending block between [currentPendingBlock.number(),
+        // latestSignedBlockNumber - 1]
+
+        // Merkle Path 1: construct the block timestamp path
+        final var tsBytes = Timestamp.PROTOBUF.toBytes(latestSignedBlockTimestamp);
+        final var tsLeaf =
+                MerkleLeaf.newBuilder().blockConsensusTimestamp(tsBytes).build();
+        final var mp1 = MerklePath.newBuilder().leaf(tsLeaf).nextPathIndex(FINAL_MERKLE_PATH_INDEX);
+
+        // Merkle Path 2: enumerate all sibling hashes for all remaining blocks
+        MerklePath.Builder mp2 = MerklePath.newBuilder()
+                .hash(currentPendingBlock.previousBlockHash())
+                .nextPathIndex(FINAL_MERKLE_PATH_INDEX);
+
+        // Create a set of siblings for each indirect block, plus another set for the signed block
+        final var totalSiblings =
+                (indirectProofBlocks.size() * UNSIGNED_BLOCK_SIBLING_COUNT) + SIGNED_BLOCK_SIBLING_COUNT;
+        final SiblingNode[] allSiblingHashes = new SiblingNode[totalSiblings];
+        final long minBlockNum = currentPendingBlock.number();
+        var currentBlockNum = minBlockNum;
+        for (int i = 0; i < indirectProofBlocks.size(); i++) {
+            // Convert first four sibling hashes
+            final var blockSiblings = Arrays.stream(
+                            indirectProofBlocks.get(currentBlockNum).siblingHashes())
+                    .map(s -> new SiblingHash(!s.isFirst(), new Hash(s.siblingHash())))
+                    .toList();
+            // Copy into the sibling hashes array
+            final var firstSiblingIndex = i * UNSIGNED_BLOCK_SIBLING_COUNT;
+            for (int j = 0; j < blockSiblings.size(); j++) {
+                final var blockSibling = blockSiblings.get(j);
+                allSiblingHashes[firstSiblingIndex + j] = SiblingNode.newBuilder()
+                        .isLeft(!blockSibling.isRight())
+                        .hash(blockSibling.hash().getBytes())
+                        .build();
+                ;
+            }
+
+            // Convert this pending block's timestamp into a sibling hash
+            final var pbTsBytes = CommonUtils.noThrowSha384HashOf(Timestamp.PROTOBUF.toBytes(
+                    indirectProofBlocks.get(currentBlockNum).blockTimestamp()));
+            // Add to the sibling hashes array
+            final var pendingBlockTimestampSiblingIndex = firstSiblingIndex + UNSIGNED_BLOCK_SIBLING_COUNT - 1;
+            // Timestamp is always a left sibling
+            allSiblingHashes[pendingBlockTimestampSiblingIndex] =
+                    SiblingNode.newBuilder().isLeft(true).hash(pbTsBytes).build();
+
+            currentBlockNum++;
+        }
+
+        // Merkle Path 2 Continued: add sibling hashes for the signed block
+        // Note: the timestamp for this (signed) block was provided in Merkle Path 1 above
+        final var signedBlock = allPendingBlocks.get(latestSignedBlockNumber);
+        final var signedBlockSiblings = signedBlock.siblingHashes();
+        final var signedBlockFirstSiblingIndex = indirectProofBlocks.size() * UNSIGNED_BLOCK_SIBLING_COUNT;
+        for (int i = 0; i < signedBlockSiblings.length; i++) {
+            final var blockSibling = signedBlockSiblings[i];
+            allSiblingHashes[signedBlockFirstSiblingIndex + i] = SiblingNode.newBuilder()
+                    .isLeft(blockSibling.isFirst())
+                    .hash(blockSibling.siblingHash())
+                    .build();
+        }
+        mp2.siblings(Arrays.stream(allSiblingHashes).toList());
+
+        // Merkle Path 3: the parent/block root path
+        final var mp3 = MerklePath.newBuilder().nextPathIndex(FINAL_NEXT_PATH_INDEX);
+
+        return StateProof.newBuilder()
+                .paths(mp1.build(), mp2.build(), mp3.build())
+                .signedBlockProof(TssSignedBlockProof.newBuilder().blockSignature(latestSignedBlockSignature))
+                .build();
+    }
+}