Fix logs and stop the node when missing delayed message is detected (EspressoSystems#715)

Author: ImJeremyHe

arbnode/espresso_delayed_message_fetcher.go

@@ -180,7 +180,7 @@ func (f *DelayedMessageFetcher) processDelayedMessage(messageWithMetadataAndPos
 	delayedMessageToProcess := delayedMessagesRead - 1
 
 	if delayedMessageToProcess > delayedCount {
-		log.Warn("delayed message fetcher is lagging behind. delayedMessagesRead: %v, delayedCount: %v", delayedMessagesRead, delayedCount)
+		log.Warn("delayed message fetcher is lagging behind", "delayedMessagesRead", delayedMessagesRead, "delayedCount", delayedCount)
 		return nil, fmt.Errorf("delayed message fetcher is lagging behind")
 	}
 	log.Debug("Getting delayed message", "delayedCount", delayedMessageToProcess)
@@ -334,6 +334,12 @@ func (d *DelayedMessageFetcher) getDelayedMessagesInRange(ctx context.Context, b
 			continue
 		}
 
+		if seqNum > lastDelayedMessageIndex+1 {
+			// Delayed message fetcher expects to fetch messages strictly in order. If a missing message is detected here,
+			// it indicates a break in sequential fetching. Recovery is not handled by this fetcher and would require additional logic.
+			log.Crit("Caff node is missing a delayed message", "seqNum", seqNum, "lastDelayedMessageIndex", lastDelayedMessageIndex)
+		}
+
 		lastDelayedMessageIndex++
 		err = d.storeDelayedMessage(batch, lastDelayedMessageIndex, *msg)
 		if err != nil {