add another scenario for offchain vote data

Author: Cmdv

cardano-db-sync/app/http-get-json-metadata.hs

@@ -143,11 +143,12 @@ runGetVote :: Text.Text -> Maybe VoteMetaHash -> DB.AnchorType -> IO ()
 runGetVote file mExpectedHash vtype = do
   respBs <- BS.readFile (Text.unpack file)
   let respLBs = fromStrict respBs
-  (mocvd, val, hsh, mWarning) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash vtype Nothing
+  (mocvd, val, hsh, mWarning, isValidJson) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash vtype Nothing
   print mocvd
   print val
   print $ bsBase16Encode hsh
   print mWarning
+  putStrLn $ "Is valid JSON: " ++ show isValidJson
 
 -- ------------------------------------------------------------------------------------------------
 

cardano-db-sync/src/Cardano/DbSync/OffChain.hs

@@ -349,8 +349,9 @@ fetchOffChainVoteData gateways time oVoteWorkQ =
     convert eres =
       case eres of
         Right sVoteData ->
-          case sovaOffChainVoteData sVoteData of
-            Just offChainData ->
+          case (sovaIsValidJson sVoteData, sovaOffChainVoteData sVoteData) of
+            -- Scenario 1: Valid JSON + Valid CIP schema
+            (True, Just offChainData) ->
               let
                 minimalBody = Vote.getMinimalBody offChainData
                 vdt =
@@ -371,7 +372,8 @@ fetchOffChainVoteData gateways time oVoteWorkQ =
                 externalUpdatesF ocvdId = map (mkexternalUpdates ocvdId) $ mListToList $ Vote.externalUpdates minimalBody
                in
                 OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
-            Nothing ->
+            -- Scenario 2: Valid JSON but invalid CIP schema
+            (True, Nothing) ->
               let
                 vdt =
                   DB.OffChainVoteData
@@ -391,6 +393,27 @@ fetchOffChainVoteData gateways time oVoteWorkQ =
                 externalUpdatesF _ = []
                in
                 OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
+            -- Scenario 3: Invalid JSON (hash matches but content is not parseable as JSON)
+            (False, _) ->
+              let
+                vdt =
+                  DB.OffChainVoteData
+                    { DB.offChainVoteDataLanguage = ""
+                    , DB.offChainVoteDataComment = Nothing
+                    , DB.offChainVoteDataBytes = sovaBytes sVoteData
+                    , DB.offChainVoteDataHash = sovaHash sVoteData
+                    , DB.offChainVoteDataJson = sovaJson sVoteData -- This will be the error message JSON
+                    , DB.offChainVoteDataVotingAnchorId = oVoteWqReferenceId oVoteWorkQ
+                    , DB.offChainVoteDataWarning = sovaWarning sVoteData
+                    , DB.offChainVoteDataIsValid = Nothing -- NULL for unparseable JSON
+                    }
+                gaF _ = Nothing
+                drepF _ = Nothing
+                authorsF _ = []
+                referencesF _ = []
+                externalUpdatesF _ = []
+               in
+                OffChainVoteResultMetadata vdt (OffChainVoteAccessors gaF drepF authorsF referencesF externalUpdatesF)
         Left err ->
           OffChainVoteResultError $
             DB.OffChainVoteFetchError

cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs

@@ -108,7 +108,7 @@ httpGetOffChainVoteDataSingle vurl metaHash anchorType = do
   let req = httpGetBytes manager request 3000000 3000000 url
   httpRes <- handleExceptT (convertHttpException url) req
   (respBS, respLBS, mContentType) <- hoistEither httpRes
-  (mocvd, decodedValue, metadataHash, mWarning) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl)
+  (mocvd, decodedValue, metadataHash, mWarning, isValidJson) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl)
   pure $
     SimplifiedOffChainVoteData
       { sovaHash = metadataHash
@@ -117,11 +117,12 @@ httpGetOffChainVoteDataSingle vurl metaHash anchorType = do
       , sovaContentType = mContentType
       , sovaOffChainVoteData = mocvd
       , sovaWarning = mWarning
+      , sovaIsValidJson = isValidJson
       }
   where
     url = OffChainVoteUrl vurl
 
-parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Maybe Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text)
+parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Maybe Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text, Bool)
 parseAndValidateVoteData bs lbs metaHash anchorType murl = do
   let metadataHash = Crypto.digest (Proxy :: Proxy Crypto.Blake2b_256) bs
   -- First check if hash matches - this is critical and must fail if mismatch
@@ -130,17 +131,23 @@ parseAndValidateVoteData bs lbs metaHash anchorType murl = do
       | metadataHash /= expectedMetaHashBs ->
           left $ OCFErrHashMismatch murl (renderByteArray expectedMetaHashBs) (renderByteArray metadataHash)
     _ -> pure (metadataHash, Nothing)
-  -- Hash matches, now decode as generic JSON (this should still fail if not valid JSON)
-  decodedValue <-
+  -- Hash matches, now try to decode as generic JSON
+  -- If this fails, we still want to store the data with is_valid = NULL and an error message
+  (decodedValue, isValidJson) <-
     case Aeson.eitherDecode' @Aeson.Value lbs of
-      Left err -> left $ OCFErrJsonDecodeFail murl (Text.pack err)
-      Right res -> pure res
-  -- Try to decode into strongly-typed vote data structure
+      Left err ->
+        -- Not valid JSON - create an error message object
+        pure (Aeson.object [("error", Aeson.String "Content is not valid JSON. See bytes column for raw data."), ("parse_error", Aeson.String $ Text.pack err)], False)
+      Right res -> pure (res, True)
+  -- Try to decode into strongly-typed vote data structure (only if JSON was valid)
   -- If this fails (e.g., doNotList is string instead of bool), we still store with is_valid = false
-  let ocvd = case Vote.eitherDecodeOffChainVoteData lbs anchorType of
-        Left _err -> Nothing  -- Don't fail, just return Nothing (will set is_valid = false)
-        Right res -> Just res
-  pure (ocvd, decodedValue, hsh, mWarning)
+  let ocvd =
+        if isValidJson
+          then case Vote.eitherDecodeOffChainVoteData lbs anchorType of
+            Left _err -> Nothing -- Don't fail, just return Nothing (will set is_valid = false)
+            Right res -> Just res
+          else Nothing -- Not valid JSON, so can't parse as CIP
+  pure (ocvd, decodedValue, hsh, mWarning, isValidJson)
 
 httpGetBytes ::
   Http.Manager ->

cardano-db-sync/src/Cardano/DbSync/Types.hs

@@ -195,6 +195,7 @@ data SimplifiedOffChainVoteData = SimplifiedOffChainVoteData
   , sovaContentType :: !(Maybe ByteString)
   , sovaOffChainVoteData :: !(Maybe Vote.OffChainVoteData)
   , sovaWarning :: !(Maybe Text)
+  , sovaIsValidJson :: !Bool
   }
 
 data Retry = Retry

cardano-db-sync/test/Cardano/DbSync/OffChain/VoteTest.hs

@@ -8,8 +8,11 @@ import Cardano.DbSync.Error (runOrThrowIO)
 import Cardano.DbSync.OffChain.Http (parseAndValidateVoteData)
 import Cardano.Prelude hiding ((%))
 import qualified Data.Aeson as Aeson
+import qualified Data.Aeson.Key as AesonKey
+import qualified Data.Aeson.KeyMap as KeyMap
 import qualified Data.ByteString as BS
 import qualified Data.ByteString.Lazy as LBS
+import qualified Data.Text as Text
 import Hedgehog
 
 tests :: IO Bool
@@ -19,10 +22,12 @@ tests =
       "Cardano.DbSync.OffChain.Vote"
       [ ("parseAndValidateVoteData handles invalid CIP format", prop_parseInvalidCIPFormat)
       , ("parseAndValidateVoteData handles valid JSON but invalid structure", prop_parseValidJsonInvalidStructure)
+      , ("parseAndValidateVoteData handles unparseable JSON", prop_parseUnparseableJson)
       ]
 
 -- | Test that we can parse JSON with incorrect field types (e.g., doNotList as string instead of bool)
 -- This is based on the issue https://github.com/IntersectMBO/cardano-db-sync/issues/1995
+-- Scenario: Valid JSON but invalid CIP schema -> is_valid = false
 prop_parseInvalidCIPFormat :: Property
 prop_parseInvalidCIPFormat = withTests 1 $ property $ do
   -- Read the test file with invalid doNotList field (string instead of bool)
@@ -32,10 +37,11 @@ prop_parseInvalidCIPFormat = withTests 1 $ property $ do
   -- Run the parser
   result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData fileContent lbsContent Nothing DB.DrepAnchor Nothing
 
-  let (mocvd, val, _hash, _warning) = result
+  let (mocvd, val, _hash, _warning, isValidJson) = result
 
   -- Should succeed in parsing generic JSON
   annotate "Successfully parsed as generic JSON"
+  assert isValidJson
 
   -- Should fail to parse into strongly-typed OffChainVoteData
   assert $ isNothing mocvd
@@ -50,6 +56,7 @@ prop_parseInvalidCIPFormat = withTests 1 $ property $ do
       failure
 
 -- | Test with completely valid JSON but not matching the CIP schema
+-- Scenario: Valid JSON but invalid CIP schema -> is_valid = false
 prop_parseValidJsonInvalidStructure :: Property
 prop_parseValidJsonInvalidStructure = property $ do
   -- Create a valid JSON that doesn't match CIP schema at all
@@ -60,8 +67,46 @@ prop_parseValidJsonInvalidStructure = property $ do
   -- This should succeed because it's valid JSON, just not matching the schema
   result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData bs lbs Nothing DB.DrepAnchor Nothing
 
-  let (mocvd, _val, _hash, _warning) = result
+  let (mocvd, _val, _hash, _warning, isValidJson) = result
 
   annotate "Successfully parsed generic JSON"
+  assert isValidJson
   -- Should not parse into OffChainVoteData
   assert $ isNothing mocvd
+
+-- | Test with completely unparseable content (not valid JSON at all)
+-- Scenario: Invalid JSON but hash matches -> is_valid = NULL
+prop_parseUnparseableJson :: Property
+prop_parseUnparseableJson = property $ do
+  -- Create content that is not valid JSON
+  let notJson = "This is just plain text, not JSON at all!"
+      bs = encodeUtf8 notJson
+      lbs = LBS.fromStrict bs
+
+  -- This should not fail, but instead return an error message in the JSON field
+  result <- liftIO $ runOrThrowIO $ runExceptT $ parseAndValidateVoteData bs lbs Nothing DB.DrepAnchor Nothing
+
+  let (mocvd, val, _hash, _warning, isValidJson) = result
+
+  annotate "Content is not valid JSON"
+  -- Should flag as invalid JSON
+  assert $ not isValidJson
+
+  -- Should not parse into OffChainVoteData
+  assert $ isNothing mocvd
+
+  -- Should have an error message in the JSON value
+  case val of
+    Aeson.Object obj -> do
+      annotate "Has error message object"
+      -- Check that error field exists
+      case KeyMap.lookup (AesonKey.fromString "error") obj of
+        Just (Aeson.String msg) -> do
+          annotate $ "Error message: " <> show msg
+          assert $ Text.isInfixOf "not valid JSON" msg
+        _ -> do
+          annotate "Expected error field with string value"
+          failure
+    _ -> do
+      annotate "Expected JSON object with error message"
+      failure

cardano-db/src/Cardano/Db/Schema/Core/OffChain.hs

@@ -90,7 +90,15 @@ offChainPoolFetchErrorEncoder =
 
 -- |
 -- Table Name: off_chain_vote_data
--- Description:
+-- Description: Stores off-chain voting anchor data with validation status
+--
+-- The is_valid column indicates the parsing status:
+--   • TRUE: Content is valid JSON AND conforms to CIP schema
+--           All related tables are populated
+--   • FALSE: Content is valid JSON BUT does not conform to CIP schema
+--           The json column contains the actual JSON, but related tables are empty
+--   • NULL: Content is not valid JSON at all
+--           The json column contains an error message, bytes column has raw data
 data OffChainVoteData = OffChainVoteData
   { offChainVoteDataVotingAnchorId :: !Id.VotingAnchorId -- noreference
   , offChainVoteDataHash :: !ByteString

doc/schema.md

@@ -1014,7 +1014,14 @@ A table containing pool offchain data fetch errors.
 
 ### `off_chain_vote_data`
 
-The table with the offchain metadata related to Vote Anchors. It accepts metadata in a more lenient way than what's decribed in CIP-100. New in 13.2-Conway.
+Stores off-chain voting anchor data with validation status. The table accepts metadata in a more lenient way than what's described in CIP-100. Only data with hash matches are stored here; hash mismatches are stored in off_chain_vote_fetch_error for retry.
+
+The is_valid column indicates the parsing status:
+  • TRUE: Content is valid JSON AND conforms to CIP-100 schema. All related fields (language, comment) and related tables (off_chain_vote_gov_action_data, off_chain_vote_drep_data, off_chain_vote_author, off_chain_vote_reference, off_chain_vote_external_update) are populated.
+  • FALSE: Hash matches and content is valid JSON BUT does not conform to CIP-100 schema. The json column contains the actual JSON, but language/comment fields and related tables remain empty.
+  • NULL: Hash matches but content is not valid JSON at all. The json column contains an error message, bytes column has raw data. Language/comment fields and related tables remain empty.
+
+New in 13.2-Conway.
 
 * Primary Id: `id`
 
@@ -1028,7 +1035,7 @@ The table with the offchain metadata related to Vote Anchors. It accepts metadat
 | `json` | jsonb | The payload as JSON. |
 | `bytes` | bytea | The raw bytes of the payload. |
 | `warning` | string | A warning that occured while validating the metadata. |
-| `is_valid` | boolean | False if the data is found invalid. db-sync leaves this field null since it normally populates off_chain_vote_fetch_error for invalid data. It can be used manually to mark some metadata invalid by clients. |
+| `is_valid` | boolean | Indicates validation status: TRUE for valid JSON conforming to CIP-100 schema, FALSE for valid JSON not conforming to CIP-100 schema, NULL for content that is not valid JSON at all. |
 
 ### `off_chain_vote_gov_action_data`