diff --git a/Cargo.lock b/Cargo.lock index 3c0c92dc..4b79324b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,15 +303,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.3" @@ -427,8 +418,8 @@ checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" [[package]] name = "cardano-multiplatform-lib" -version = "1.0.3" -source = "git+https://github.com/dcSpark/cardano-multiplatform-lib?branch=metadata-and-addr#8d5e3d8e97b57924f4625695b4ad8379064df03a" +version = "3.1.3" +source = "git+https://github.com/dcSpark/cardano-multiplatform-lib?branch=metadata-and-addr#74462e5c442648b81da8e3dbb7781d817690565b" dependencies = [ "base64", "bech32 0.7.3", @@ -439,7 +430,6 @@ dependencies = [ "digest 0.9.0", "ed25519-bip32", "fraction", - "getrandom", "hex", "itertools", "js-sys", @@ -452,7 +442,8 @@ dependencies = [ "schemars", "serde", "serde_json", - "sha2 0.9.9", + "sha2", + "unicode-segmentation", "wasm-bindgen", ] @@ -599,6 +590,40 @@ dependencies = [ "bitflags", ] +[[package]] +name = "cml-cip25" +version = "1.0.0" +dependencies = [ + "cbor_event", + "cml-core", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "cml-core" +version = "0.1.0" +dependencies = [ + "base64", + "bech32 0.7.3", + "cbor_event", + "cfg-if 1.0.0", + "derivative", + "fraction", + "getrandom", + "hex", + "itertools", + "linked-hash-map", + "num-bigint 0.4.3", + "num-integer", + "rand", + "schemars", + "serde", + "serde_json", + "wasm-bindgen", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -818,6 +843,17 @@ dependencies = [ "syn", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.9.0" @@ -833,7 +869,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" dependencies = [ - "block-buffer 0.10.3", + "block-buffer", "crypto-common", "subtle", ] @@ -1697,12 +1733,6 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "os_str_bytes" version = "6.4.1" @@ -2248,6 +2278,7 @@ version = "0.1.0" dependencies = [ "anyhow", "cardano-multiplatform-lib", + "cml-cip25", "dotenv", "entity", "futures", @@ -2639,19 +2670,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if 1.0.0", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.6" @@ -2820,7 +2838,7 @@ dependencies = [ "serde", "serde_json", "sha-1", - "sha2 0.10.6", + "sha2", "smallvec", "sqlformat", "sqlx-rt", @@ -2848,7 +2866,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "sha2 0.10.6", + "sha2", "sqlx-core", "sqlx-rt", "syn", @@ -3447,9 +3465,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if 1.0.0", "serde", @@ -3459,9 +3477,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" dependencies = [ "bumpalo", "log", @@ -3486,9 +3504,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3496,9 +3514,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", @@ -3509,9 +3527,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" [[package]] name = "web-sys" diff --git a/indexer/reparse/Cargo.toml b/indexer/reparse/Cargo.toml index bd8206d2..74a1601a 100644 --- a/indexer/reparse/Cargo.toml +++ b/indexer/reparse/Cargo.toml @@ -18,3 +18,4 @@ entity = { path = "../entity" } futures = "0.3.21" cardano-multiplatform-lib = { git = "https://github.com/dcSpark/cardano-multiplatform-lib", branch = "metadata-and-addr" } hex = "0.4.0" +cml-cip25 = { "path" = "../../../cardano-multiplatform-lib/cip25/rust" } diff --git a/indexer/reparse/src/reparse.rs b/indexer/reparse/src/reparse.rs index 9f63734e..e6a4154d 100644 --- a/indexer/reparse/src/reparse.rs +++ b/indexer/reparse/src/reparse.rs @@ -1,3 +1,4 @@ +use cml_cip25::serialization::FromBytes; use entity::{ prelude::*, sea_orm::{prelude::*, JoinType, QueryOrder, QuerySelect}, @@ -11,11 +12,86 @@ pub async fn start_reparse(conn: DatabaseConnection) -> anyhow::Result<()> { reparse_addresses(&conn, 0).await?; reparse_tx_out(&conn, 0).await?; reparse_txs(&conn, 0).await?; + // note: cip25 errors are extremely common from projects accidentally not following it, so we ignore them + // reparse_nft(&conn, 0).await?; Ok(()) } static PAGE_SIZE: usize = 8192 * 4; +async fn reparse_nft(conn: &DatabaseConnection, start_index: u64) -> Result<(), DbErr> { + let cip25_count = Cip25Entry::find().count(conn).await?; + let mut cip25_stream = Cip25Entry::find() + .order_by_asc(Cip25EntryColumn::Id) + .filter(Cip25EntryColumn::Id.gt(start_index)) + .paginate(conn, PAGE_SIZE) + .into_stream(); + + while let Some(cip25_entries) = &cip25_stream.try_next().await? { + println!( + "cip25 entries: {} / {} ({:.1}%)", + cip25_entries.first().unwrap().id, + cip25_count, + (100.0 * cip25_entries.first().unwrap().id as f64) / (cip25_count as f64) + ); + for cip25_entry in cip25_entries { + // Option 1: + // let name = asset...get_str('name').as_text(); + // let image = asset.get_str('image').as_text(); + // match cml_cip25::MetadataDetails::from_bytes(cip25_entry.payload.clone()) { + // Err(_) => {} + // Ok(details) => { + // let name = details.name.to_str(); + // let image = String::from(&details.image).as_str(); + // } + // } + + // Option 2: + // match cardano_multiplatform_lib::metadata::TransactionMetadatum::from_bytes( + // cip25_entry.payload.clone(), + // ) { + // Err(_) => {} + // Ok(metadatum) => { + // let assetMap = metadatum.as_map().unwrap(); + // let name = assetMap.get_str("name").and_then(|val| val.as_text()); + // let image_base = assetMap.get_str("image"); + // match image_base.as_ref() { + // Err(_) => {} + // Ok(base) => match base.as_text() { + // Ok(_) => {} + // Err(_) => { + // let mut result: String = "".to_string(); + // if let Ok(list) = base.as_list().as_ref() { + // for i in 0..list.len() { + // result += &list.get(i).as_text().unwrap(); + // } + // } + // } + // }, + // } + // } + // }; + + if let Err(e) = &cml_cip25::MetadataDetails::from_bytes(cip25_entry.payload.clone()) { + let asset = NativeAsset::find() + .filter(NativeAssetColumn::Id.eq(cip25_entry.asset_id)) + .one(conn) + .await? + .unwrap(); + println!( + "\nFailed cip25 entry {}.{} {:?} {}\n", + hex::encode(&asset.policy_id), + hex::encode(&asset.asset_name), + e, + hex::encode(&cip25_entry.payload) + ); + }; + } + } + println!("Done parsing transactions"); + Ok(()) +} + async fn reparse_txs(conn: &DatabaseConnection, start_index: u64) -> Result<(), DbErr> { let tx_count = Transaction::find().count(conn).await?; let mut tx_stream = Transaction::find()