Add logic for writing signatures to db inside SourcifyDatabaseService (#2357)

* Add signature-util for extracting SignatureData from ABIs

* Add signature tables interfaces

* Add logic for writing signatures to SourcifyDatabaseService

* Add tests

* address PR feedback

* Ignore ABI fragments with custom type

Author: manuelwedler

services/server/src/server/services/VerificationService.ts

@@ -345,14 +345,14 @@ export class VerificationService {
     return verificationId;
   }
 
-  private async verifyViaWorker(
+  private verifyViaWorker(
     verificationId: VerificationJobId,
     functionName: string,
     input:
       | VerifyFromJsonInput
       | VerifyFromMetadataInput
       | VerifyFromEtherscanInput,
-  ): Promise<void> {
+  ): void {
     const task = this.workerPool
       .run(input, { name: functionName })
       .then((output: VerifyOutput) => {

services/server/src/server/services/storageServices/AbstractDatabaseService.ts

@@ -1,6 +1,6 @@
 import { VerificationExport } from "@ethereum-sourcify/lib-sourcify";
 import * as DatabaseUtil from "../utils/database-util";
-import { bytesFromString } from "../utils/database-util";
+import { bytesFromString, Tables } from "../utils/database-util";
 import { Database, DatabaseOptions } from "../utils/Database";
 import { PoolClient, QueryResult } from "pg";
 
@@ -44,7 +44,7 @@ export default abstract class AbstractDatabaseService {
   async insertNewVerifiedContract(
     databaseColumns: DatabaseUtil.DatabaseColumns,
     client: PoolClient,
-  ): Promise<string> {
+  ): Promise<Tables.VerifiedContract["id"]> {
     try {
       let recompiledCreationCodeInsertResult:
         | QueryResult<Pick<DatabaseUtil.Tables.Code, "bytecode_hash">>
@@ -127,7 +127,7 @@ export default abstract class AbstractDatabaseService {
   async updateExistingVerifiedContract(
     databaseColumns: DatabaseUtil.DatabaseColumns,
     client: PoolClient,
-  ): Promise<string> {
+  ): Promise<Tables.VerifiedContract["id"]> {
     // runtime bytecodes must exist
     if (databaseColumns.recompiledRuntimeCode.bytecode === undefined) {
       throw new Error("Missing normalized runtime bytecode");
@@ -224,8 +224,8 @@ export default abstract class AbstractDatabaseService {
     poolClient: PoolClient,
   ): Promise<{
     type: "update" | "insert";
-    verifiedContractId: string;
-    oldVerifiedContractId?: string;
+    verifiedContractId: Tables.VerifiedContract["id"];
+    oldVerifiedContractId?: Tables.VerifiedContract["id"];
   }> {
     this.validateVerificationBeforeStoring(verification);
 

services/server/src/server/services/storageServices/SourcifyDatabaseService.ts

@@ -12,6 +12,7 @@ import {
   Field,
   FIELDS_TO_STORED_PROPERTIES,
   StoredProperties,
+  Tables,
 } from "../utils/database-util";
 import {
   ContractData,
@@ -28,6 +29,7 @@ import {
   VerificationJob,
   Match,
   VerificationJobId,
+  BytesKeccak,
 } from "../../types";
 import Path from "path";
 import {
@@ -37,6 +39,7 @@ import {
   toMatchLevel,
 } from "../utils/util";
 import { getAddress, id as keccak256Str } from "ethers";
+import { extractSignaturesFromAbi } from "../utils/signature-util";
 import { BadRequestError } from "../../../common/errors";
 import { RWStorageIdentifiers } from "./identifiers";
 import semver from "semver";
@@ -884,6 +887,57 @@ export class SourcifyDatabaseService
     });
   }
 
+  private async storeSignatures(
+    poolClient: PoolClient,
+    verifiedContractId: Tables.VerifiedContract["id"],
+    verification: VerificationExport,
+  ): Promise<void> {
+    try {
+      const compiledContractResult =
+        await this.database.getCompilationIdForVerifiedContract(
+          verifiedContractId,
+          poolClient,
+        );
+      if (compiledContractResult.rowCount === 0) {
+        throw new Error(
+          `No compilation found for verifiedContractId ${verifiedContractId}`,
+        );
+      }
+      const compilationId = compiledContractResult.rows[0].compilation_id;
+
+      const abi = verification.compilation.contractCompilerOutput.abi;
+      if (!abi) {
+        throw new Error("No ABI found in compilation output");
+      }
+
+      const signatureData = extractSignaturesFromAbi(abi);
+      const signatureColumns = signatureData.map((sig) => ({
+        signature_hash_32: bytesFromString<BytesKeccak>(sig.signatureHash32),
+        signature: sig.signature,
+        signature_type: sig.signatureType,
+      }));
+
+      await this.database.insertSignatures(signatureColumns, poolClient);
+      await this.database.insertCompiledContractSignatures(
+        compilationId,
+        signatureColumns,
+        poolClient,
+      );
+
+      logger.info("Stored signatures to SourcifyDatabase", {
+        verifiedContractId,
+        compilationId,
+        signatureCount: signatureData.length,
+      });
+    } catch (error) {
+      // Don't throw on errors, the job should not fail
+      logger.error("Error storing signatures", {
+        verifiedContractId,
+        error: error,
+      });
+    }
+  }
+
   // Override this method to include the SourcifyMatch
   async storeVerificationWithPoolClient(
     poolClient: PoolClient,
@@ -892,7 +946,7 @@ export class SourcifyDatabaseService
       verificationId: VerificationJobId;
       finishTime: Date;
     },
-  ): Promise<void> {
+  ): Promise<{ verifiedContractId: Tables.VerifiedContract["id"] }> {
     try {
       const { type, verifiedContractId, oldVerifiedContractId } =
         await super.insertOrUpdateVerification(verification, poolClient);
@@ -962,6 +1016,8 @@ export class SourcifyDatabaseService
           poolClient,
         );
       }
+
+      return { verifiedContractId: verifiedContractId };
     } catch (error: any) {
       logger.error("Error storing verification", {
         error: error,
@@ -970,19 +1026,30 @@ export class SourcifyDatabaseService
     }
   }
 
-  // Override this method to include the SourcifyMatch
   async storeVerification(
     verification: VerificationExport,
     jobData?: {
       verificationId: VerificationJobId;
       finishTime: Date;
     },
   ): Promise<void> {
+    const { verifiedContractId } = await this.withTransaction(
+      async (transactionPoolClient) => {
+        return await this.storeVerificationWithPoolClient(
+          transactionPoolClient,
+          verification,
+          jobData,
+        );
+      },
+    );
+
+    // Separate transaction because storing the verification should not fail
+    // if signatures cannot be stored
     await this.withTransaction(async (transactionPoolClient) => {
-      await this.storeVerificationWithPoolClient(
+      await this.storeSignatures(
         transactionPoolClient,
+        verifiedContractId,
         verification,
-        jobData,
       );
     });
   }

services/server/src/server/services/utils/Database.ts

@@ -1,5 +1,5 @@
 import { Pool, PoolClient, QueryResult } from "pg";
-import { Bytes } from "../../types";
+import { Bytes, BytesKeccak } from "../../types";
 import {
   bytesFromString,
   GetSourcifyMatchByChainAddressResult,
@@ -285,6 +285,16 @@ ${
     );
   }
 
+  async getCompilationIdForVerifiedContract(
+    verifiedContractId: Tables.VerifiedContract["id"],
+    poolClient?: PoolClient,
+  ): Promise<QueryResult<Pick<Tables.VerifiedContract, "compilation_id">>> {
+    return await (poolClient || this.pool).query(
+      `SELECT compilation_id FROM verified_contracts WHERE id = $1`,
+      [verifiedContractId],
+    );
+  }
+
   async insertSourcifyMatch(
     {
       verified_contract_id,
@@ -716,6 +726,68 @@ ${
     );
   }
 
+  async insertSignatures(
+    signatures: Omit<Tables.Signatures, "signature_hash_4">[],
+    poolClient?: PoolClient,
+  ): Promise<void> {
+    if (signatures.length === 0) {
+      return;
+    }
+
+    const valueIndexes: string[] = [];
+    const queryValues: (BytesKeccak | string)[] = [];
+
+    signatures.forEach((_, index) => {
+      const baseIndex = index * 2 + 1;
+      valueIndexes.push(`($${baseIndex}, $${baseIndex + 1})`);
+    });
+
+    signatures.forEach(({ signature_hash_32, signature }) => {
+      queryValues.push(signature_hash_32, signature);
+    });
+
+    await (poolClient || this.pool).query(
+      `INSERT INTO ${this.schema}.signatures (signature_hash_32, signature) 
+       VALUES ${valueIndexes.join(", ")} 
+       ON CONFLICT (signature_hash_32) DO NOTHING`,
+      queryValues,
+    );
+  }
+
+  async insertCompiledContractSignatures(
+    compilation_id: string,
+    signatures: Omit<
+      Tables.CompiledContractsSignatures,
+      "id" | "compilation_id"
+    >[],
+    poolClient?: PoolClient,
+  ): Promise<void> {
+    if (signatures.length === 0) {
+      return;
+    }
+
+    const valueIndexes: string[] = [];
+    const queryValues: (BytesKeccak | string)[] = [];
+
+    signatures.forEach((_, index) => {
+      const baseIndex = index * 3 + 1;
+      valueIndexes.push(
+        `($${baseIndex}, $${baseIndex + 1}, $${baseIndex + 2})`,
+      );
+    });
+
+    signatures.forEach(({ signature_hash_32, signature_type }) => {
+      queryValues.push(compilation_id, signature_hash_32, signature_type);
+    });
+
+    await (poolClient || this.pool).query(
+      `INSERT INTO ${this.schema}.compiled_contracts_signatures (compilation_id, signature_hash_32, signature_type) 
+       VALUES ${valueIndexes.join(", ")} 
+       ON CONFLICT (compilation_id, signature_hash_32, signature_type) DO NOTHING`,
+      queryValues,
+    );
+  }
+
   async insertVerifiedContract(
     poolClient: PoolClient,
     {

services/server/src/server/services/utils/database-util.ts

@@ -164,6 +164,19 @@ export namespace Tables {
     onchain_runtime_code: Nullable<Bytes>;
     creation_transaction_hash: Nullable<Bytes>;
   }
+
+  export interface Signatures {
+    signature_hash_32: BytesKeccak;
+    signature_hash_4: Bytes;
+    signature: string;
+  }
+
+  export interface CompiledContractsSignatures {
+    id: string;
+    compilation_id: string;
+    signature_hash_32: BytesKeccak;
+    signature_type: "function" | "event" | "error";
+  }
 }
 
 export interface SourceInformation {

services/server/src/server/services/utils/signature-util.ts

@@ -0,0 +1,39 @@
+import { id as keccak256str, Fragment, JsonFragment } from "ethers";
+
+export interface SignatureData {
+  signature: string;
+  signatureHash32: string;
+  signatureType: "function" | "event" | "error";
+}
+
+export function extractSignaturesFromAbi(abi: JsonFragment[]): SignatureData[] {
+  const signatures: SignatureData[] = [];
+
+  for (const item of abi) {
+    let fragment: Fragment;
+    try {
+      fragment = Fragment.from(item);
+    } catch (error) {
+      // Ignore invalid fragments
+      // e.g. with custom type as they can appear in library ABIs
+      continue;
+    }
+    switch (fragment.type) {
+      case "function":
+      case "event":
+      case "error":
+        signatures.push(getSignatureData(fragment));
+    }
+  }
+
+  return signatures;
+}
+
+function getSignatureData(fragment: Fragment): SignatureData {
+  const signature = fragment.format("sighash");
+  return {
+    signature,
+    signatureHash32: keccak256str(signature),
+    signatureType: fragment.type as "function" | "event" | "error",
+  };
+}

services/server/test/helpers/helpers.ts

@@ -291,6 +291,8 @@ export async function resetDatabase(sourcifyDatabase: Pool) {
   );
   await sourcifyDatabase.query("DELETE FROM verified_contracts");
   await sourcifyDatabase.query("DELETE FROM contract_deployments");
+  await sourcifyDatabase.query("DELETE FROM compiled_contracts_signatures");
+  await sourcifyDatabase.query("DELETE FROM signatures");
   await sourcifyDatabase.query("DELETE FROM compiled_contracts_sources");
   await sourcifyDatabase.query("DELETE FROM sources");
   await sourcifyDatabase.query("DELETE FROM compiled_contracts");