refactor!: use chainhook to listen for chain events instead of a direct stacks api connection (#200)

* chore: add chainhook client

* chore: single predicate draft

* fix: remove old code

* fix: test builds

* test: get contract abi

* test: first sip-019 notif

* test: print event chainhook observer

* feat: consume deployments

* test: chain tip

* fix: unify chainhook events processors

* feat: rollback contract deployments

* feat: rollback sft mint

* chore: draft for sip-019 storage

* feat: keep sip-019 notifs

* fix: notifications table inserts

* fix: update code

* fix: some tests

* fix: admin rpc

* fix: rollback contracts and sip-013 mints

* test: divide into suites

* ci: codecov

* chore: progress

* chore: update deps

* fix: remove abi

* fix: block cache

* chore: migrations

* fix: upgrade chainhook client

* chore: upgrade api toolkit

* chore: divide test files

* test: ft, nft

* test: sft

* fix: notification tests

* fix: dynamic refresh

* fix: token queue tests

* fix: admin rpc tests

* fix: api tests

* chore: remove contract import admin rpc

* feat: start validating with chainhook

* fix: eslint

* fix: admin rpc tests

* ci: run matrix tests properly

* fix: skip ingested blocks

* feat: return asset_identifier in FT responses (#229)

* test: zero token nft

* fix: refactor GCS image cache upload script to typescript (#232)

* fix: image cache ts

* test: more

* fix: code

* build: use v2 docker compose

* fix: admin tests

* fix: try finally

* test: debug logs

* test: only 1

* test: one more

* fix: not the timeout

* fix: different ports

* fix: improve test servers

* fix: add cause to error message

* fix: try 127.0.0.1

* fix: try timeout sleep

* fix: standard close

* test: more

* fix: improve message

* fix: echo status

* fix: detect

* fix: finally

* fix: x

* fix: timeouts

* fix: revert debug stuff

* chore: remove old test

* fix: store chainhook predicate in local file for reuse after restart (#233)

fix: add tests

* fix: process image cache in stages for memory optimization (#234)

* fix: try to use native sharp pipeline

* fix: job queue controls, profiler

* fix: tests

* fix: upgrade toolkit

* fix: improve error propagation

* fix: listen to stream errors

* fix: process cached image step by step

* fix: report errors retried immediately

* fix: image stream

* fix: wait for sharp stream correctly

* chore: upgrade to node 20 (#235)

* feat: provide detailed error responses for tokens with user/contract errors (#236)

* chore: demote httperror to usererror

* fix: add errors

* fix: custom messages

* feat: add testnet support

Author: rafaelcr

.eslintrc.js

@@ -9,7 +9,15 @@ module.exports = {
     ecmaVersion: 2020,
     sourceType: 'module',
   },
-  ignorePatterns: ['*.config.js', 'config/*', '*.mjs', 'tests/*.js', 'client/*'],
+  ignorePatterns: [
+    '*.config.js',
+    'config/*',
+    '*.mjs',
+    'tests/**/*.js',
+    'client/*',
+    'coverage/*',
+    'dist/*',
+  ],
   plugins: ['@typescript-eslint', 'eslint-plugin-tsdoc', 'prettier'],
   rules: {
     'prettier/prettier': 'error',

.github/workflows/ci.yml

@@ -18,15 +18,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
           node-version-file: '.nvmrc'
 
       - name: Cache node modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-node-modules
         with:
@@ -49,6 +49,10 @@ jobs:
         run: npm run lint:prettier
 
   test:
+    strategy:
+      fail-fast: false
+      matrix:
+        suite: [admin, api, chainhook, token-queue]
     runs-on: ubuntu-latest
     env:
       API_HOST: 127.0.0.1
@@ -67,7 +71,7 @@ jobs:
       STACKS_NODE_RPC_HOST: 127.0.0.1
       STACKS_NODE_RPC_PORT: 24440
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -77,7 +81,7 @@ jobs:
           node-version-file: '.nvmrc'
 
       - name: Cache node modules
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         env:
           cache-name: cache-node-modules
         with:
@@ -100,10 +104,12 @@ jobs:
           npm run testenv:logs -- --no-color &> docker-compose-logs.txt &
 
       - name: Run tests
-        run: npm run test -- --coverage
+        run: npm run test:${{ matrix.suite }} -- --coverage
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Print integration environment logs
         run: cat docker-compose-logs.txt
@@ -119,14 +125,14 @@ jobs:
       - lint
       - test
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.GH_TOKEN || secrets.GITHUB_TOKEN }}
           fetch-depth: 0
           persist-credentials: false
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v3
+        uses: cycjimmy/semantic-release-action@v4
         id: semantic
         # Only run on non-PR events or only PRs that aren't from forks
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

.nvmrc

@@ -1 +1 @@
-18
+20

.vscode/launch.json

@@ -75,5 +75,69 @@
       "preLaunchTask": "npm: testenv:run",
       "postDebugTask": "npm: testenv:stop",
     },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Jest: Admin",
+      "program": "${workspaceFolder}/node_modules/jest/bin/jest",
+      "args": [
+        "--testTimeout=3600000",
+        "--runInBand",
+        "--no-cache",
+        "${workspaceFolder}/tests/admin/"
+      ],
+      "outputCapture": "std",
+      "console": "integratedTerminal",
+      "preLaunchTask": "npm: testenv:run",
+      "postDebugTask": "npm: testenv:stop",
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Jest: API",
+      "program": "${workspaceFolder}/node_modules/jest/bin/jest",
+      "args": [
+        "--testTimeout=3600000",
+        "--runInBand",
+        "--no-cache",
+        "${workspaceFolder}/tests/api/"
+      ],
+      "outputCapture": "std",
+      "console": "integratedTerminal",
+      "preLaunchTask": "npm: testenv:run",
+      "postDebugTask": "npm: testenv:stop",
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Jest: Chainhook",
+      "program": "${workspaceFolder}/node_modules/jest/bin/jest",
+      "args": [
+        "--testTimeout=3600000",
+        "--runInBand",
+        "--no-cache",
+        "${workspaceFolder}/tests/chainhook/",
+      ],
+      "outputCapture": "std",
+      "console": "integratedTerminal",
+      "preLaunchTask": "npm: testenv:run",
+      "postDebugTask": "npm: testenv:stop",
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Jest: Token Queue",
+      "program": "${workspaceFolder}/node_modules/jest/bin/jest",
+      "args": [
+        "--testTimeout=3600000",
+        "--runInBand",
+        "--no-cache",
+        "${workspaceFolder}/tests/token-queue/"
+      ],
+      "outputCapture": "std",
+      "console": "integratedTerminal",
+      "preLaunchTask": "npm: testenv:run",
+      "postDebugTask": "npm: testenv:stop",
+    },
   ]
 }

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18-alpine
+FROM node:20-alpine
 
 WORKDIR /app
 COPY . .

config/image-cache.js

@@ -1,7 +1,7 @@
 version: '3.7'
 services:
   postgres:
-    image: "postgres:14"
+    image: "postgres:15"
     ports:
       - "5432:5432"
     environment:

docker/docker-compose.dev.postgres.yml

@@ -13,26 +13,37 @@ export function up(pgm: MigrationBuilder): void {
     principal: {
       type: 'text',
       notNull: true,
+      unique: true,
     },
     sip: {
       type: 'sip_number',
       notNull: true,
     },
-    abi: {
-      type: 'jsonb',
+    token_count: {
+      type: 'numeric',
+    },
+    fungible_token_name: {
+      type: 'text',
+    },
+    non_fungible_token_name: {
+      type: 'text',
+    },
+    block_height: {
+      type: 'int',
+      notNull: true,
+    },
+    index_block_hash: {
+      type: 'text',
       notNull: true,
     },
     tx_id: {
       type: 'text',
       notNull: true,
     },
-    block_height: {
+    tx_index: {
       type: 'int',
       notNull: true,
     },
-    token_count: {
-      type: 'numeric',
-    },
     created_at: {
       type: 'timestamptz',
       default: pgm.func('(NOW())'),
@@ -42,7 +53,5 @@ export function up(pgm: MigrationBuilder): void {
       type: 'timestamptz',
     },
   });
-  pgm.createConstraint('smart_contracts', 'smart_contracts_principal_unique', 'UNIQUE(principal)');
   pgm.createIndex('smart_contracts', [{ name: 'block_height', sort: 'DESC' }]);
-  pgm.createIndex('smart_contracts', ['principal']);
 }