From 2a5e20698416dde5a8fb8289947390c8b168c2a9 Mon Sep 17 00:00:00 2001 From: "blacksmith-sh[bot]" <157653362+blacksmith-sh[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:57:43 +0000 Subject: [PATCH 1/7] Migrate workflows to Blacksmith --- .github/workflows/ci.yml | 2 +- .github/workflows/cleanup.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/customer-docker.yml | 2 +- .github/workflows/dev-docker.yml | 2 +- .github/workflows/flow-api-client.yml | 2 +- .github/workflows/flow.yml | 2 +- .github/workflows/golang-lint.yml | 2 +- .github/workflows/release-drafter.yml | 2 +- .github/workflows/rust-lint.yml | 2 +- .github/workflows/stable-docker.yml | 2 +- .github/workflows/ui-build.yml | 2 +- .github/workflows/ui-lint.yml | 2 +- .github/workflows/update-aws-rds-bundle-shasum.yml | 2 +- .github/workflows/update-docker-compose-stable.yaml | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a7afb5ae0..696592e4e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - runner: [ubuntu-latest] + runner: [blacksmith-16vcpu-ubuntu-2404] postgres-version: [13, 14, 15, 16, 17] runs-on: ${{ matrix.runner }} timeout-minutes: 30 diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index e607a892c8..8ded6e3ea6 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -6,7 +6,7 @@ on: jobs: flow_cleanup: - runs-on: ubuntu-24.04 + runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: checkout sources diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4700abb278..653eee75d7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -11,7 +11,7 @@ on: jobs: analyze: name: Analyze (${{ matrix.language }}) - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-24.04' }} + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} permissions: # required for all workflows diff --git a/.github/workflows/customer-docker.yml b/.github/workflows/customer-docker.yml index 56f049b9fd..26e1f40ea9 100644 --- a/.github/workflows/customer-docker.yml +++ b/.github/workflows/customer-docker.yml @@ -12,7 +12,7 @@ jobs: docker-build: strategy: matrix: - runner: [ubuntu-latest] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/dev-docker.yml b/.github/workflows/dev-docker.yml index 1504c7896d..5c1fd08db4 100644 --- a/.github/workflows/dev-docker.yml +++ b/.github/workflows/dev-docker.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: # ubuntu-latest leverages larger GH runner pool & completes in ~30s instead of ~3m - runner: [ubuntu-latest] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/flow-api-client.yml b/.github/workflows/flow-api-client.yml index c7cb01f19d..e507c2dceb 100644 --- a/.github/workflows/flow-api-client.yml +++ b/.github/workflows/flow-api-client.yml @@ -5,7 +5,7 @@ on: - main jobs: build: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 permissions: contents: write steps: diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 03eeea8395..a510bc655d 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -15,7 +15,7 @@ jobs: strategy: fail-fast: false matrix: - runner: [ubuntu-latest-16-cores] + runner: [blacksmith-16vcpu-ubuntu-2404] db-version: [{pg: 15, mysql: 'mysql-gtid', mongo: '6.0'}, {pg: 16, mysql: 'mysql-pos', mongo: '7.0'}, {pg: 17, mysql: 'maria', mongo: '8.0'}] runs-on: ${{ matrix.runner }} timeout-minutes: 30 diff --git a/.github/workflows/golang-lint.yml b/.github/workflows/golang-lint.yml index c30d5d665b..201937f408 100644 --- a/.github/workflows/golang-lint.yml +++ b/.github/workflows/golang-lint.yml @@ -12,7 +12,7 @@ permissions: jobs: golangci: name: lint - runs-on: [ubuntu-24.04] + runs-on: [blacksmith-4vcpu-ubuntu-2404] steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index c8bc2f3686..ad09317125 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -18,7 +18,7 @@ jobs: permissions: contents: write pull-requests: write - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6 env: diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml index b880ceccbc..15370635da 100644 --- a/.github/workflows/rust-lint.yml +++ b/.github/workflows/rust-lint.yml @@ -13,7 +13,7 @@ jobs: pull-requests: write strategy: matrix: - runner: [ubuntu-latest] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 diff --git a/.github/workflows/stable-docker.yml b/.github/workflows/stable-docker.yml index d77ab4f610..2e25b6bdfa 100644 --- a/.github/workflows/stable-docker.yml +++ b/.github/workflows/stable-docker.yml @@ -9,7 +9,7 @@ jobs: docker-build: strategy: matrix: - runner: [ubuntu-24.04] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml index 7f72372792..1f2de1dd95 100644 --- a/.github/workflows/ui-build.yml +++ b/.github/workflows/ui-build.yml @@ -12,7 +12,7 @@ jobs: name: Build & Test UI strategy: matrix: - runner: [ubuntu-24.04] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} steps: - name: checkout diff --git a/.github/workflows/ui-lint.yml b/.github/workflows/ui-lint.yml index 0e765779a0..79875af082 100644 --- a/.github/workflows/ui-lint.yml +++ b/.github/workflows/ui-lint.yml @@ -16,7 +16,7 @@ jobs: name: Run UI linters strategy: matrix: - runner: [ubuntu-24.04] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} steps: - name: checkout diff --git a/.github/workflows/update-aws-rds-bundle-shasum.yml b/.github/workflows/update-aws-rds-bundle-shasum.yml index 9fcd215f86..77fe6271a5 100644 --- a/.github/workflows/update-aws-rds-bundle-shasum.yml +++ b/.github/workflows/update-aws-rds-bundle-shasum.yml @@ -19,7 +19,7 @@ env: jobs: update-docker-compose-tag: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: diff --git a/.github/workflows/update-docker-compose-stable.yaml b/.github/workflows/update-docker-compose-stable.yaml index b6ec63f081..4e8d74788f 100644 --- a/.github/workflows/update-docker-compose-stable.yaml +++ b/.github/workflows/update-docker-compose-stable.yaml @@ -22,7 +22,7 @@ env: jobs: update-docker-compose-tag: - runs-on: ubuntu-latest + runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: From 8c7f26abf3a7683ec806d9c28fe2f779dad3bae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Thu, 28 Aug 2025 20:10:06 +0000 Subject: [PATCH 2/7] smaller skus --- .github/workflows/ci.yml | 2 +- .github/workflows/cleanup.yml | 2 +- .github/workflows/customer-docker.yml | 2 +- .github/workflows/flow-api-client.yml | 2 +- .github/workflows/release-drafter.yml | 2 +- .github/workflows/update-aws-rds-bundle-shasum.yml | 8 ++++---- .github/workflows/update-docker-compose-stable.yaml | 6 +++--- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 696592e4e2..634068c43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - runner: [blacksmith-16vcpu-ubuntu-2404] + runner: [blacksmith-4vcpu-ubuntu-2404] postgres-version: [13, 14, 15, 16, 17] runs-on: ${{ matrix.runner }} timeout-minutes: 30 diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 8ded6e3ea6..65edf9893c 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -6,7 +6,7 @@ on: jobs: flow_cleanup: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: checkout sources diff --git a/.github/workflows/customer-docker.yml b/.github/workflows/customer-docker.yml index 26e1f40ea9..3b78d4a9a3 100644 --- a/.github/workflows/customer-docker.yml +++ b/.github/workflows/customer-docker.yml @@ -12,7 +12,7 @@ jobs: docker-build: strategy: matrix: - runner: [blacksmith-4vcpu-ubuntu-2404] + runner: [blacksmith-2vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/flow-api-client.yml b/.github/workflows/flow-api-client.yml index e507c2dceb..333ba576ec 100644 --- a/.github/workflows/flow-api-client.yml +++ b/.github/workflows/flow-api-client.yml @@ -5,7 +5,7 @@ on: - main jobs: build: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-2vcpu-ubuntu-2404 permissions: contents: write steps: diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index ad09317125..442a290cff 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -18,7 +18,7 @@ jobs: permissions: contents: write pull-requests: write - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6 env: diff --git a/.github/workflows/update-aws-rds-bundle-shasum.yml b/.github/workflows/update-aws-rds-bundle-shasum.yml index 77fe6271a5..33801669ca 100644 --- a/.github/workflows/update-aws-rds-bundle-shasum.yml +++ b/.github/workflows/update-aws-rds-bundle-shasum.yml @@ -5,7 +5,7 @@ on: - cron: '0 15 * * *' workflow_dispatch: inputs: {} - + permissions: issues: write pull-requests: write @@ -19,7 +19,7 @@ env: jobs: update-docker-compose-tag: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: @@ -38,7 +38,7 @@ jobs: git add -u git commit -m 'chore(automated): update `aws-rds-bundle-shasum` in Dockerfile' - + git push -u origin "${PR_BRANCH}" --force-with-lease PR_ID=$(gh pr list --label "${PR_LABEL}" --head "${PR_BRANCH}" --json number | jq -r '.[0].number // ""') @@ -46,7 +46,7 @@ jobs: PR_ID=$(gh pr create -l "$PR_LABEL" -t "$PR_TITLE" --body "") fi - + gh pr merge --auto --squash env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/update-docker-compose-stable.yaml b/.github/workflows/update-docker-compose-stable.yaml index 4e8d74788f..499b3e8fdd 100644 --- a/.github/workflows/update-docker-compose-stable.yaml +++ b/.github/workflows/update-docker-compose-stable.yaml @@ -8,7 +8,7 @@ on: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" - + permissions: issues: write pull-requests: write @@ -22,7 +22,7 @@ env: jobs: update-docker-compose-tag: - runs-on: blacksmith-4vcpu-ubuntu-2404 + runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: @@ -49,7 +49,7 @@ jobs: PR_ID=$(gh pr create -l "$PR_LABEL" -t "$PR_TITLE" --body "") fi - + gh pr merge --auto --squash env: GH_TOKEN: ${{ github.token }} From cdcc56a67d8c9e05a8dfe8db979723d843b95385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 29 Aug 2025 15:07:08 +0000 Subject: [PATCH 3/7] patience --- .github/workflows/flow.yml | 4 ++-- flow/e2e/test_utils.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index a510bc655d..066bb0c380 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -18,7 +18,7 @@ jobs: runner: [blacksmith-16vcpu-ubuntu-2404] db-version: [{pg: 15, mysql: 'mysql-gtid', mongo: '6.0'}, {pg: 16, mysql: 'mysql-pos', mongo: '7.0'}, {pg: 17, mysql: 'maria', mongo: '8.0'}] runs-on: ${{ matrix.runner }} - timeout-minutes: 30 + timeout-minutes: 180 services: catalog: image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine @@ -362,7 +362,7 @@ jobs: ./peer-flow worker & ./peer-flow snapshot-worker & ./peer-flow api --port 8112 --gateway-port 8113 & - go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 900s -args -test.gocoverdir="$PWD/coverage" + go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 9000s -args -test.gocoverdir="$PWD/coverage" killall peer-flow sleep 1 go tool covdata textfmt -i=coverage -o ../coverage.out diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index 6aafc8a866..7a0fc0c7a2 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -754,6 +754,7 @@ func EnvEqualRecordBatches(t *testing.T, env WorkflowRun, q *model.QRecordBatch, } func EnvWaitFor(t *testing.T, env WorkflowRun, timeout time.Duration, reason string, f func() bool) { + timeout = time.Hour * 2 t.Helper() t.Log("WaitFor", reason, time.Now()) From 1f115b9134ea2a7e74ffee8a7b70755c8c52d6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 29 Aug 2025 16:08:16 +0000 Subject: [PATCH 4/7] dtolnay-stable --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 634068c43e..103722f41f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,8 @@ jobs: steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: dtolnay/rust-toolchain@stable + - name: generate or hydrate protos uses: ./.github/actions/genprotos From 87297516165514067ebeeda7a29c39e328e6a77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 29 Aug 2025 16:54:07 +0000 Subject: [PATCH 5/7] more port --- .github/workflows/flow.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 066bb0c380..3febea815a 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -69,6 +69,12 @@ jobs: - 4317:4317 steps: + - name: Expand local port range to prevent exhaustion + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="15000 65000" + sudo sysctl -w net.ipv4.tcp_fin_timeout=30 + sudo sysctl -w net.ipv4.tcp_tw_reuse=1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: generate or hydrate protos From 34eefc05416776165d08be4764a012fdf6d8c426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 29 Aug 2025 18:51:19 +0000 Subject: [PATCH 6/7] p8 --- .github/workflows/flow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 3febea815a..e5c1b13bf8 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -368,7 +368,7 @@ jobs: ./peer-flow worker & ./peer-flow snapshot-worker & ./peer-flow api --port 8112 --gateway-port 8113 & - go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 9000s -args -test.gocoverdir="$PWD/coverage" + go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 8 ./... -timeout 9000s -args -test.gocoverdir="$PWD/coverage" killall peer-flow sleep 1 go tool covdata textfmt -i=coverage -o ../coverage.out From 399f2e219f6b8e62dacbf9285180de36caab99d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Fri, 29 Aug 2025 20:02:47 +0000 Subject: [PATCH 7/7] GOMAXPROCS --- .github/workflows/flow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index e5c1b13bf8..d4f6683996 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -368,7 +368,7 @@ jobs: ./peer-flow worker & ./peer-flow snapshot-worker & ./peer-flow api --port 8112 --gateway-port 8113 & - go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 8 ./... -timeout 9000s -args -test.gocoverdir="$PWD/coverage" + GOMAXPROCS=8 go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 8 ./... -timeout 9000s -args -test.gocoverdir="$PWD/coverage" killall peer-flow sleep 1 go tool covdata textfmt -i=coverage -o ../coverage.out