diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a7afb5ae0..103722f41f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - runner: [ubuntu-latest] + runner: [blacksmith-4vcpu-ubuntu-2404] postgres-version: [13, 14, 15, 16, 17] runs-on: ${{ matrix.runner }} timeout-minutes: 30 @@ -32,6 +32,8 @@ jobs: steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 + - uses: dtolnay/rust-toolchain@stable + - name: generate or hydrate protos uses: ./.github/actions/genprotos diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index e607a892c8..65edf9893c 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -6,7 +6,7 @@ on: jobs: flow_cleanup: - runs-on: ubuntu-24.04 + runs-on: blacksmith-2vcpu-ubuntu-2404 timeout-minutes: 60 steps: - name: checkout sources diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4700abb278..653eee75d7 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -11,7 +11,7 @@ on: jobs: analyze: name: Analyze (${{ matrix.language }}) - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-24.04' }} + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} permissions: # required for all workflows diff --git a/.github/workflows/customer-docker.yml b/.github/workflows/customer-docker.yml index 56f049b9fd..3b78d4a9a3 100644 --- a/.github/workflows/customer-docker.yml +++ b/.github/workflows/customer-docker.yml @@ -12,7 +12,7 @@ jobs: docker-build: strategy: matrix: - runner: [ubuntu-latest] + runner: [blacksmith-2vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/dev-docker.yml b/.github/workflows/dev-docker.yml index 1504c7896d..5c1fd08db4 100644 --- a/.github/workflows/dev-docker.yml +++ b/.github/workflows/dev-docker.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: # ubuntu-latest leverages larger GH runner pool & completes in ~30s instead of ~3m - runner: [ubuntu-latest] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/flow-api-client.yml b/.github/workflows/flow-api-client.yml index c7cb01f19d..333ba576ec 100644 --- a/.github/workflows/flow-api-client.yml +++ b/.github/workflows/flow-api-client.yml @@ -5,7 +5,7 @@ on: - main jobs: build: - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 permissions: contents: write steps: diff --git a/.github/workflows/flow.yml b/.github/workflows/flow.yml index 03eeea8395..d4f6683996 100644 --- a/.github/workflows/flow.yml +++ b/.github/workflows/flow.yml @@ -15,10 +15,10 @@ jobs: strategy: fail-fast: false matrix: - runner: [ubuntu-latest-16-cores] + runner: [blacksmith-16vcpu-ubuntu-2404] db-version: [{pg: 15, mysql: 'mysql-gtid', mongo: '6.0'}, {pg: 16, mysql: 'mysql-pos', mongo: '7.0'}, {pg: 17, mysql: 'maria', mongo: '8.0'}] runs-on: ${{ matrix.runner }} - timeout-minutes: 30 + timeout-minutes: 180 services: catalog: image: imresamu/postgis:${{ matrix.db-version.pg }}-3.5-alpine @@ -69,6 +69,12 @@ jobs: - 4317:4317 steps: + - name: Expand local port range to prevent exhaustion + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="15000 65000" + sudo sysctl -w net.ipv4.tcp_fin_timeout=30 + sudo sysctl -w net.ipv4.tcp_tw_reuse=1 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - name: generate or hydrate protos @@ -362,7 +368,7 @@ jobs: ./peer-flow worker & ./peer-flow snapshot-worker & ./peer-flow api --port 8112 --gateway-port 8113 & - go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 32 ./... -timeout 900s -args -test.gocoverdir="$PWD/coverage" + GOMAXPROCS=8 go test -cover -coverpkg github.com/PeerDB-io/peerdb/flow/... -p 8 ./... -timeout 9000s -args -test.gocoverdir="$PWD/coverage" killall peer-flow sleep 1 go tool covdata textfmt -i=coverage -o ../coverage.out diff --git a/.github/workflows/golang-lint.yml b/.github/workflows/golang-lint.yml index c30d5d665b..201937f408 100644 --- a/.github/workflows/golang-lint.yml +++ b/.github/workflows/golang-lint.yml @@ -12,7 +12,7 @@ permissions: jobs: golangci: name: lint - runs-on: [ubuntu-24.04] + runs-on: [blacksmith-4vcpu-ubuntu-2404] steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index c8bc2f3686..442a290cff 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -18,7 +18,7 @@ jobs: permissions: contents: write pull-requests: write - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: release-drafter/release-drafter@b1476f6e6eb133afa41ed8589daba6dc69b4d3f5 # v6 env: diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml index b880ceccbc..15370635da 100644 --- a/.github/workflows/rust-lint.yml +++ b/.github/workflows/rust-lint.yml @@ -13,7 +13,7 @@ jobs: pull-requests: write strategy: matrix: - runner: [ubuntu-latest] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 diff --git a/.github/workflows/stable-docker.yml b/.github/workflows/stable-docker.yml index d77ab4f610..2e25b6bdfa 100644 --- a/.github/workflows/stable-docker.yml +++ b/.github/workflows/stable-docker.yml @@ -9,7 +9,7 @@ jobs: docker-build: strategy: matrix: - runner: [ubuntu-24.04] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} permissions: contents: read diff --git a/.github/workflows/ui-build.yml b/.github/workflows/ui-build.yml index 7f72372792..1f2de1dd95 100644 --- a/.github/workflows/ui-build.yml +++ b/.github/workflows/ui-build.yml @@ -12,7 +12,7 @@ jobs: name: Build & Test UI strategy: matrix: - runner: [ubuntu-24.04] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} steps: - name: checkout diff --git a/.github/workflows/ui-lint.yml b/.github/workflows/ui-lint.yml index 0e765779a0..79875af082 100644 --- a/.github/workflows/ui-lint.yml +++ b/.github/workflows/ui-lint.yml @@ -16,7 +16,7 @@ jobs: name: Run UI linters strategy: matrix: - runner: [ubuntu-24.04] + runner: [blacksmith-4vcpu-ubuntu-2404] runs-on: ${{ matrix.runner }} steps: - name: checkout diff --git a/.github/workflows/update-aws-rds-bundle-shasum.yml b/.github/workflows/update-aws-rds-bundle-shasum.yml index 9fcd215f86..33801669ca 100644 --- a/.github/workflows/update-aws-rds-bundle-shasum.yml +++ b/.github/workflows/update-aws-rds-bundle-shasum.yml @@ -5,7 +5,7 @@ on: - cron: '0 15 * * *' workflow_dispatch: inputs: {} - + permissions: issues: write pull-requests: write @@ -19,7 +19,7 @@ env: jobs: update-docker-compose-tag: - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: @@ -38,7 +38,7 @@ jobs: git add -u git commit -m 'chore(automated): update `aws-rds-bundle-shasum` in Dockerfile' - + git push -u origin "${PR_BRANCH}" --force-with-lease PR_ID=$(gh pr list --label "${PR_LABEL}" --head "${PR_BRANCH}" --json number | jq -r '.[0].number // ""') @@ -46,7 +46,7 @@ jobs: PR_ID=$(gh pr create -l "$PR_LABEL" -t "$PR_TITLE" --body "") fi - + gh pr merge --auto --squash env: GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/update-docker-compose-stable.yaml b/.github/workflows/update-docker-compose-stable.yaml index b6ec63f081..499b3e8fdd 100644 --- a/.github/workflows/update-docker-compose-stable.yaml +++ b/.github/workflows/update-docker-compose-stable.yaml @@ -8,7 +8,7 @@ on: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" - + permissions: issues: write pull-requests: write @@ -22,7 +22,7 @@ env: jobs: update-docker-compose-tag: - runs-on: ubuntu-latest + runs-on: blacksmith-2vcpu-ubuntu-2404 steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 with: @@ -49,7 +49,7 @@ jobs: PR_ID=$(gh pr create -l "$PR_LABEL" -t "$PR_TITLE" --body "") fi - + gh pr merge --auto --squash env: GH_TOKEN: ${{ github.token }} diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go index 6aafc8a866..7a0fc0c7a2 100644 --- a/flow/e2e/test_utils.go +++ b/flow/e2e/test_utils.go @@ -754,6 +754,7 @@ func EnvEqualRecordBatches(t *testing.T, env WorkflowRun, q *model.QRecordBatch, } func EnvWaitFor(t *testing.T, env WorkflowRun, timeout time.Duration, reason string, f func() bool) { + timeout = time.Hour * 2 t.Helper() t.Log("WaitFor", reason, time.Now())