Skip to content

Commit 67dff51

Browse files
yeya24yeya24
authored
Merge pull request #1 from TheKongers/add-foundational-corbench-code
added foundational corbench code in preperation for moving code over
2 parents d065181 + 32d31aa commit 67dff51

File tree

9 files changed

+378
-2
lines changed

9 files changed

+378
-2
lines changed

‎README.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
1-
# test-infra
2-
Cortex E2E benchmarking tool
1+
# Corbench
2+
Benchmarking tool for cortex
3+
4+
Refer to [corbench/README.md](corbench/README.md)

‎go.mod

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
module github.com/cortexproject/test-infra
2+
3+
go 1.24.3
4+
5+
require (
6+
github.com/aws/aws-sdk-go v1.55.7
7+
gopkg.in/alecthomas/kingpin.v2 v2.2.6
8+
gopkg.in/yaml.v2 v2.4.0
9+
k8s.io/api v0.31.2
10+
k8s.io/apiextensions-apiserver v0.31.2
11+
k8s.io/apimachinery v0.31.2
12+
k8s.io/client-go v0.31.2
13+
k8s.io/cloud-provider-gcp v0.0.0-20241115222652-d74b2e1d6653
14+
sigs.k8s.io/aws-iam-authenticator v0.6.27
15+
)
16+
17+
require (
18+
cloud.google.com/go/compute/metadata v0.7.0 // indirect
19+
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
20+
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
21+
github.com/beorn7/perks v1.0.1 // indirect
22+
github.com/cespare/xxhash/v2 v2.3.0 // indirect
23+
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
24+
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
25+
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
26+
github.com/go-logr/logr v1.4.2 // indirect
27+
github.com/go-openapi/jsonpointer v0.21.0 // indirect
28+
github.com/go-openapi/jsonreference v0.21.0 // indirect
29+
github.com/go-openapi/swag v0.23.0 // indirect
30+
github.com/gofrs/flock v0.8.1 // indirect
31+
github.com/gogo/protobuf v1.3.2 // indirect
32+
github.com/golang/protobuf v1.5.4 // indirect
33+
github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
34+
github.com/google/go-cmp v0.7.0 // indirect
35+
github.com/google/gofuzz v1.2.0 // indirect
36+
github.com/google/uuid v1.6.0 // indirect
37+
github.com/imdario/mergo v0.3.16 // indirect
38+
github.com/jmespath/go-jmespath v0.4.0 // indirect
39+
github.com/josharian/intern v1.0.0 // indirect
40+
github.com/json-iterator/go v1.1.12 // indirect
41+
github.com/mailru/easyjson v0.7.7 // indirect
42+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
43+
github.com/modern-go/reflect2 v1.0.2 // indirect
44+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
45+
github.com/onsi/gomega v1.33.1 // indirect
46+
github.com/prometheus/client_golang v1.22.0 // indirect
47+
github.com/prometheus/client_model v0.6.2 // indirect
48+
github.com/prometheus/common v0.64.0 // indirect
49+
github.com/prometheus/procfs v0.15.1 // indirect
50+
github.com/rogpeppe/go-internal v1.13.1 // indirect
51+
github.com/sirupsen/logrus v1.9.3 // indirect
52+
github.com/spf13/pflag v1.0.5 // indirect
53+
github.com/x448/float16 v0.8.4 // indirect
54+
golang.org/x/net v0.41.0 // indirect
55+
golang.org/x/oauth2 v0.30.0 // indirect
56+
golang.org/x/sys v0.33.0 // indirect
57+
golang.org/x/term v0.32.0 // indirect
58+
golang.org/x/text v0.26.0 // indirect
59+
golang.org/x/time v0.12.0 // indirect
60+
google.golang.org/protobuf v1.36.6 // indirect
61+
gopkg.in/inf.v0 v0.9.1 // indirect
62+
gopkg.in/yaml.v3 v3.0.1 // indirect
63+
k8s.io/klog/v2 v2.130.1 // indirect
64+
k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
65+
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
66+
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
67+
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
68+
sigs.k8s.io/yaml v1.4.0 // indirect
69+
)
70+
71+
// Remove broken version.
72+
exclude k8s.io/cloud-provider-gcp/providers v0.0.0-00010101000000-000000000000

‎go.sum

Lines changed: 189 additions & 0 deletions
Large diffs are not rendered by default.

‎scripts/README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
## Grafana Dashboard Sync Script
2+
3+
This script updates the grafana dashboard deployment based on the grafana dashboard jsons in /corbench/c-manifests/cluster-infra/dashboards
4+
5+
Simply update the grafana dashboard json, then in this /scripts directory run the following script to update the config file
6+
7+
```bash
8+
./sync-corbench-dashboards.sh
9+
```

‎scripts/sync-corbench-dashboards.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
3+
echo 'apiVersion: v1' > ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
4+
echo 'kind: ConfigMap' >> ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
5+
echo 'metadata:' >> ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
6+
echo ' name: grafana-dashboards' >> ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
7+
echo 'data:' >> ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
8+
9+
# Loop over files in ../corbench/c-manifests/cluster-infra/dashboards.
10+
for file in $(ls ../corbench/c-manifests/cluster-infra/dashboards); do
11+
# Read the file content.
12+
content=$(cat ../corbench/c-manifests/cluster-infra/dashboards/$file)
13+
echo " $file: |" >> ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
14+
echo "$content" | sed 's/^/ /' >> ../corbench/c-manifests/cluster-infra/grafana_dashboard_dashboards_noparse.yaml
15+
done

‎tools/cortex-builder/Dockerfile

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM --platform=linux/amd64 golang:1.24.3-bullseye
2+
3+
RUN mkdir -p /go/src/github.com
4+
5+
COPY ./build.sh /go/src/github.com/build.sh
6+
7+
RUN chmod +x /go/src/github.com/build.sh
8+
9+
ENTRYPOINT ["/go/src/github.com/build.sh"]

‎tools/cortex-builder/README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
### Cortex-Builder
2+
3+
This is used for building Cortex binaries from Pull Requests and running them on containers.
4+
This tool can be used to build binaries for the Pull Request being benchmarked or tested.
5+
6+
You can upload this docker image in the correct linux/amd64 archetecture used by this tool to dockerhub with the build-multi-arch.sh
7+
8+
### Building Docker Image to Upload
9+
10+
The [build-multi-arch.sh](./build-multi-arch.sh) script builds and pushes the cortex-builder docker image to dockerhub using the correct archetecture(linux/amd64).
11+
12+
### Built Binaries
13+
14+
The builder will automatically pull and create the cortex binary in the volume directory

‎tools/cortex-builder/build-multi-arch.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
# Set your Docker Hub username or organization
4+
DOCKER_ORG="corbench"
5+
IMAGE_NAME="cortex-builder"
6+
TAG="latest"
7+
8+
# Enable Docker BuildKit
9+
export DOCKER_BUILDKIT=1
10+
11+
# Create a new builder instance if it doesn't exist
12+
docker buildx inspect multi-arch-builder &>/dev/null || docker buildx create --name multi-arch-builder --use
13+
14+
# Build and push the multi-architecture image
15+
# Note: We're only building for amd64 since that's what your EKS runs on
16+
docker buildx build --platform linux/amd64 \
17+
-t $DOCKER_ORG/$IMAGE_NAME:$TAG \
18+
-f Dockerfile \
19+
--push \
20+
.
21+
22+
echo "AMD64 image built and pushed successfully"

‎tools/cortex-builder/build.sh

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
DIR="/go/src/github.com/cortexproject/cortex"
4+
5+
if [[ -z $PR_NUMBER || -z $VOLUME_DIR || -z $GITHUB_ORG || -z $GITHUB_REPO ]]; then
6+
echo "ERROR:: environment variables not set correctly"
7+
exit 1;
8+
fi
9+
10+
# Clone the repository with a shallow clone
11+
echo ">> Cloning repository $GITHUB_ORG/$GITHUB_REPO (shallow clone)"
12+
if ! git clone --depth 1 https://github.com/$GITHUB_ORG/$GITHUB_REPO.git $DIR; then
13+
echo "ERROR:: Cloning of repo $GITHUB_ORG/$GITHUB_REPO failed"
14+
exit 1;
15+
fi
16+
17+
cd $DIR || exit 1
18+
19+
echo ">> Fetching Pull Request $GITHUB_ORG/$GITHUB_REPO/pull/$PR_NUMBER"
20+
if ! git fetch origin pull/$PR_NUMBER/head:pr-branch; then
21+
echo "ERROR:: Fetching of PR $PR_NUMBER failed"
22+
exit 1;
23+
fi
24+
25+
git checkout pr-branch
26+
27+
echo ">> Building Cortex binaries"
28+
if ! make BUILD_IN_CONTAINER=false cmd/cortex/cortex; then
29+
echo "ERROR:: Building of Cortex binaries failed"
30+
exit 1;
31+
fi
32+
33+
echo ">> Copy files to volume"
34+
# Copy the main Cortex binary
35+
if [ -f "./cmd/cortex/cortex-amd64" ]; then
36+
cp ./cmd/cortex/cortex-amd64 $VOLUME_DIR/cortex
37+
elif [ -f "./cmd/cortex/cortex" ]; then
38+
cp ./cmd/cortex/cortex $VOLUME_DIR/cortex
39+
else
40+
echo "ERROR:: Cortex binary not found"
41+
exit 1;
42+
fi
43+
44+
echo ">> Cortex build completed successfully"

0 commit comments

Comments
 (0)