Added corbench code without manifest files

.gitignore

@@ -30,3 +30,9 @@ go.work.sum
 # Editor/IDE
 # .idea/
 # .vscode/
+
+# Authentication file
+auth_file.yaml
+
+# Infra binary
+corbench/infra/infra

README.md

@@ -1,4 +1,21 @@
 # Corbench
 Benchmarking tool for cortex
 
-Refer to [corbench/README.md](corbench/README.md)
+Refer to [corbench/README.md](corbench/README.md) for documetation on how to deploy the tool, and how the tool works.
+
+## Tool Usage
+
+Create a PR in the Cortex repository, and you are able to run the following commands in the comments of a PR
+
+**Available Commands:**
+-   To start benchmark: `/corbench <cortex-tag>`
+-   To stop benchmark: `/corbench cancel`
+-   To print help: `/corbench help`
+
+**Cortex tag format:** `master-<commit-hash>` (e.g., `master-6b3bd7b`) This will be the version of cortex that your PR will be benchmarked against
+-   MAKE SURE TO PICK AN EXISTING CORTEX TAG FROM BELOW LINK, OR DEPLOYMENT WILL BE STUCK AND YOU WILL HAVE TO CANCEL AND START AGAIN
+-   Check what cortex tags are avaliable to choose from at https://hub.docker.com/r/cortexproject/cortex/tags
+
+**Examples:**
+-   `/corbench master-6b3bd7b`
+-   `/corbench master-9861229`

corbench/Dockerfile

@@ -0,0 +1,16 @@
+FROM --platform=linux/amd64 golang:1.15-alpine
+
+WORKDIR /corbench
+
+RUN apk add git make
+
+# Copy Makefiles and manifests
+# Need 'cd' since ghActions ignores WORKDIR
+# Need 'eval' to prevent bash keywords be run as commands
+COPY ./ ./
+
+RUN echo -e '#!/bin/sh\ncd /corbench\neval "$@"' >/bin/docker_entrypoint
+
+RUN chmod u+x /bin/docker_entrypoint
+
+ENTRYPOINT ["docker_entrypoint"]

corbench/Makefile

@@ -0,0 +1,105 @@
+INFRA_CMD  ?= ./infra/infra
+PROVIDER   ?= eks
+MANIFEST_FILE = c-manifests
+
+cluster_create:
+	${INFRA_CMD} ${PROVIDER} cluster create -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-v AWS_ACCOUNT_ID:$$(echo ${EKS_WORKER_ROLE_ARN} | cut -d':' -f5) \
+		-f ${MANIFEST_FILE}/cluster_${PROVIDER}.yaml
+ifeq (${PROVIDER},eks)
+	./setup-ebs-csi.sh
+endif
+
+cluster_resource_apply:
+	${INFRA_CMD} ${PROVIDER} resource apply -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} -v DOMAIN_NAME:${DOMAIN_NAME} -v RELEASE:${RELEASE} \
+		-v GRAFANA_ADMIN_PASSWORD:${GRAFANA_ADMIN_PASSWORD} \
+		-v SERVICEACCOUNT_CLIENT_EMAIL:${SERVICEACCOUNT_CLIENT_EMAIL} \
+		-v OAUTH_TOKEN="$(printf ${OAUTH_TOKEN} | base64 -w 0)" \
+		-v WH_SECRET="$(printf ${WH_SECRET} | base64 -w 0)" \
+		-v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \
+		-f ${MANIFEST_FILE}/cluster-infra
+	# Patch secrets immediately after deployment with correct values since for some reason 1_secrets isn't working properly
+	export AWS_ACCESS_KEY_ID=$$(grep "accesskeyid:" ${AUTH_FILE} | awk '{print $$2}') && \
+	export AWS_SECRET_ACCESS_KEY=$$(grep "secretaccesskey:" ${AUTH_FILE} | awk '{print $$2}') && \
+	kubectl patch secret oauth-token -p '{"data":{"oauth":"'$$(printf "${OAUTH_TOKEN}" | base64)'"}}' && \
+	kubectl patch secret whsecret -p '{"data":{"whsecret":"'$$(printf "${WH_SECRET}" | base64)'"}}' && \
+	kubectl rollout restart deployment/comment-monitor
+
+cluster_delete:
+	${INFRA_CMD} ${PROVIDER} cluster delete -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-v AWS_ACCOUNT_ID:$$(echo ${EKS_WORKER_ROLE_ARN} | cut -d':' -f5) \
+		-f ${MANIFEST_FILE}/cluster_${PROVIDER}.yaml
+
+
+BENCHMARK_DIRECTORY := $(if $(BENCHMARK_DIRECTORY),$(BENCHMARK_DIRECTORY),c-manifests/benchmarks)
+
+CORBENCH_DIR ?= .
+
+.PHONY: deploy
+deploy: node_create resource_apply
+
+.PHONY: clean
+clean: resource_delete node_delete
+
+# Default PR_NUMBER to 'default' if not set to avoid invalid nodegroup names
+PR_NUMBER ?= default
+
+node_create:
+	${INFRA_CMD} ${PROVIDER} nodes create -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/nodes_${PROVIDER}.yaml
+
+resource_apply:
+	$(INFRA_CMD) ${PROVIDER} resource apply -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} \
+		-v PR_NUMBER:${PR_NUMBER} -v RELEASE:${RELEASE} -v DOMAIN_NAME:${DOMAIN_NAME} \
+		-v GITHUB_ORG:${GITHUB_ORG} -v GITHUB_REPO:${GITHUB_REPO} \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/remote-write
+
+# Required because namespace and cluster-role are not part of the created nodes
+resource_delete:
+	$(INFRA_CMD) ${PROVIDER} resource delete -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/remote-write/1c_cluster-role-binding.yaml \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/remote-write/1a_namespace.yaml
+
+node_delete:
+	$(INFRA_CMD) ${PROVIDER} nodes delete -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/nodes_${PROVIDER}.yaml
+
+all_nodes_running:
+	$(INFRA_CMD) ${PROVIDER} nodes check-running -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/nodes_${PROVIDER}.yaml
+
+all_nodes_deleted:
+	$(INFRA_CMD) ${PROVIDER} nodes check-deleted -a ${AUTH_FILE} \
+		-v ZONE:${ZONE} -v GKE_PROJECT_ID:${GKE_PROJECT_ID} \
+		-v EKS_WORKER_ROLE_ARN:${EKS_WORKER_ROLE_ARN} -v EKS_CLUSTER_ROLE_ARN:${EKS_CLUSTER_ROLE_ARN} \
+		-v EKS_SUBNET_IDS:${EKS_SUBNET_IDS} -v SEPARATOR:${SEPARATOR} \
+		-v CLUSTER_NAME:${CLUSTER_NAME} -v PR_NUMBER:${PR_NUMBER} \
+		-f ${CORBENCH_DIR}/${BENCHMARK_DIRECTORY}/nodes_${PROVIDER}.yaml

corbench/README.md

@@ -0,0 +1,16 @@
+# Automated Cortex E2E Testing and Benchmarking
+
+## Corbench Setup
+
+To deploy this tool, follow the steps here: [Getting Started](docs/deployment.md)
+
+# Archetecture
+
+![Cortex Benchmarking Archetecture](docs/Blank%20diagram%20(2).png)
+
+1. START: User opens a Github PR in the cortex repository and sends a /corbench PR comment to start a benchmark test. Github then makes a POST request with the command as the payload to the Comment Monitor Webhook Server (step 2). 
+2. The Comment Monitor Webhook Server monitors for /corbench calls from Cortex github PRs. Upon recieving a POST request from a Github PR, the server verifies the validity of the command (i.e. does the command exist?). The server then initiates a dispatch event to the PR (step 3) as well as posts an update comment indicating the status of the tests, links to grafana dashboards, etc, or a warning message if the command was used incorrectly/with wrong syntax.
+3. The PR runs the dispatch event using the docker image of the Corbench repo code to initiate benchmarks (step 4). 
+4. Within the Corbench docker image, two versions of Cortex that will be benchmarked are built & deployed in separate Kubernetes nodes to EKS. After this, we can start to deploy the metrics pushers (Avalanche). Avalanche instances will be deployed to EKS and will simulate prometheus instances pushing load to Cortex’s remote write (or other micro-service specific) endpoints with mock data.
+6. While the Metrics Pushers push load into the Cortex instances, prometheus will continuously scrape metrics from the Cortex instances and the kubernetes nodes they run on.
+7. Grafana displays collected benchmark metrics & github notifier notifies the original pr of results & other info