From d2454dfda72e21e7cc52deaca8379ee7bf817875 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 15 Jul 2025 14:58:21 -0400 Subject: [PATCH 01/35] initialize perfcomp module and task --- .evergreen/config.yml | 11 +++++ Taskfile.yml | 35 +++++++------- etc/perf-pr-comment.sh | 7 +++ go.work | 1 + internal/cmd/perfcomp/energystatistics.go | 7 +++ .../cmd/perfcomp/energystatistics_test.go | 7 +++ internal/cmd/perfcomp/go.mod | 22 +++++++++ internal/cmd/perfcomp/go.sum | 48 +++++++++++++++++++ internal/cmd/perfcomp/main.go | 7 +++ 9 files changed, 128 insertions(+), 17 deletions(-) create mode 100755 etc/perf-pr-comment.sh create mode 100644 internal/cmd/perfcomp/energystatistics.go create mode 100644 internal/cmd/perfcomp/energystatistics_test.go create mode 100644 internal/cmd/perfcomp/go.mod create mode 100644 internal/cmd/perfcomp/go.sum create mode 100644 internal/cmd/perfcomp/main.go diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 05832906d1..bd63c1a5f4 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -285,6 +285,17 @@ functions: echo "Response Body: $response_body" echo "HTTP Status: $http_status" + send-perf-pr-comment: + - command: subprocess.exec + type: test + params: + binary: bash + env: + COMMIT: "${github_commit}" + VERSION_ID: ${version_id} + include_expansions_in_env: [perf_uri_private_endpoint] + args: [*task-runner, perf-pr-comment] + run-enterprise-auth-tests: - command: ec2.assume_role params: diff --git a/Taskfile.yml b/Taskfile.yml index 3473cb4981..3d8d69c1c5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,16 +1,15 @@ # See https://taskfile.dev/usage/ -version: '3' +version: "3" env: TEST_TIMEOUT: 1800 LONG_TEST_TIMEOUT: 3600 -dotenv: ['.test.env'] +dotenv: [".test.env"] tasks: - ### Utility tasks. ### - default: + default: deps: [build, check-license, check-fmt, check-modules, lint, test-short] add-license: bash etc/check_license.sh -a @@ -36,7 +35,7 @@ tasks: build-aws-ecs-test: go test -c ./internal/test/aws -o aws.testbin - cross-compile: + cross-compile: - GOOS=linux GOARCH=386 go build ./... - GOOS=linux GOARCH=arm go build ./... - GOOS=linux GOARCH=arm64 go build ./... @@ -44,7 +43,7 @@ tasks: - GOOS=linux GOARCH=ppc64le go build ./... - GOOS=linux GOARCH=s390x go build ./... - check-fmt: + check-fmt: deps: [install-lll] cmds: - bash etc/check_fmt.sh @@ -57,9 +56,9 @@ tasks: api-report: bash etc/api_report.sh - install-libmongocrypt: + install-libmongocrypt: cmds: [bash etc/install-libmongocrypt.sh] - status: + status: - test -d install || test -d /cygdrive/c/libmongocrypt/bin run-docker: bash etc/run_docker.sh @@ -70,11 +69,13 @@ tasks: pr-task: bash etc/pr-task.sh + perf-pr-comment: bash etc/perf-pr-comment.sh + # Lint with various GOOS and GOARCH tasks to catch static analysis failures that may only affect # specific operating systems or architectures. For example, staticcheck will only check for 64-bit # alignment of atomically accessed variables on 32-bit architectures (see # https://staticcheck.io/docs/checks#SA1027) - lint: + lint: cmds: - GOOS=linux GOARCH=386 etc/golangci-lint.sh - GOOS=linux GOARCH=arm etc/golangci-lint.sh @@ -100,7 +101,7 @@ tasks: test-oidc-remote: bash etc/run-oidc-remote-test.sh - test-atlas-connect: + test-atlas-connect: - go test -v -run ^TestAtlas$ go.mongodb.org/mongo-driver/v2/internal/cmd/testatlas -args "$ATLAS_REPL" "$ATLAS_SHRD" "$ATLAS_FREE" "$ATLAS_TLS11" "$ATLAS_TLS12" "$ATLAS_SERVERLESS" "$ATLAS_SRV_REPL" "$ATLAS_SRV_SHRD" "$ATLAS_SRV_FREE" "$ATLAS_SRV_TLS11" "$ATLAS_SRV_TLS12" "$ATLAS_SRV_SERVERLESS" >> test.suite test-awskms: bash etc/run-awskms-test.sh @@ -113,9 +114,9 @@ tasks: ### Local FaaS tasks. ### build-faas-awslambda: - requires: + requires: vars: [MONGODB_URI] - cmds: + cmds: - make -c internal/cmd/faas/awslambda ### Evergreen specific tasks. ### @@ -130,7 +131,7 @@ tasks: - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration/unified -run TestUnifiedSpec/atlas-data-lake-testing >> spec_test.suite - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestAtlasDataLake >> spec_test.suite - evg-test-enterprise-auth: + evg-test-enterprise-auth: - go run -tags gssapi ./internal/cmd/testentauth/main.go evg-test-oidc-auth: @@ -184,15 +185,15 @@ tasks: ### Benchmark specific tasks and support. ### benchmark: deps: [perf-files] - cmds: + cmds: - go test ${BUILD_TAGS} -benchmem -bench=. ./benchmark | test benchmark.suite - driver-benchmark: - cmds: + driver-benchmark: + cmds: - go test ./internal/cmd/benchmark -v --fullRun | tee perf.suite ### Internal tasks. ### - install-lll: + install-lll: internal: true cmds: - go install github.com/walle/lll/...@latest diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh new file mode 100755 index 0000000000..eb41326322 --- /dev/null +++ b/etc/perf-pr-comment.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# perf-pr-comment +# Generates a report of Go Driver perf changes for the current branch. + +set -eux + +go run ./internal/cmd/perfcomp/main.go ./internal/cmd/perfcomp/energystatistics.go diff --git a/go.work b/go.work index 23ad2ff8a7..9f345c684c 100644 --- a/go.work +++ b/go.work @@ -7,6 +7,7 @@ use ( ./examples/_logger/zerolog ./internal/cmd/benchmark ./internal/cmd/compilecheck + ./internal/cmd/perfcomp ./internal/cmd/faas/awslambda/mongodb ./internal/test/goleak ) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go new file mode 100644 index 0000000000..da67b57a0c --- /dev/null +++ b/internal/cmd/perfcomp/energystatistics.go @@ -0,0 +1,7 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go new file mode 100644 index 0000000000..da67b57a0c --- /dev/null +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -0,0 +1,7 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod new file mode 100644 index 0000000000..6446bed897 --- /dev/null +++ b/internal/cmd/perfcomp/go.mod @@ -0,0 +1,22 @@ +module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp + +go 1.23 + +replace go.mongodb.org/mongo-driver/v2 => ../../../ + +require ( + go.mongodb.org/mongo-driver/v2 v2.2.2 + gonum.org/v1/gonum v0.16.0 +) + +require ( + github.com/golang/snappy v1.0.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + golang.org/x/crypto v0.33.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/text v0.23.0 // indirect +) diff --git a/internal/cmd/perfcomp/go.sum b/internal/cmd/perfcomp/go.sum new file mode 100644 index 0000000000..49f669457a --- /dev/null +++ b/internal/cmd/perfcomp/go.sum @@ -0,0 +1,48 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go new file mode 100644 index 0000000000..da67b57a0c --- /dev/null +++ b/internal/cmd/perfcomp/main.go @@ -0,0 +1,7 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main From e42e875140e575fd264f593a742f6722ce1bf14c Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 15 Jul 2025 15:05:48 -0400 Subject: [PATCH 02/35] set up main file --- internal/cmd/perfcomp/main.go | 160 ++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index da67b57a0c..eb8c0495e2 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -5,3 +5,163 @@ // a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "strings" + "time" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +type RawData struct { + Info struct { + Project string `bson:"project"` + Version string `bson:"version"` + Variant string `bson:"variant"` + Order int64 `bson:"order"` + TaskName string `bson:"task_name"` + TaskID string `bson:"task_id"` + Execution int64 `bson:"execution"` + Mainline bool `bson:"mainline"` + OverrideInfo struct { + OverrideMainline bool `bson:"override_mainline"` + BaseOrder interface{} `bson:"base_order"` + Reason interface{} `bson:"reason"` + User interface{} `bson:"user"` + } + TestName string `bson:"test_name"` + Args map[string]interface{} `bson:"args"` + } + CreatedAt interface{} `bson:"created_at"` + CompletedAt interface{} `bson:"completed_at"` + Rollups struct { + Stats []struct { + Name string `bson:"name"` + Val float64 `bson:"val"` + Metadata interface{} `bson:"metadata"` + } + } + FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` +} + +func main() { + // Connect to analytics node + uri := os.Getenv("perf_uri_private_endpoint") + if uri == "" { + log.Panic("perf_uri_private_endpoint env variable is not set") + } + + client, err1 := mongo.Connect(options.Client().ApplyURI(uri)) + if err1 != nil { + log.Panicf("Error connecting client: %v", err1) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err2 := client.Ping(ctx, nil) + if err2 != nil { + log.Panicf("Error pinging MongoDB Analytics: %v", err2) + } + fmt.Println("Successfully connected to MongoDB Analytics node.") + + coll := client.Database("expanded_metrics").Collection("raw_results") + version := os.Getenv("VERSION_ID") + if version == "" { + log.Panic("could not retrieve version") + } + + // Get and pre-process raw data + patchRawData, err3 := findRawData(version, coll) + if err3 != nil { + log.Panicf("Error getting raw data: %v", err3) + } + + mainlineCommits, err4 := parseMainelineCommits(patchRawData) + if err4 != nil { + log.Panicf("Error parsing commits: %v", err4) + } + + mainlineVersion := "mongo_go_driver_" + mainlineCommits[0] + mainlineRawData, err5 := findRawData(mainlineVersion, coll) + if err5 != nil { + log.Panicf("Could not retrieve mainline raw data") + } + + if len(mainlineRawData) != len(patchRawData) { + log.Panicf("Path and mainline data length do not match.") + } + + // TODO: Calculate energy statistics + + // Disconnect client + err0 := client.Disconnect(context.Background()) + if err0 != nil { + log.Panicf("Failed to disconnect client: %v", err0) + } + +} + +func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { + filter := bson.D{ + {"info.project", "mongo-go-driver"}, + {"info.version", version}, + {"info.variant", "perf"}, + {"info.task_name", "perf"}, + } + + findOptions := options.Find() + + findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cursor, err := coll.Find(findCtx, filter, findOptions) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + fmt.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) + + var rawData []RawData + for cursor.Next(findCtx) { + var rd RawData + if err := cursor.Decode(&rd); err != nil { + return nil, err + } + rawData = append(rawData, rd) + } + + if err := cursor.Err(); err != nil { + return nil, err + } + + return rawData, nil +} + +func parseMainelineCommits(rawData []RawData) ([]string, error) { + commits := make([]string, 0, len(rawData)) + for i, rd := range rawData { + taskID := rd.Info.TaskID + pieces := strings.Split(taskID, "_") // Format: mongo_go_driver_perf_perf_patch___ + for j, p := range pieces { + if p == "patch" { + if len(pieces) < j+2 { + return nil, errors.New("task ID doesn't hold commit SHA") + } + commits = append(commits, pieces[j+1]) + break + } + } + if len(commits) < i+1 { // didn't find SHA in task_ID + return nil, errors.New("task ID doesn't hold commit SHA") + } + } + return commits, nil +} From 54d428942f48e910f8229ab1389402161949d15b Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 15 Jul 2025 15:08:14 -0400 Subject: [PATCH 03/35] cleanup --- Taskfile.yml | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 3d8d69c1c5..8b2c7df0e3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,15 +1,16 @@ # See https://taskfile.dev/usage/ -version: "3" +version: '3' env: TEST_TIMEOUT: 1800 LONG_TEST_TIMEOUT: 3600 -dotenv: [".test.env"] +dotenv: ['.test.env'] tasks: + ### Utility tasks. ### - default: + default: deps: [build, check-license, check-fmt, check-modules, lint, test-short] add-license: bash etc/check_license.sh -a @@ -35,7 +36,7 @@ tasks: build-aws-ecs-test: go test -c ./internal/test/aws -o aws.testbin - cross-compile: + cross-compile: - GOOS=linux GOARCH=386 go build ./... - GOOS=linux GOARCH=arm go build ./... - GOOS=linux GOARCH=arm64 go build ./... @@ -43,7 +44,7 @@ tasks: - GOOS=linux GOARCH=ppc64le go build ./... - GOOS=linux GOARCH=s390x go build ./... - check-fmt: + check-fmt: deps: [install-lll] cmds: - bash etc/check_fmt.sh @@ -56,9 +57,9 @@ tasks: api-report: bash etc/api_report.sh - install-libmongocrypt: + install-libmongocrypt: cmds: [bash etc/install-libmongocrypt.sh] - status: + status: - test -d install || test -d /cygdrive/c/libmongocrypt/bin run-docker: bash etc/run_docker.sh @@ -75,7 +76,7 @@ tasks: # specific operating systems or architectures. For example, staticcheck will only check for 64-bit # alignment of atomically accessed variables on 32-bit architectures (see # https://staticcheck.io/docs/checks#SA1027) - lint: + lint: cmds: - GOOS=linux GOARCH=386 etc/golangci-lint.sh - GOOS=linux GOARCH=arm etc/golangci-lint.sh @@ -101,7 +102,7 @@ tasks: test-oidc-remote: bash etc/run-oidc-remote-test.sh - test-atlas-connect: + test-atlas-connect: - go test -v -run ^TestAtlas$ go.mongodb.org/mongo-driver/v2/internal/cmd/testatlas -args "$ATLAS_REPL" "$ATLAS_SHRD" "$ATLAS_FREE" "$ATLAS_TLS11" "$ATLAS_TLS12" "$ATLAS_SERVERLESS" "$ATLAS_SRV_REPL" "$ATLAS_SRV_SHRD" "$ATLAS_SRV_FREE" "$ATLAS_SRV_TLS11" "$ATLAS_SRV_TLS12" "$ATLAS_SRV_SERVERLESS" >> test.suite test-awskms: bash etc/run-awskms-test.sh @@ -114,9 +115,9 @@ tasks: ### Local FaaS tasks. ### build-faas-awslambda: - requires: + requires: vars: [MONGODB_URI] - cmds: + cmds: - make -c internal/cmd/faas/awslambda ### Evergreen specific tasks. ### @@ -131,7 +132,7 @@ tasks: - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration/unified -run TestUnifiedSpec/atlas-data-lake-testing >> spec_test.suite - ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./internal/integration -run TestAtlasDataLake >> spec_test.suite - evg-test-enterprise-auth: + evg-test-enterprise-auth: - go run -tags gssapi ./internal/cmd/testentauth/main.go evg-test-oidc-auth: @@ -185,15 +186,15 @@ tasks: ### Benchmark specific tasks and support. ### benchmark: deps: [perf-files] - cmds: + cmds: - go test ${BUILD_TAGS} -benchmem -bench=. ./benchmark | test benchmark.suite - driver-benchmark: - cmds: + driver-benchmark: + cmds: - go test ./internal/cmd/benchmark -v --fullRun | tee perf.suite ### Internal tasks. ### - install-lll: + install-lll: internal: true cmds: - go install github.com/walle/lll/...@latest From 57f2cba9d8ed09433887fe4a8978a5e114b2ad8a Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 15 Jul 2025 17:13:00 -0400 Subject: [PATCH 04/35] energy stats algorithm and test --- internal/cmd/perfcomp/energystatistics.go | 39 ++++++++++++ .../cmd/perfcomp/energystatistics_test.go | 62 +++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index da67b57a0c..ec074c6416 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -5,3 +5,42 @@ // a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 package main + +import ( + "math" + + "gonum.org/v1/gonum/mat" +) + +// Given two matrices, this function returns +// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) +func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { + n, _ := x.Dims() + m, _ := y.Dims() + nf := float64(n) + mf := float64(m) + A := getDistance(x, y) / (nf * mf) // E|X-Y| + B := getDistance(x, x) / (nf * nf) // E|X-X'| + C := getDistance(y, y) / (mf * mf) // E|Y-Y'| + + E := 2*A - B - C // D^2(F_x, F_y) + T := ((nf * mf) / (nf + mf)) * E + H := E / (2 * A) + return E, T, H +} + +// Given two vectors (expected 1 col), +// this function returns the sum of distances between each pair. +func getDistance(x, y *mat.Dense) float64 { + xrows, _ := x.Dims() + yrows, _ := y.Dims() + + var sum float64 + + for i := 0; i < xrows; i++ { + for j := 0; j < yrows; j++ { + sum += math.Sqrt(math.Pow((x.At(i, 0) - y.At(j, 0)), 2)) + } + } + return sum +} diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index da67b57a0c..3960a32825 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -5,3 +5,65 @@ // a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gonum.org/v1/gonum/mat" +) + +func createTestVectors(start1 int, stop1 int, step1 int, start2 int, stop2 int, step2 int) (*mat.Dense, *mat.Dense) { + xData := []float64{} + i := start1 + for i < stop1 { + xData = append(xData, float64(i)) + i += step1 + } + + yData := []float64{} + j := start2 + for j < stop2 { + yData = append(yData, float64(j)) + j += step2 + } + + x := mat.NewDense(len(xData), 1, xData) + y := mat.NewDense(len(yData), 1, yData) + + return x, y +} + +func TestEnergyStatistics(t *testing.T) { + + t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { + x, y := createTestVectors(1, 100, 1, 1, 105, 1) + e, tstat, h := getEnergyStatistics(x, y) + + del := 1e-3 + + assert.InDelta(t, 0.160, e, del) // |0.160 - e| < 1/100 + assert.InDelta(t, 8.136, tstat, del) + assert.InDelta(t, 0.002, h, del) + }) + + t.Run("different distributions should have large e,t,h values", func(t *testing.T) { + x, y := createTestVectors(1, 100, 1, 10000, 13000, 14) + e, tstat, h := getEnergyStatistics(x, y) + del := 1e-3 + + assert.InDelta(t, 21859.691, e, del) + assert.InDelta(t, 1481794.709, tstat, del) + assert.InDelta(t, 0.954, h, del) + }) + + t.Run("uni-variate distributions", func(t *testing.T) { + x, y := createTestVectors(1, 300, 1, 1000, 5000, 10) + e, tstat, h := getEnergyStatistics(x, y) + del := 1e-3 + + assert.InDelta(t, 4257.009, e, del) + assert.InDelta(t, 1481794.709, tstat, del) + assert.InDelta(t, 0.954, h, del) + }) +} From 0966d614d2dd3842cc6b678aa365624f56fcd898 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 16 Jul 2025 12:24:08 -0400 Subject: [PATCH 05/35] apply raw results to energy stats, needs debug --- .evergreen/config.yml | 1 + internal/cmd/perfcomp/energystatistics.go | 2 +- .../cmd/perfcomp/energystatistics_test.go | 10 +-- internal/cmd/perfcomp/main.go | 73 ++++++++++++++++++- 4 files changed, 79 insertions(+), 7 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index bd63c1a5f4..e1ed36cfff 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -687,6 +687,7 @@ tasks: binary: bash args: [*task-runner, driver-benchmark] - func: send-perf-data + - func: send-perf-pr-comment - name: test-standalone-noauth-nossl tags: ["test", "standalone"] diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index ec074c6416..0ab16d674f 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -14,7 +14,7 @@ import ( // Given two matrices, this function returns // (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) -func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { +func GetEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { n, _ := x.Dims() m, _ := y.Dims() nf := float64(n) diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index 3960a32825..dc75adda02 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -38,7 +38,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 1, 105, 1) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h := GetEnergyStatistics(x, y) del := 1e-3 @@ -49,7 +49,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("different distributions should have large e,t,h values", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 10000, 13000, 14) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h := GetEnergyStatistics(x, y) del := 1e-3 assert.InDelta(t, 21859.691, e, del) @@ -59,11 +59,11 @@ func TestEnergyStatistics(t *testing.T) { t.Run("uni-variate distributions", func(t *testing.T) { x, y := createTestVectors(1, 300, 1, 1000, 5000, 10) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h := GetEnergyStatistics(x, y) del := 1e-3 assert.InDelta(t, 4257.009, e, del) - assert.InDelta(t, 1481794.709, tstat, del) - assert.InDelta(t, 0.954, h, del) + assert.InDelta(t, 728381.015, tstat, del) + assert.InDelta(t, 0.748, h, del) }) } diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index eb8c0495e2..ddc6f23a8f 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -11,13 +11,16 @@ import ( "errors" "fmt" "log" + "math" "os" + "sort" "strings" "time" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" + "gonum.org/v1/gonum/mat" ) type RawData struct { @@ -51,6 +54,15 @@ type RawData struct { FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` } +type EnergyStats struct { + Benchmark string + PatchVersion string + MainlineVersion string + E float64 + T float64 + H float64 +} + func main() { // Connect to analytics node uri := os.Getenv("perf_uri_private_endpoint") @@ -98,7 +110,12 @@ func main() { log.Panicf("Path and mainline data length do not match.") } - // TODO: Calculate energy statistics + // Calculate energy statistics + energyStats, err := getEnergyStatsForAllBenchMarks(patchRawData, mainlineRawData) + if err != nil { + log.Panicf("Error calculating energy stats: %v", err) + } + fmt.Printf("Successfully retrieved %d energy stats.\n", len(energyStats)) // Disconnect client err0 := client.Disconnect(context.Background()) @@ -165,3 +182,57 @@ func parseMainelineCommits(rawData []RawData) ([]string, error) { } return commits, nil } + +func getEnergyStatsForOneBenchmark(xRaw RawData, yRaw RawData) (*EnergyStats, error) { + + var x []float64 + var y []float64 + for _, stat := range xRaw.Rollups.Stats { + x = append(x, stat.Val) + } + for _, stat := range yRaw.Rollups.Stats { + y = append(y, stat.Val) + } + + for i := range (int)(math.Min((float64)(len(xRaw.Rollups.Stats)), float64(len(yRaw.Rollups.Stats)))) { + if xRaw.Rollups.Stats[i].Name != yRaw.Rollups.Stats[i].Name { + return nil, errors.New("measurements do not match") + } + } + + e, t, h := GetEnergyStatistics(mat.NewDense(len(x), 1, x), mat.NewDense(len(y), 1, y)) + return &EnergyStats{ + Benchmark: xRaw.Info.TestName, + PatchVersion: xRaw.Info.Version, + MainlineVersion: yRaw.Info.Version, + E: e, + T: t, + H: h, + }, nil +} + +func getEnergyStatsForAllBenchMarks(patchRawData []RawData, mainlineRawData []RawData) ([]*EnergyStats, error) { + + sort.Slice(patchRawData, func(i, j int) bool { + return patchRawData[i].Info.TestName < patchRawData[j].Info.TestName + }) + sort.Slice(mainlineRawData, func(i, j int) bool { + return mainlineRawData[i].Info.TestName < mainlineRawData[j].Info.TestName + }) + + var energyStats []*EnergyStats + for i := range patchRawData { + if testname := patchRawData[i].Info.TestName; testname != mainlineRawData[i].Info.TestName { + return nil, errors.New("tests do not match") + } + + es, err := getEnergyStatsForOneBenchmark(patchRawData[i], mainlineRawData[i]) + if err != nil { + return nil, err + } + energyStats = append(energyStats, es) + + fmt.Printf("%s | H-score: %.4f\n", es.Benchmark, es.H) + } + return energyStats, nil +} From bd863864ec2b28ba9575e74ebefd199db529de90 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 18 Jul 2025 14:53:07 -0400 Subject: [PATCH 06/35] get stable regions and calc for hscore --- internal/cmd/perfcomp/energystatistics.go | 29 +++- .../cmd/perfcomp/energystatistics_test.go | 13 ++ internal/cmd/perfcomp/main.go | 164 ++++++++---------- 3 files changed, 115 insertions(+), 91 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index 0ab16d674f..e37b28c127 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -19,13 +19,34 @@ func GetEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { m, _ := y.Dims() nf := float64(n) mf := float64(m) - A := getDistance(x, y) / (nf * mf) // E|X-Y| - B := getDistance(x, x) / (nf * nf) // E|X-X'| - C := getDistance(y, y) / (mf * mf) // E|Y-Y'| + + var A float64 // E|X-Y| + if nf > 0 && mf > 0 { + A = getDistance(x, y) / (nf * mf) + } else { + A = 0 + } + var B float64 // E|X-X'| + if nf > 0 { + B = getDistance(x, x) / (nf * nf) + } else { + B = 0 + } + var C float64 // E|Y-Y'| + if mf > 0 { + C = getDistance(y, y) / (mf * mf) + } else { + C = 0 + } E := 2*A - B - C // D^2(F_x, F_y) T := ((nf * mf) / (nf + mf)) * E - H := E / (2 * A) + var H float64 + if A > 0 { + H = E / (2 * A) + } else { + H = 0 + } return E, T, H } diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index dc75adda02..48afbdc9fd 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -66,4 +66,17 @@ func TestEnergyStatistics(t *testing.T) { assert.InDelta(t, 728381.015, tstat, del) assert.InDelta(t, 0.748, h, del) }) + + t.Run("equal distributions should have all 0 values", func(t *testing.T) { + x := mat.NewDense(10, 1, []float64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) + y := mat.NewDense(1, 1, []float64{1}) + + e, tstat, h := GetEnergyStatistics(x, y) + del := 1e-3 + + assert.InDelta(t, 0.0, e, del) + assert.InDelta(t, 0.0, tstat, del) + assert.InDelta(t, 0.0, h, del) + }) + } diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index ddc6f23a8f..3076856f1e 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -11,10 +11,7 @@ import ( "errors" "fmt" "log" - "math" "os" - "sort" - "strings" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -55,12 +52,14 @@ type RawData struct { } type EnergyStats struct { - Benchmark string - PatchVersion string - MainlineVersion string - E float64 - T float64 - H float64 + Benchmark string + Measurement string + PatchVersion string + StableRegionValues []float64 + PatchValues []float64 + E float64 + T float64 + H float64 } func main() { @@ -83,39 +82,25 @@ func main() { } fmt.Println("Successfully connected to MongoDB Analytics node.") - coll := client.Database("expanded_metrics").Collection("raw_results") + db := client.Database("expanded_metrics") version := os.Getenv("VERSION_ID") if version == "" { log.Panic("could not retrieve version") } - // Get and pre-process raw data - patchRawData, err3 := findRawData(version, coll) + // Get raw data, most recent stable region, and calculate energy stats + patchRawData, err3 := findRawData(version, db.Collection("raw_results")) if err3 != nil { log.Panicf("Error getting raw data: %v", err3) } - mainlineCommits, err4 := parseMainelineCommits(patchRawData) + allEnergyStats, err4 := getEnergyStatsForAllBenchMarks(patchRawData, db.Collection("stable_regions")) if err4 != nil { - log.Panicf("Error parsing commits: %v", err4) + log.Panicf("Error getting raw data: %v", err4) } - - mainlineVersion := "mongo_go_driver_" + mainlineCommits[0] - mainlineRawData, err5 := findRawData(mainlineVersion, coll) - if err5 != nil { - log.Panicf("Could not retrieve mainline raw data") - } - - if len(mainlineRawData) != len(patchRawData) { - log.Panicf("Path and mainline data length do not match.") - } - - // Calculate energy statistics - energyStats, err := getEnergyStatsForAllBenchMarks(patchRawData, mainlineRawData) - if err != nil { - log.Panicf("Error calculating energy stats: %v", err) + for _, es := range allEnergyStats { + fmt.Printf("%s | %s | E: %f | T: %f | H: %f\n", es.Benchmark, es.Measurement, es.E, es.T, es.H) } - fmt.Printf("Successfully retrieved %d energy stats.\n", len(energyStats)) // Disconnect client err0 := client.Disconnect(context.Background()) @@ -162,77 +147,82 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { return rawData, nil } -func parseMainelineCommits(rawData []RawData) ([]string, error) { - commits := make([]string, 0, len(rawData)) - for i, rd := range rawData { - taskID := rd.Info.TaskID - pieces := strings.Split(taskID, "_") // Format: mongo_go_driver_perf_perf_patch___ - for j, p := range pieces { - if p == "patch" { - if len(pieces) < j+2 { - return nil, errors.New("task ID doesn't hold commit SHA") - } - commits = append(commits, pieces[j+1]) - break - } - } - if len(commits) < i+1 { // didn't find SHA in task_ID - return nil, errors.New("task ID doesn't hold commit SHA") - } +func findLastStableRegion(testname string, measurement string, coll *mongo.Collection) ([]float64, error) { + filter := bson.D{ + {"time_series_info.project", "mongo-go-driver"}, + {"time_series_info.variant", "perf"}, + {"time_series_info.task", "perf"}, + {"time_series_info.test", testname}, + {"time_series_info.measurement", measurement}, + {"last", true}, + {"contexts", []string{"GoDriver perf (h-score)"}}, } - return commits, nil -} + projection := bson.D{ + {"values", 1}, + } + findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}).SetProjection(projection) -func getEnergyStatsForOneBenchmark(xRaw RawData, yRaw RawData) (*EnergyStats, error) { + findCtx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() - var x []float64 - var y []float64 - for _, stat := range xRaw.Rollups.Stats { - x = append(x, stat.Val) - } - for _, stat := range yRaw.Rollups.Stats { - y = append(y, stat.Val) + var result bson.M + err := coll.FindOne(findCtx, filter, findOptions).Decode(&result) + if err != nil { + return nil, err } - for i := range (int)(math.Min((float64)(len(xRaw.Rollups.Stats)), float64(len(yRaw.Rollups.Stats)))) { - if xRaw.Rollups.Stats[i].Name != yRaw.Rollups.Stats[i].Name { - return nil, errors.New("measurements do not match") + valuesSlice, ok := result["values"].(bson.A) + if !ok { + return nil, errors.New("values is not of type bson.A") + } + var values []float64 + for _, v := range valuesSlice { + number, ok := v.(float64) + if !ok { + return nil, errors.New("value is not float64") } + values = append(values, number) } - - e, t, h := GetEnergyStatistics(mat.NewDense(len(x), 1, x), mat.NewDense(len(y), 1, y)) - return &EnergyStats{ - Benchmark: xRaw.Info.TestName, - PatchVersion: xRaw.Info.Version, - MainlineVersion: yRaw.Info.Version, - E: e, - T: t, - H: h, - }, nil + return values, nil } -func getEnergyStatsForAllBenchMarks(patchRawData []RawData, mainlineRawData []RawData) ([]*EnergyStats, error) { - - sort.Slice(patchRawData, func(i, j int) bool { - return patchRawData[i].Info.TestName < patchRawData[j].Info.TestName - }) - sort.Slice(mainlineRawData, func(i, j int) bool { - return mainlineRawData[i].Info.TestName < mainlineRawData[j].Info.TestName - }) - +// For a specific test and measurement +func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { + testname := rd.Info.TestName var energyStats []*EnergyStats - for i := range patchRawData { - if testname := patchRawData[i].Info.TestName; testname != mainlineRawData[i].Info.TestName { - return nil, errors.New("tests do not match") - } - es, err := getEnergyStatsForOneBenchmark(patchRawData[i], mainlineRawData[i]) + for i := range rd.Rollups.Stats { + measurement := rd.Rollups.Stats[i].Name + patchVal := []float64{rd.Rollups.Stats[i].Val} + stableRegionVals, err := findLastStableRegion(testname, measurement, coll) if err != nil { return nil, err } - energyStats = append(energyStats, es) - - fmt.Printf("%s | H-score: %.4f\n", es.Benchmark, es.H) + e, t, h := GetEnergyStatistics(mat.NewDense(len(stableRegionVals), 1, stableRegionVals), mat.NewDense(1, 1, patchVal)) + es := EnergyStats{ + Benchmark: testname, + Measurement: measurement, + PatchVersion: rd.Info.Version, + StableRegionValues: stableRegionVals, + PatchValues: patchVal, + E: e, + T: t, + H: h, + } + energyStats = append(energyStats, &es) } + return energyStats, nil } + +func getEnergyStatsForAllBenchMarks(patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { + var allEnergyStats []*EnergyStats + for _, rd := range patchRawData { + energyStats, err := getEnergyStatsForOneBenchmark(rd, coll) + if err != nil { + return nil, err + } + allEnergyStats = append(allEnergyStats, energyStats...) + } + return allEnergyStats, nil +} From 1ec634f0430ad6d28f5c6c2d3111caf30e504d61 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 18 Jul 2025 16:50:28 -0400 Subject: [PATCH 07/35] formatted comment output --- .evergreen/config.yml | 1 - internal/cmd/perfcomp/main.go | 116 ++++++++++++++++++++++------------ 2 files changed, 74 insertions(+), 43 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index e1ed36cfff..6ae40c46d3 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -291,7 +291,6 @@ functions: params: binary: bash env: - COMMIT: "${github_commit}" VERSION_ID: ${version_id} include_expansions_in_env: [perf_uri_private_endpoint] args: [*task-runner, perf-pr-comment] diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 3076856f1e..09017f34f5 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -8,10 +8,10 @@ package main import ( "context" - "errors" "fmt" "log" "os" + "strings" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -51,15 +51,40 @@ type RawData struct { FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` } +type StableRegion struct { + TimeSeriesInfo struct { + Project string `bson:"project"` + Variant string `bson:"variant"` + Task string `bson:"task"` + Test string `bson:"test"` + Measurement string `bson:"measurement"` + Args map[string]interface{} `bson:"args"` + } + Start interface{} `bson:"start"` + End interface{} `bson:"end"` + Values []float64 `bson:"values"` + StartOrder int64 `bson:"start_order"` + EndOrder int64 `bson:"end_order"` + Mean float64 `bson:"mean"` + Std float64 `bson:"std"` + Median float64 `bson:"median"` + Max float64 `bson:"max"` + Min float64 `bson:"min"` + CoefficientOfVariation float64 `bson:"coefficient_of_variation"` + LastSuccessfulUpdate interface{} `bson:"last_successful_update"` + Last bool `bson:"last"` + Contexts []interface{} `bson:"contexts"` +} + type EnergyStats struct { - Benchmark string - Measurement string - PatchVersion string - StableRegionValues []float64 - PatchValues []float64 - E float64 - T float64 - H float64 + Benchmark string + Measurement string + PatchVersion string + StableRegion StableRegion + PatchValues []float64 + E float64 + T float64 + H float64 } func main() { @@ -98,9 +123,7 @@ func main() { if err4 != nil { log.Panicf("Error getting raw data: %v", err4) } - for _, es := range allEnergyStats { - fmt.Printf("%s | %s | E: %f | T: %f | H: %f\n", es.Benchmark, es.Measurement, es.E, es.T, es.H) - } + fmt.Println(generatePRComment(allEnergyStats, version)) // Disconnect client err0 := client.Disconnect(context.Background()) @@ -147,7 +170,7 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { return rawData, nil } -func findLastStableRegion(testname string, measurement string, coll *mongo.Collection) ([]float64, error) { +func findLastStableRegion(testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { filter := bson.D{ {"time_series_info.project", "mongo-go-driver"}, {"time_series_info.variant", "perf"}, @@ -157,33 +180,19 @@ func findLastStableRegion(testname string, measurement string, coll *mongo.Colle {"last", true}, {"contexts", []string{"GoDriver perf (h-score)"}}, } - projection := bson.D{ - {"values", 1}, - } - findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}).SetProjection(projection) + + findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) findCtx, cancel := context.WithTimeout(context.Background(), 180*time.Second) defer cancel() - var result bson.M - err := coll.FindOne(findCtx, filter, findOptions).Decode(&result) + var sr *StableRegion + err := coll.FindOne(findCtx, filter, findOptions).Decode(&sr) if err != nil { return nil, err } - valuesSlice, ok := result["values"].(bson.A) - if !ok { - return nil, errors.New("values is not of type bson.A") - } - var values []float64 - for _, v := range valuesSlice { - number, ok := v.(float64) - if !ok { - return nil, errors.New("value is not float64") - } - values = append(values, number) - } - return values, nil + return sr, nil } // For a specific test and measurement @@ -194,20 +203,20 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ for i := range rd.Rollups.Stats { measurement := rd.Rollups.Stats[i].Name patchVal := []float64{rd.Rollups.Stats[i].Val} - stableRegionVals, err := findLastStableRegion(testname, measurement, coll) + stableRegion, err := findLastStableRegion(testname, measurement, coll) if err != nil { return nil, err } - e, t, h := GetEnergyStatistics(mat.NewDense(len(stableRegionVals), 1, stableRegionVals), mat.NewDense(1, 1, patchVal)) + e, t, h := GetEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) es := EnergyStats{ - Benchmark: testname, - Measurement: measurement, - PatchVersion: rd.Info.Version, - StableRegionValues: stableRegionVals, - PatchValues: patchVal, - E: e, - T: t, - H: h, + Benchmark: testname, + Measurement: measurement, + PatchVersion: rd.Info.Version, + StableRegion: *stableRegion, + PatchValues: patchVal, + E: e, + T: t, + H: h, } energyStats = append(energyStats, &es) } @@ -226,3 +235,26 @@ func getEnergyStatsForAllBenchMarks(patchRawData []RawData, coll *mongo.Collecti } return allEnergyStats, nil } + +func generatePRComment(energyStats []*EnergyStats, version string) string { + + var comment strings.Builder + var testCount int64 + + comment.WriteString("# đź‘‹GoDriver Performance\n") + fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., h-score > 0.6):\n", version) + comment.WriteString("| Benchmark | Measurement | H-Score | Stable Reg Avg,Med,Std | Patch Value |\n| --- | --- | --- | --- | --- |\n") + for _, es := range energyStats { + testCount += 1 + if es.H > 0.6 { + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f,%.4f,%.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) + } + } + + if testCount == 0 { + comment.WriteString("There were no significant changes to the performance to report.") + } + comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") + + return comment.String() +} From aac6faf19b2483d19dc3cdf3fb6f7bab8e1010ad Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 21 Jul 2025 11:14:54 -0400 Subject: [PATCH 08/35] testing w z-score --- internal/cmd/perfcomp/energystatistics.go | 7 +++++++ internal/cmd/perfcomp/main.go | 11 +++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index e37b28c127..1095fac7f2 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -65,3 +65,10 @@ func getDistance(x, y *mat.Dense) float64 { } return sum } + +func GetZScore(x, u, o float64) float64 { + if u == 0 || o == 0 { + return 0 + } + return (x - u) / o +} diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 09017f34f5..f4fa690b4d 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -85,6 +85,7 @@ type EnergyStats struct { E float64 T float64 H float64 + Z float64 } func main() { @@ -217,6 +218,7 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ E: e, T: t, H: h, + Z: GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std), } energyStats = append(energyStats, &es) } @@ -242,12 +244,13 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { var testCount int64 comment.WriteString("# đź‘‹GoDriver Performance\n") - fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., h-score > 0.6):\n", version) - comment.WriteString("| Benchmark | Measurement | H-Score | Stable Reg Avg,Med,Std | Patch Value |\n| --- | --- | --- | --- | --- |\n") + fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n", version) + comment.WriteString("| Benchmark | Measurement | H-Score | Z-Score | Stable Reg Avg,Med,Std | Patch Value |\n| --- | --- | --- | --- | --- | --- |\n") for _, es := range energyStats { testCount += 1 - if es.H > 0.6 { - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f,%.4f,%.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) + // if es.H > 0.6 { + if es.Z > 1.96 || es.Z < -1.96 { + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f,%.4f,%.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) } } From 5b60b5d865cb50fe476278e16ebea491b3d24fb8 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 21 Jul 2025 11:45:12 -0400 Subject: [PATCH 09/35] add calc for percent change --- internal/cmd/perfcomp/energystatistics.go | 7 ++++ internal/cmd/perfcomp/main.go | 42 ++++++++++++----------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index 1095fac7f2..32317efb45 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -72,3 +72,10 @@ func GetZScore(x, u, o float64) float64 { } return (x - u) / o } + +func GetPercentageChange(x, u float64) float64 { + if u == 0 { + return 0 + } + return ((x - u ) / u) * 100 +} diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index f4fa690b4d..c70b31db88 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -77,15 +77,16 @@ type StableRegion struct { } type EnergyStats struct { - Benchmark string - Measurement string - PatchVersion string - StableRegion StableRegion - PatchValues []float64 - E float64 - T float64 - H float64 - Z float64 + Benchmark string + Measurement string + PatchVersion string + StableRegion StableRegion + PatchValues []float64 + PercentChange float64 + E float64 + T float64 + H float64 + Z float64 } func main() { @@ -210,15 +211,16 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ } e, t, h := GetEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) es := EnergyStats{ - Benchmark: testname, - Measurement: measurement, - PatchVersion: rd.Info.Version, - StableRegion: *stableRegion, - PatchValues: patchVal, - E: e, - T: t, - H: h, - Z: GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std), + Benchmark: testname, + Measurement: measurement, + PatchVersion: rd.Info.Version, + StableRegion: *stableRegion, + PatchValues: patchVal, + PercentChange: GetPercentageChange(patchVal[0], stableRegion.Mean), + E: e, + T: t, + H: h, + Z: GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std), } energyStats = append(energyStats, &es) } @@ -245,12 +247,12 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { comment.WriteString("# đź‘‹GoDriver Performance\n") fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n", version) - comment.WriteString("| Benchmark | Measurement | H-Score | Z-Score | Stable Reg Avg,Med,Std | Patch Value |\n| --- | --- | --- | --- | --- | --- |\n") + comment.WriteString("| Benchmark | Measurement | H-Score | Z-Score | % Change | Stable Reg Avg,Med,Std | Patch Value |\n| --- | --- | --- | --- | --- | --- | --- |\n") for _, es := range energyStats { testCount += 1 // if es.H > 0.6 { if es.Z > 1.96 || es.Z < -1.96 { - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f,%.4f,%.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | %.4f,%.4f,%.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) } } From 8cc29fde0614aad701d2720352587e7e928a093b Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 10:16:36 -0400 Subject: [PATCH 10/35] check-modules --- go.work | 6 ++++-- internal/cmd/perfcomp/go.mod | 8 +++++++- internal/cmd/perfcomp/go.sum | 8 ++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/go.work b/go.work index 9f345c684c..8b75fc17cb 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,6 @@ -go 1.23 +go 1.23.0 + +toolchain go1.24.4 use ( . @@ -7,7 +9,7 @@ use ( ./examples/_logger/zerolog ./internal/cmd/benchmark ./internal/cmd/compilecheck - ./internal/cmd/perfcomp ./internal/cmd/faas/awslambda/mongodb + ./internal/cmd/perfcomp ./internal/test/goleak ) diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod index 6446bed897..a7139c5960 100644 --- a/internal/cmd/perfcomp/go.mod +++ b/internal/cmd/perfcomp/go.mod @@ -1,17 +1,22 @@ module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp -go 1.23 +go 1.23.0 + +toolchain go1.24.4 replace go.mongodb.org/mongo-driver/v2 => ../../../ require ( + github.com/stretchr/testify v1.10.0 go.mongodb.org/mongo-driver/v2 v2.2.2 gonum.org/v1/gonum v0.16.0 ) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect @@ -19,4 +24,5 @@ require ( golang.org/x/crypto v0.33.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/text v0.23.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/internal/cmd/perfcomp/go.sum b/internal/cmd/perfcomp/go.sum index 49f669457a..6d55f6e933 100644 --- a/internal/cmd/perfcomp/go.sum +++ b/internal/cmd/perfcomp/go.sum @@ -6,6 +6,10 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= @@ -46,3 +50,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 3bc9d34b39273e6b63818f32da3d9aa065467651 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 10:25:09 -0400 Subject: [PATCH 11/35] go 1.23 --- go.work | 2 +- internal/cmd/perfcomp/go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.work b/go.work index 8b75fc17cb..009bd9e583 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.23.0 +go 1.23 toolchain go1.24.4 diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod index a7139c5960..df03c347f5 100644 --- a/internal/cmd/perfcomp/go.mod +++ b/internal/cmd/perfcomp/go.mod @@ -1,6 +1,6 @@ module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp -go 1.23.0 +go 1.23 toolchain go1.24.4 From 1e161d98424a24bba304f2d1eb495c362a068d37 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 11:02:53 -0400 Subject: [PATCH 12/35] remove toolchain --- go.work | 2 -- internal/cmd/perfcomp/go.mod | 2 -- 2 files changed, 4 deletions(-) diff --git a/go.work b/go.work index 009bd9e583..4805c5f3bd 100644 --- a/go.work +++ b/go.work @@ -1,7 +1,5 @@ go 1.23 -toolchain go1.24.4 - use ( . ./examples/_logger/logrus diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod index df03c347f5..4c147cf610 100644 --- a/internal/cmd/perfcomp/go.mod +++ b/internal/cmd/perfcomp/go.mod @@ -2,8 +2,6 @@ module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp go 1.23 -toolchain go1.24.4 - replace go.mongodb.org/mongo-driver/v2 => ../../../ require ( From ffe0ff534b71c36cc59f47c1baf22552679914e1 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 11:17:07 -0400 Subject: [PATCH 13/35] try toolchain go 1.23.10 --- go.work | 4 +++- internal/cmd/perfcomp/go.mod | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.work b/go.work index 4805c5f3bd..b7b28bb5c5 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,6 @@ -go 1.23 +go 1.23.0 + +toolchain go1.23.10 use ( . diff --git a/internal/cmd/perfcomp/go.mod b/internal/cmd/perfcomp/go.mod index 4c147cf610..3b42147b1c 100644 --- a/internal/cmd/perfcomp/go.mod +++ b/internal/cmd/perfcomp/go.mod @@ -1,6 +1,8 @@ module go.mongodb.go/mongo-driver/v2/internal/cmd/perfcomp -go 1.23 +go 1.23.0 + +toolchain go1.23.10 replace go.mongodb.org/mongo-driver/v2 => ../../../ From 5bedb8db8ace66f5244ffe6da7ddb3572b24ea47 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 12:04:11 -0400 Subject: [PATCH 14/35] probably not good --- etc/compile_check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/compile_check.sh b/etc/compile_check.sh index 457b8e9e92..f7780f8443 100755 --- a/etc/compile_check.sh +++ b/etc/compile_check.sh @@ -6,7 +6,7 @@ set -x # show all commands being run COMPILE_CHECK_DIR="internal/cmd/compilecheck" ARCHITECTURES=("386" "arm" "arm64" "ppc64le" "s390x") -BUILD_CMD="${GC} build -buildvcs=false" +BUILD_CMD="go build -buildvcs=false" # compile_check will attempt to build the internal/test/compilecheck project # using the provided Go version. This is to simulate an end-to-end use case. From 8d4f7a8008c37e4b5e5f19dbc85974a378b6e04d Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 13:38:15 -0400 Subject: [PATCH 15/35] cleanup --- internal/cmd/perfcomp/energystatistics.go | 2 + .../cmd/perfcomp/energystatistics_test.go | 4 +- internal/cmd/perfcomp/main.go | 75 ++++++++++--------- 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index 32317efb45..bc20fafc6d 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -66,6 +66,7 @@ func getDistance(x, y *mat.Dense) float64 { return sum } +// Get Z score for result x, compared to mean u and st dev o. func GetZScore(x, u, o float64) float64 { if u == 0 || o == 0 { return 0 @@ -73,6 +74,7 @@ func GetZScore(x, u, o float64) float64 { return (x - u) / o } +// Get percentage change for result x compared to mean u. func GetPercentageChange(x, u float64) float64 { if u == 0 { return 0 diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index 48afbdc9fd..ffbe6349bb 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -15,13 +15,13 @@ import ( func createTestVectors(start1 int, stop1 int, step1 int, start2 int, stop2 int, step2 int) (*mat.Dense, *mat.Dense) { xData := []float64{} + yData := []float64{} + i := start1 for i < stop1 { xData = append(xData, float64(i)) i += step1 } - - yData := []float64{} j := start2 for j < stop2 { yData = append(yData, float64(j)) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index c70b31db88..8758653369 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -77,16 +77,16 @@ type StableRegion struct { } type EnergyStats struct { - Benchmark string - Measurement string - PatchVersion string - StableRegion StableRegion - PatchValues []float64 - PercentChange float64 - E float64 - T float64 - H float64 - Z float64 + Benchmark string + Measurement string + PatchVersion string + StableRegion StableRegion + PatchValues []float64 + PChange float64 + E float64 + T float64 + H float64 + Z float64 } func main() { @@ -96,6 +96,11 @@ func main() { log.Panic("perf_uri_private_endpoint env variable is not set") } + version := os.Getenv("VERSION_ID") + if version == "" { + log.Panic("could not retrieve version") + } + client, err1 := mongo.Connect(options.Client().ApplyURI(uri)) if err1 != nil { log.Panicf("Error connecting client: %v", err1) @@ -110,10 +115,6 @@ func main() { fmt.Println("Successfully connected to MongoDB Analytics node.") db := client.Database("expanded_metrics") - version := os.Getenv("VERSION_ID") - if version == "" { - log.Panic("could not retrieve version") - } // Get raw data, most recent stable region, and calculate energy stats patchRawData, err3 := findRawData(version, db.Collection("raw_results")) @@ -123,7 +124,7 @@ func main() { allEnergyStats, err4 := getEnergyStatsForAllBenchMarks(patchRawData, db.Collection("stable_regions")) if err4 != nil { - log.Panicf("Error getting raw data: %v", err4) + log.Panicf("Error getting energy statistics: %v", err4) } fmt.Println(generatePRComment(allEnergyStats, version)) @@ -132,7 +133,6 @@ func main() { if err0 != nil { log.Panicf("Failed to disconnect client: %v", err0) } - } func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { @@ -172,6 +172,7 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { return rawData, nil } +// Find the most recent stable region of the mainline version for a specific test/measurement func findLastStableRegion(testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { filter := bson.D{ {"time_series_info.project", "mongo-go-driver"}, @@ -193,7 +194,6 @@ func findLastStableRegion(testname string, measurement string, coll *mongo.Colle if err != nil { return nil, err } - return sr, nil } @@ -205,22 +205,27 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ for i := range rd.Rollups.Stats { measurement := rd.Rollups.Stats[i].Name patchVal := []float64{rd.Rollups.Stats[i].Val} + stableRegion, err := findLastStableRegion(testname, measurement, coll) if err != nil { return nil, err } + + pChange := GetPercentageChange(patchVal[0], stableRegion.Mean) e, t, h := GetEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) + z := GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) + es := EnergyStats{ - Benchmark: testname, - Measurement: measurement, - PatchVersion: rd.Info.Version, - StableRegion: *stableRegion, - PatchValues: patchVal, - PercentChange: GetPercentageChange(patchVal[0], stableRegion.Mean), - E: e, - T: t, - H: h, - Z: GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std), + Benchmark: testname, + Measurement: measurement, + PatchVersion: rd.Info.Version, + StableRegion: *stableRegion, + PatchValues: patchVal, + PChange: pChange, + E: e, + T: t, + H: h, + Z: z, } energyStats = append(energyStats, &es) } @@ -241,25 +246,25 @@ func getEnergyStatsForAllBenchMarks(patchRawData []RawData, coll *mongo.Collecti } func generatePRComment(energyStats []*EnergyStats, version string) string { - var comment strings.Builder - var testCount int64 - comment.WriteString("# đź‘‹GoDriver Performance\n") fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n", version) - comment.WriteString("| Benchmark | Measurement | H-Score | Z-Score | % Change | Stable Reg Avg,Med,Std | Patch Value |\n| --- | --- | --- | --- | --- | --- | --- |\n") + comment.WriteString("| Benchmark | Measurement | H-Score | Z-Score | % Change | Stable Reg | Patch Value |\n| --- | --- | --- | --- | --- | --- | --- |\n") + + var testCount int64 for _, es := range energyStats { - testCount += 1 - // if es.H > 0.6 { if es.Z > 1.96 || es.Z < -1.96 { - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | %.4f,%.4f,%.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) + testCount += 1 + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.PChange, es.StableRegion.Mean, es.StableRegion.Std, es.PatchValues[0]) } } if testCount == 0 { + comment.Reset() + comment.WriteString("# đź‘‹GoDriver Performance\n") comment.WriteString("There were no significant changes to the performance to report.") } - comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") + comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") return comment.String() } From 0cd511e05e267a5aae3512e4a4f253a84037b5c4 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 22 Jul 2025 16:32:53 -0400 Subject: [PATCH 16/35] remove perfcomp from go work --- etc/compile_check.sh | 2 +- etc/perf-pr-comment.sh | 4 +++- go.work | 5 +---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/etc/compile_check.sh b/etc/compile_check.sh index f7780f8443..457b8e9e92 100755 --- a/etc/compile_check.sh +++ b/etc/compile_check.sh @@ -6,7 +6,7 @@ set -x # show all commands being run COMPILE_CHECK_DIR="internal/cmd/compilecheck" ARCHITECTURES=("386" "arm" "arm64" "ppc64le" "s390x") -BUILD_CMD="go build -buildvcs=false" +BUILD_CMD="${GC} build -buildvcs=false" # compile_check will attempt to build the internal/test/compilecheck project # using the provided Go version. This is to simulate an end-to-end use case. diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index eb41326322..83ac74271d 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -4,4 +4,6 @@ set -eux -go run ./internal/cmd/perfcomp/main.go ./internal/cmd/perfcomp/energystatistics.go +pushd ./internal/cmd/perfcomp >/dev/null || exist +GOWORK=off go run . +popd >/dev/null diff --git a/go.work b/go.work index b7b28bb5c5..23ad2ff8a7 100644 --- a/go.work +++ b/go.work @@ -1,6 +1,4 @@ -go 1.23.0 - -toolchain go1.23.10 +go 1.23 use ( . @@ -10,6 +8,5 @@ use ( ./internal/cmd/benchmark ./internal/cmd/compilecheck ./internal/cmd/faas/awslambda/mongodb - ./internal/cmd/perfcomp ./internal/test/goleak ) From e30d8d77f551ed4840a390b8d8f83cad73fb7861 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 23 Jul 2025 10:08:02 -0400 Subject: [PATCH 17/35] updated performance context --- internal/cmd/perfcomp/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 8758653369..b5c1ee5432 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -181,7 +181,7 @@ func findLastStableRegion(testname string, measurement string, coll *mongo.Colle {"time_series_info.test", testname}, {"time_series_info.measurement", measurement}, {"last", true}, - {"contexts", []string{"GoDriver perf (h-score)"}}, + {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, } findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) From 01c755545166b8bd83003d47284fe20574683c1d Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 23 Jul 2025 11:28:03 -0400 Subject: [PATCH 18/35] changes --- .evergreen/config.yml | 2 +- .../cmd/perfcomp/energystatistics_test.go | 19 ++++------ internal/cmd/perfcomp/main.go | 35 ++++++++++++++----- 3 files changed, 34 insertions(+), 22 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6ae40c46d3..d29d80591b 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -292,7 +292,7 @@ functions: binary: bash env: VERSION_ID: ${version_id} - include_expansions_in_env: [perf_uri_private_endpoint] + include_expansions_in_env: [PERF_URI_PRIVATE_ENDPOINT] args: [*task-runner, perf-pr-comment] run-enterprise-auth-tests: diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index ffbe6349bb..cce7bd9f77 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -17,15 +17,11 @@ func createTestVectors(start1 int, stop1 int, step1 int, start2 int, stop2 int, xData := []float64{} yData := []float64{} - i := start1 - for i < stop1 { + for i := start1; i < stop1; i += step1 { xData = append(xData, float64(i)) - i += step1 } - j := start2 - for j < stop2 { + for j := start2; j < stop2; j += step2 { yData = append(yData, float64(j)) - j += step2 } x := mat.NewDense(len(xData), 1, xData) @@ -41,8 +37,8 @@ func TestEnergyStatistics(t *testing.T) { e, tstat, h := GetEnergyStatistics(x, y) del := 1e-3 - - assert.InDelta(t, 0.160, e, del) // |0.160 - e| < 1/100 + // Limit precision of comparison to 3 digits after the decimal. + assert.InDelta(t, 0.160, e, del) // |0.160 - e| < 0.001 assert.InDelta(t, 8.136, tstat, del) assert.InDelta(t, 0.002, h, del) }) @@ -72,11 +68,10 @@ func TestEnergyStatistics(t *testing.T) { y := mat.NewDense(1, 1, []float64{1}) e, tstat, h := GetEnergyStatistics(x, y) - del := 1e-3 - assert.InDelta(t, 0.0, e, del) - assert.InDelta(t, 0.0, tstat, del) - assert.InDelta(t, 0.0, h, del) + assert.Equal(t, 0.0, e) + assert.Equal(t, 0.0, tstat) + assert.Equal(t, 0.0, h) }) } diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index b5c1ee5432..e42380857c 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -10,6 +10,7 @@ import ( "context" "fmt" "log" + "math" "os" "strings" "time" @@ -91,14 +92,14 @@ type EnergyStats struct { func main() { // Connect to analytics node - uri := os.Getenv("perf_uri_private_endpoint") + uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") if uri == "" { - log.Panic("perf_uri_private_endpoint env variable is not set") + log.Panic("PERF_URI_PRIVATE_ENDPOINT env variable is not set") } version := os.Getenv("VERSION_ID") if version == "" { - log.Panic("could not retrieve version") + log.Panic("VERSION_ID env variable is not set") } client, err1 := mongo.Connect(options.Client().ApplyURI(uri)) @@ -150,7 +151,11 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { cursor, err := coll.Find(findCtx, filter, findOptions) if err != nil { - return nil, err + log.Panicf( + "Error retrieving raw data for version %q: %v", + version, + err, + ) } defer cursor.Close(findCtx) @@ -160,13 +165,20 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { for cursor.Next(findCtx) { var rd RawData if err := cursor.Decode(&rd); err != nil { - return nil, err + break } rawData = append(rawData, rd) } - if err := cursor.Err(); err != nil { - return nil, err + if err == nil { + err = cursor.Err() + } + if err != nil { + log.Panicf( + "Error decoding raw data from version %q: %v", + version, + err, + ) } return rawData, nil @@ -208,7 +220,12 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ stableRegion, err := findLastStableRegion(testname, measurement, coll) if err != nil { - return nil, err + log.Panicf( + "Error finding last stable region for test %q, measurement %q: %v", + testname, + measurement, + err, + ) } pChange := GetPercentageChange(patchVal[0], stableRegion.Mean) @@ -253,7 +270,7 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { var testCount int64 for _, es := range energyStats { - if es.Z > 1.96 || es.Z < -1.96 { + if math.Abs(es.Z) > 1.96 { testCount += 1 fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.PChange, es.StableRegion.Mean, es.StableRegion.Std, es.PatchValues[0]) } From ffac5da1af81b5a1eb8203f2458df69049988a79 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 23 Jul 2025 14:31:42 -0400 Subject: [PATCH 19/35] apply code review changes Co-authored-by: Matt Dale <9760375+matthewdale@users.noreply.github.com> --- internal/cmd/perfcomp/main.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index e42380857c..f3ac91c10f 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -144,12 +144,10 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { {"info.task_name", "perf"}, } - findOptions := options.Find() - findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - cursor, err := coll.Find(findCtx, filter, findOptions) + cursor, err := coll.Find(findCtx, filter) if err != nil { log.Panicf( "Error retrieving raw data for version %q: %v", @@ -162,16 +160,13 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { fmt.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) var rawData []RawData - for cursor.Next(findCtx) { - var rd RawData - if err := cursor.Decode(&rd); err != nil { - break - } - rawData = append(rawData, rd) - } - - if err == nil { - err = cursor.Err() + err = cursor.All(findCtx, &rawData) + if err != nil { + log.Panicf( + "Error decoding raw data from version %q: %v", + version, + err, + ) } if err != nil { log.Panicf( From 00d3ca296e9caad8f8f96aa3e0a3c3aa50a8305b Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 23 Jul 2025 14:38:24 -0400 Subject: [PATCH 20/35] cleanup --- internal/cmd/perfcomp/main.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index f3ac91c10f..0c6bdcf0cd 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -168,13 +168,6 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { err, ) } - if err != nil { - log.Panicf( - "Error decoding raw data from version %q: %v", - version, - err, - ) - } return rawData, nil } From cd09f2210bc7815c2be4068d16f94a0d682c81b5 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Wed, 23 Jul 2025 16:24:15 -0400 Subject: [PATCH 21/35] version ID as args --- etc/perf-pr-comment.sh | 2 +- internal/cmd/perfcomp/main.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index 83ac74271d..e2d4512eb1 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -5,5 +5,5 @@ set -eux pushd ./internal/cmd/perfcomp >/dev/null || exist -GOWORK=off go run . +GOWORK=off go run . ${VERSION_ID} popd >/dev/null diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 0c6bdcf0cd..5e789b0cd6 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -97,9 +97,9 @@ func main() { log.Panic("PERF_URI_PRIVATE_ENDPOINT env variable is not set") } - version := os.Getenv("VERSION_ID") + version := os.Args[len(os.Args)-1] if version == "" { - log.Panic("VERSION_ID env variable is not set") + log.Panic("could not get VERSION_ID") } client, err1 := mongo.Connect(options.Client().ApplyURI(uri)) From b890d7712a4d33dbc941e69605eb7a5acb011bfa Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Thu, 24 Jul 2025 10:34:12 -0400 Subject: [PATCH 22/35] changes --- internal/cmd/perfcomp/energystatistics.go | 14 +- internal/cmd/perfcomp/main.go | 186 ++++++++++++---------- 2 files changed, 106 insertions(+), 94 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go index bc20fafc6d..5789f394d3 100644 --- a/internal/cmd/perfcomp/energystatistics.go +++ b/internal/cmd/perfcomp/energystatistics.go @@ -67,17 +67,11 @@ func getDistance(x, y *mat.Dense) float64 { } // Get Z score for result x, compared to mean u and st dev o. -func GetZScore(x, u, o float64) float64 { - if u == 0 || o == 0 { - return 0 - } - return (x - u) / o +func GetZScore(x, mu, sigma float64) float64 { + return (x - mu) / sigma } // Get percentage change for result x compared to mean u. -func GetPercentageChange(x, u float64) float64 { - if u == 0 { - return 0 - } - return ((x - u ) / u) * 100 +func GetPercentageChange(x, mu float64) float64 { + return ((x - mu ) / mu) * 100 } diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 5e789b0cd6..be3a0f5522 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -21,46 +21,59 @@ import ( "gonum.org/v1/gonum/mat" ) +// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. +// Must use GOWORK=off to run this test. + +type OverrideInfo struct { + OverrideMainline bool `bson:"override_mainline"` + BaseOrder interface{} `bson:"base_order"` + Reason interface{} `bson:"reason"` + User interface{} `bson:"user"` +} + +type Info struct { + Project string `bson:"project"` + Version string `bson:"version"` + Variant string `bson:"variant"` + Order int64 `bson:"order"` + TaskName string `bson:"task_name"` + TaskID string `bson:"task_id"` + Execution int64 `bson:"execution"` + Mainline bool `bson:"mainline"` + OverrideInfo OverrideInfo + TestName string `bson:"test_name"` + Args map[string]interface{} `bson:"args"` +} + +type Stat struct { + Name string `bson:"name"` + Val float64 `bson:"val"` + Metadata interface{} `bson:"metadata"` +} + +type Rollups struct { + Stats []Stat +} + type RawData struct { - Info struct { - Project string `bson:"project"` - Version string `bson:"version"` - Variant string `bson:"variant"` - Order int64 `bson:"order"` - TaskName string `bson:"task_name"` - TaskID string `bson:"task_id"` - Execution int64 `bson:"execution"` - Mainline bool `bson:"mainline"` - OverrideInfo struct { - OverrideMainline bool `bson:"override_mainline"` - BaseOrder interface{} `bson:"base_order"` - Reason interface{} `bson:"reason"` - User interface{} `bson:"user"` - } - TestName string `bson:"test_name"` - Args map[string]interface{} `bson:"args"` - } - CreatedAt interface{} `bson:"created_at"` - CompletedAt interface{} `bson:"completed_at"` - Rollups struct { - Stats []struct { - Name string `bson:"name"` - Val float64 `bson:"val"` - Metadata interface{} `bson:"metadata"` - } - } + Info Info + CreatedAt interface{} `bson:"created_at"` + CompletedAt interface{} `bson:"completed_at"` + Rollups Rollups FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` } +type TimeSeriesInfo struct { + Project string `bson:"project"` + Variant string `bson:"variant"` + Task string `bson:"task"` + Test string `bson:"test"` + Measurement string `bson:"measurement"` + Args map[string]interface{} `bson:"args"` +} + type StableRegion struct { - TimeSeriesInfo struct { - Project string `bson:"project"` - Variant string `bson:"variant"` - Task string `bson:"task"` - Test string `bson:"test"` - Measurement string `bson:"measurement"` - Args map[string]interface{} `bson:"args"` - } + TimeSeriesInfo TimeSeriesInfo Start interface{} `bson:"start"` End interface{} `bson:"end"` Values []float64 `bson:"values"` @@ -78,61 +91,66 @@ type StableRegion struct { } type EnergyStats struct { - Benchmark string - Measurement string - PatchVersion string - StableRegion StableRegion - PatchValues []float64 - PChange float64 - E float64 - T float64 - H float64 - Z float64 + Benchmark string + Measurement string + PatchVersion string + StableRegion StableRegion + PatchValues []float64 + PercentChange float64 + EnergyStatistic float64 + TestStatistic float64 + HScore float64 + ZScore float64 } +const expandedMetricsDB = "expanded_metrics" +const rawResultsColl = "raw_results" +const stableRegionsColl = "stable_regions" + func main() { - // Connect to analytics node + // Check for variables uri := os.Getenv("PERF_URI_PRIVATE_ENDPOINT") if uri == "" { - log.Panic("PERF_URI_PRIVATE_ENDPOINT env variable is not set") + log.Fatal("PERF_URI_PRIVATE_ENDPOINT env variable is not set") } version := os.Args[len(os.Args)-1] if version == "" { - log.Panic("could not get VERSION_ID") + log.Fatal("could not get VERSION_ID") } - client, err1 := mongo.Connect(options.Client().ApplyURI(uri)) - if err1 != nil { - log.Panicf("Error connecting client: %v", err1) + // Connect to analytics node + client, err := mongo.Connect(options.Client().ApplyURI(uri)) + if err != nil { + log.Fatalf("Error connecting client: %v", err) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err2 := client.Ping(ctx, nil) - if err2 != nil { - log.Panicf("Error pinging MongoDB Analytics: %v", err2) + err = client.Ping(ctx, nil) + if err != nil { + log.Fatalf("Error pinging MongoDB Analytics: %v", err) } - fmt.Println("Successfully connected to MongoDB Analytics node.") + log.Println("Successfully connected to MongoDB Analytics node.") - db := client.Database("expanded_metrics") + db := client.Database(expandedMetricsDB) // Get raw data, most recent stable region, and calculate energy stats - patchRawData, err3 := findRawData(version, db.Collection("raw_results")) - if err3 != nil { - log.Panicf("Error getting raw data: %v", err3) + patchRawData, err := findRawData(version, db.Collection(rawResultsColl)) + if err != nil { + log.Fatalf("Error getting raw data: %v", err) } - allEnergyStats, err4 := getEnergyStatsForAllBenchMarks(patchRawData, db.Collection("stable_regions")) - if err4 != nil { - log.Panicf("Error getting energy statistics: %v", err4) + allEnergyStats, err := getEnergyStatsForAllBenchMarks(patchRawData, db.Collection(stableRegionsColl)) + if err != nil { + log.Fatalf("Error getting energy statistics: %v", err) } - fmt.Println(generatePRComment(allEnergyStats, version)) + log.Println(generatePRComment(allEnergyStats, version)) // Disconnect client - err0 := client.Disconnect(context.Background()) - if err0 != nil { - log.Panicf("Failed to disconnect client: %v", err0) + err = client.Disconnect(context.Background()) + if err != nil { + log.Fatalf("Failed to disconnect client: %v", err) } } @@ -149,27 +167,27 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { cursor, err := coll.Find(findCtx, filter) if err != nil { - log.Panicf( + log.Fatalf( "Error retrieving raw data for version %q: %v", version, err, ) } - defer cursor.Close(findCtx) + defer func() { err = cursor.Close(findCtx) }() - fmt.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) + log.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) var rawData []RawData err = cursor.All(findCtx, &rawData) if err != nil { - log.Panicf( + log.Fatalf( "Error decoding raw data from version %q: %v", version, err, ) } - return rawData, nil + return rawData, err } // Find the most recent stable region of the mainline version for a specific test/measurement @@ -208,7 +226,7 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ stableRegion, err := findLastStableRegion(testname, measurement, coll) if err != nil { - log.Panicf( + log.Fatalf( "Error finding last stable region for test %q, measurement %q: %v", testname, measurement, @@ -221,16 +239,16 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ z := GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) es := EnergyStats{ - Benchmark: testname, - Measurement: measurement, - PatchVersion: rd.Info.Version, - StableRegion: *stableRegion, - PatchValues: patchVal, - PChange: pChange, - E: e, - T: t, - H: h, - Z: z, + Benchmark: testname, + Measurement: measurement, + PatchVersion: rd.Info.Version, + StableRegion: *stableRegion, + PatchValues: patchVal, + PercentChange: pChange, + EnergyStatistic: e, + TestStatistic: t, + HScore: h, + ZScore: z, } energyStats = append(energyStats, &es) } @@ -258,9 +276,9 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { var testCount int64 for _, es := range energyStats { - if math.Abs(es.Z) > 1.96 { + if math.Abs(es.ZScore) > 1.96 { testCount += 1 - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.H, es.Z, es.PChange, es.StableRegion.Mean, es.StableRegion.Std, es.PatchValues[0]) + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Std, es.PatchValues[0]) } } From 97e768dfc89e97b1bbd11016d388f7c6dc0d3434 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Thu, 24 Jul 2025 10:43:20 -0400 Subject: [PATCH 23/35] move energy stats to main.go --- etc/perf-pr-comment.sh | 2 +- internal/cmd/perfcomp/energystatistics.go | 77 ------------------- .../cmd/perfcomp/energystatistics_test.go | 8 +- internal/cmd/perfcomp/main.go | 70 ++++++++++++++++- 4 files changed, 72 insertions(+), 85 deletions(-) delete mode 100644 internal/cmd/perfcomp/energystatistics.go diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index e2d4512eb1..fd5dbfbad7 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -5,5 +5,5 @@ set -eux pushd ./internal/cmd/perfcomp >/dev/null || exist -GOWORK=off go run . ${VERSION_ID} +GOWORK=off go run main.go ${VERSION_ID} popd >/dev/null diff --git a/internal/cmd/perfcomp/energystatistics.go b/internal/cmd/perfcomp/energystatistics.go deleted file mode 100644 index 5789f394d3..0000000000 --- a/internal/cmd/perfcomp/energystatistics.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (C) MongoDB, Inc. 2025-present. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - -package main - -import ( - "math" - - "gonum.org/v1/gonum/mat" -) - -// Given two matrices, this function returns -// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) -func GetEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { - n, _ := x.Dims() - m, _ := y.Dims() - nf := float64(n) - mf := float64(m) - - var A float64 // E|X-Y| - if nf > 0 && mf > 0 { - A = getDistance(x, y) / (nf * mf) - } else { - A = 0 - } - var B float64 // E|X-X'| - if nf > 0 { - B = getDistance(x, x) / (nf * nf) - } else { - B = 0 - } - var C float64 // E|Y-Y'| - if mf > 0 { - C = getDistance(y, y) / (mf * mf) - } else { - C = 0 - } - - E := 2*A - B - C // D^2(F_x, F_y) - T := ((nf * mf) / (nf + mf)) * E - var H float64 - if A > 0 { - H = E / (2 * A) - } else { - H = 0 - } - return E, T, H -} - -// Given two vectors (expected 1 col), -// this function returns the sum of distances between each pair. -func getDistance(x, y *mat.Dense) float64 { - xrows, _ := x.Dims() - yrows, _ := y.Dims() - - var sum float64 - - for i := 0; i < xrows; i++ { - for j := 0; j < yrows; j++ { - sum += math.Sqrt(math.Pow((x.At(i, 0) - y.At(j, 0)), 2)) - } - } - return sum -} - -// Get Z score for result x, compared to mean u and st dev o. -func GetZScore(x, mu, sigma float64) float64 { - return (x - mu) / sigma -} - -// Get percentage change for result x compared to mean u. -func GetPercentageChange(x, mu float64) float64 { - return ((x - mu ) / mu) * 100 -} diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index cce7bd9f77..a476435258 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -34,7 +34,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 1, 105, 1) - e, tstat, h := GetEnergyStatistics(x, y) + e, tstat, h := getEnergyStatistics(x, y) del := 1e-3 // Limit precision of comparison to 3 digits after the decimal. @@ -45,7 +45,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("different distributions should have large e,t,h values", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 10000, 13000, 14) - e, tstat, h := GetEnergyStatistics(x, y) + e, tstat, h := getEnergyStatistics(x, y) del := 1e-3 assert.InDelta(t, 21859.691, e, del) @@ -55,7 +55,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("uni-variate distributions", func(t *testing.T) { x, y := createTestVectors(1, 300, 1, 1000, 5000, 10) - e, tstat, h := GetEnergyStatistics(x, y) + e, tstat, h := getEnergyStatistics(x, y) del := 1e-3 assert.InDelta(t, 4257.009, e, del) @@ -67,7 +67,7 @@ func TestEnergyStatistics(t *testing.T) { x := mat.NewDense(10, 1, []float64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) y := mat.NewDense(1, 1, []float64{1}) - e, tstat, h := GetEnergyStatistics(x, y) + e, tstat, h := getEnergyStatistics(x, y) assert.Equal(t, 0.0, e) assert.Equal(t, 0.0, tstat) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index be3a0f5522..54d4a31633 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -234,9 +234,9 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ ) } - pChange := GetPercentageChange(patchVal[0], stableRegion.Mean) - e, t, h := GetEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) - z := GetZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) + pChange := getPercentageChange(patchVal[0], stableRegion.Mean) + e, t, h := getEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) + z := getZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) es := EnergyStats{ Benchmark: testname, @@ -291,3 +291,67 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { comment.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please check out the Evergreen perf task for this patch.*") return comment.String() } + +// Given two matrices, this function returns +// (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) +func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { + n, _ := x.Dims() + m, _ := y.Dims() + nf := float64(n) + mf := float64(m) + + var A float64 // E|X-Y| + if nf > 0 && mf > 0 { + A = getDistance(x, y) / (nf * mf) + } else { + A = 0 + } + var B float64 // E|X-X'| + if nf > 0 { + B = getDistance(x, x) / (nf * nf) + } else { + B = 0 + } + var C float64 // E|Y-Y'| + if mf > 0 { + C = getDistance(y, y) / (mf * mf) + } else { + C = 0 + } + + E := 2*A - B - C // D^2(F_x, F_y) + T := ((nf * mf) / (nf + mf)) * E + var H float64 + if A > 0 { + H = E / (2 * A) + } else { + H = 0 + } + return E, T, H +} + +// Given two vectors (expected 1 col), +// this function returns the sum of distances between each pair. +func getDistance(x, y *mat.Dense) float64 { + xrows, _ := x.Dims() + yrows, _ := y.Dims() + + var sum float64 + + for i := 0; i < xrows; i++ { + for j := 0; j < yrows; j++ { + sum += math.Sqrt(math.Pow((x.At(i, 0) - y.At(j, 0)), 2)) + } + } + return sum +} + +// Get Z score for result x, compared to mean u and st dev o. +func getZScore(x, mu, sigma float64) float64 { + return (x - mu) / sigma +} + +// Get percentage change for result x compared to mean u. +func getPercentageChange(x, mu float64) float64 { + return ((x - mu) / mu) * 100 +} From ee5e23d3ec6696175b94ff9e9b6b08237887406f Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Thu, 24 Jul 2025 10:56:22 -0400 Subject: [PATCH 24/35] move toplevel comment --- internal/cmd/perfcomp/main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 54d4a31633..c36631bda0 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -4,6 +4,9 @@ // not use this file except in compliance with the License. You may obtain // a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. +// Must use GOWORK=off to run this test. + package main import ( @@ -21,9 +24,6 @@ import ( "gonum.org/v1/gonum/mat" ) -// This module cannot be included in the workspace since it requires a version of Gonum that is not compatible with the Go Driver. -// Must use GOWORK=off to run this test. - type OverrideInfo struct { OverrideMainline bool `bson:"override_mainline"` BaseOrder interface{} `bson:"base_order"` @@ -278,7 +278,7 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { for _, es := range energyStats { if math.Abs(es.ZScore) > 1.96 { testCount += 1 - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Std, es.PatchValues[0]) + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Med: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) } } From e814f1fd3bb9edda9cf31487bf1f612067a62e51 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 25 Jul 2025 14:58:37 -0400 Subject: [PATCH 25/35] added error propogation for malformed inputs in energy stat calc --- .../cmd/perfcomp/energystatistics_test.go | 24 +++++- internal/cmd/perfcomp/main.go | 78 ++++++++++++++----- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index a476435258..c4c38eab2f 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -34,7 +34,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 1, 105, 1) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h, _ := getEnergyStatistics(x, y) del := 1e-3 // Limit precision of comparison to 3 digits after the decimal. @@ -45,7 +45,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("different distributions should have large e,t,h values", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 10000, 13000, 14) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h, _ := getEnergyStatistics(x, y) del := 1e-3 assert.InDelta(t, 21859.691, e, del) @@ -55,7 +55,7 @@ func TestEnergyStatistics(t *testing.T) { t.Run("uni-variate distributions", func(t *testing.T) { x, y := createTestVectors(1, 300, 1, 1000, 5000, 10) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h, _ := getEnergyStatistics(x, y) del := 1e-3 assert.InDelta(t, 4257.009, e, del) @@ -67,11 +67,27 @@ func TestEnergyStatistics(t *testing.T) { x := mat.NewDense(10, 1, []float64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) y := mat.NewDense(1, 1, []float64{1}) - e, tstat, h := getEnergyStatistics(x, y) + e, tstat, h, _ := getEnergyStatistics(x, y) assert.Equal(t, 0.0, e) assert.Equal(t, 0.0, tstat) assert.Equal(t, 0.0, h) }) + t.Run("energy stats returns errors on malformed input", func(t *testing.T) { + x := mat.NewDense(2, 2, make([]float64, 4)) + y := mat.NewDense(2, 3, make([]float64, 6)) + + _, _, _, err := getEnergyStatistics(x, y) + assert.NotEqual(t, nil, err) + assert.ErrorContains(t, err, "both inputs must have the same number of columns") + + x = mat.NewDense(2, 2, make([]float64, 4)) + y = mat.NewDense(3, 2, make([]float64, 6)) + + _, _, _, err = getEnergyStatistics(x, y) + assert.NotEqual(t, nil, err) + assert.ErrorContains(t, err, "both inputs must be column vectors") + }) + } diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index c36631bda0..4156973aea 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -234,9 +234,18 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ ) } - pChange := getPercentageChange(patchVal[0], stableRegion.Mean) - e, t, h := getEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) - z := getZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) + var z float64 + var pChange float64 + e, t, h, err := getEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) + if err != nil { + log.Printf("Could not calculate energy stats for test %q, measurement %q: %v", testname, measurement, err) + z = 0 + pChange = 0 + } else { + z = getZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) + pChange = getPercentageChange(patchVal[0], stableRegion.Mean) + + } es := EnergyStats{ Benchmark: testname, @@ -294,64 +303,91 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { // Given two matrices, this function returns // (e, t, h) = (E-statistic, test statistic, e-coefficient of inhomogeneity) -func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64) { - n, _ := x.Dims() - m, _ := y.Dims() - nf := float64(n) - mf := float64(m) +func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { + xrows, xcols := x.Dims() + yrows, ycols := y.Dims() + + if xcols != ycols { + return 0, 0, 0, fmt.Errorf("both inputs must have the same number of columns") + } + + xrowsf := float64(xrows) + yrowsf := float64(yrows) var A float64 // E|X-Y| - if nf > 0 && mf > 0 { - A = getDistance(x, y) / (nf * mf) + if xrowsf > 0 && yrowsf > 0 { + dist, err := getDistance(x, y) + if err != nil { + return 0, 0, 0, err + } + A = dist / (xrowsf * yrowsf) } else { A = 0 } var B float64 // E|X-X'| - if nf > 0 { - B = getDistance(x, x) / (nf * nf) + if xrowsf > 0 { + dist, err := getDistance(x, x) + if err != nil { + return 0, 0, 0, err + } + B = dist / (xrowsf * xrowsf) } else { B = 0 } var C float64 // E|Y-Y'| - if mf > 0 { - C = getDistance(y, y) / (mf * mf) + if yrowsf > 0 { + dist, err := getDistance(y, y) + if err != nil { + return 0, 0, 0, err + } + C = dist / (yrowsf * yrowsf) } else { C = 0 } E := 2*A - B - C // D^2(F_x, F_y) - T := ((nf * mf) / (nf + mf)) * E + T := ((xrowsf * yrowsf) / (xrowsf + yrowsf)) * E var H float64 if A > 0 { H = E / (2 * A) } else { H = 0 } - return E, T, H + return E, T, H, nil } // Given two vectors (expected 1 col), // this function returns the sum of distances between each pair. -func getDistance(x, y *mat.Dense) float64 { - xrows, _ := x.Dims() - yrows, _ := y.Dims() +func getDistance(x, y *mat.Dense) (float64, error) { + xrows, xcols := x.Dims() + yrows, ycols := y.Dims() + + if xcols != 1 || ycols != 1 { + return 0, fmt.Errorf("both inputs must be column vectors") + } var sum float64 for i := 0; i < xrows; i++ { for j := 0; j < yrows; j++ { - sum += math.Sqrt(math.Pow((x.At(i, 0) - y.At(j, 0)), 2)) + sum += math.Abs(x.At(i, 0) - y.At(j, 0)) } } - return sum + return sum, nil } // Get Z score for result x, compared to mean u and st dev o. func getZScore(x, mu, sigma float64) float64 { + if sigma == 0 { + return math.NaN() + } return (x - mu) / sigma } // Get percentage change for result x compared to mean u. func getPercentageChange(x, mu float64) float64 { + if mu == 0 { + return math.NaN() + } return ((x - mu) / mu) * 100 } From 2a73974532e36042501860c12276326a3efed98f Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 25 Jul 2025 16:23:14 -0400 Subject: [PATCH 26/35] Rename values for clarity --- internal/cmd/perfcomp/main.go | 103 ++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 49 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 4156973aea..734e9b3519 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -25,10 +25,10 @@ import ( ) type OverrideInfo struct { - OverrideMainline bool `bson:"override_mainline"` - BaseOrder interface{} `bson:"base_order"` - Reason interface{} `bson:"reason"` - User interface{} `bson:"user"` + OverrideMainline bool `bson:"override_mainline"` + BaseOrder any `bson:"base_order"` + Reason any `bson:"reason"` + User any `bson:"user"` } type Info struct { @@ -41,14 +41,14 @@ type Info struct { Execution int64 `bson:"execution"` Mainline bool `bson:"mainline"` OverrideInfo OverrideInfo - TestName string `bson:"test_name"` - Args map[string]interface{} `bson:"args"` + TestName string `bson:"test_name"` + Args map[string]any `bson:"args"` } type Stat struct { - Name string `bson:"name"` - Val float64 `bson:"val"` - Metadata interface{} `bson:"metadata"` + Name string `bson:"name"` + Val float64 `bson:"val"` + Metadata any `bson:"metadata"` } type Rollups struct { @@ -57,37 +57,37 @@ type Rollups struct { type RawData struct { Info Info - CreatedAt interface{} `bson:"created_at"` - CompletedAt interface{} `bson:"completed_at"` + CreatedAt any `bson:"created_at"` + CompletedAt any `bson:"completed_at"` Rollups Rollups FailedRollupAttempts int64 `bson:"failed_rollup_attempts"` } type TimeSeriesInfo struct { - Project string `bson:"project"` - Variant string `bson:"variant"` - Task string `bson:"task"` - Test string `bson:"test"` - Measurement string `bson:"measurement"` - Args map[string]interface{} `bson:"args"` + Project string `bson:"project"` + Variant string `bson:"variant"` + Task string `bson:"task"` + Test string `bson:"test"` + Measurement string `bson:"measurement"` + Args map[string]any `bson:"args"` } type StableRegion struct { TimeSeriesInfo TimeSeriesInfo - Start interface{} `bson:"start"` - End interface{} `bson:"end"` - Values []float64 `bson:"values"` - StartOrder int64 `bson:"start_order"` - EndOrder int64 `bson:"end_order"` - Mean float64 `bson:"mean"` - Std float64 `bson:"std"` - Median float64 `bson:"median"` - Max float64 `bson:"max"` - Min float64 `bson:"min"` - CoefficientOfVariation float64 `bson:"coefficient_of_variation"` - LastSuccessfulUpdate interface{} `bson:"last_successful_update"` - Last bool `bson:"last"` - Contexts []interface{} `bson:"contexts"` + Start any `bson:"start"` + End any `bson:"end"` + Values []float64 `bson:"values"` + StartOrder int64 `bson:"start_order"` + EndOrder int64 `bson:"end_order"` + Mean float64 `bson:"mean"` + Std float64 `bson:"std"` + Median float64 `bson:"median"` + Max float64 `bson:"max"` + Min float64 `bson:"min"` + CoefficientOfVariation float64 `bson:"coefficient_of_variation"` + LastSuccessfulUpdate any `bson:"last_successful_update"` + Last bool `bson:"last"` + Contexts []any `bson:"contexts"` } type EnergyStats struct { @@ -95,7 +95,7 @@ type EnergyStats struct { Measurement string PatchVersion string StableRegion StableRegion - PatchValues []float64 + MeasurementVal float64 PercentChange float64 EnergyStatistic float64 TestStatistic float64 @@ -221,43 +221,48 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ var energyStats []*EnergyStats for i := range rd.Rollups.Stats { - measurement := rd.Rollups.Stats[i].Name - patchVal := []float64{rd.Rollups.Stats[i].Val} + measName := rd.Rollups.Stats[i].Name + measVal := rd.Rollups.Stats[i].Val - stableRegion, err := findLastStableRegion(testname, measurement, coll) + stableRegion, err := findLastStableRegion(testname, measName, coll) if err != nil { log.Fatalf( "Error finding last stable region for test %q, measurement %q: %v", testname, - measurement, + measName, err, ) } - var z float64 + // The performance analyzer compares the measurement value from the patch to a stable region that succeeds the latest change point. + // For example, if there were 5 measurements since the last change point, then the stable region is the 5 latest values for the measurement. + stabilityRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) + measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton + + estat, tstat, hscore, err := getEnergyStatistics(stabilityRegionVec, measValVec) + var zscore float64 var pChange float64 - e, t, h, err := getEnergyStatistics(mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values), mat.NewDense(1, 1, patchVal)) if err != nil { - log.Printf("Could not calculate energy stats for test %q, measurement %q: %v", testname, measurement, err) - z = 0 + log.Printf("Could not calculate energy stats for test %q, measurement %q: %v", testname, measName, err) + zscore = 0 pChange = 0 } else { - z = getZScore(patchVal[0], stableRegion.Mean, stableRegion.Std) - pChange = getPercentageChange(patchVal[0], stableRegion.Mean) + zscore = getZScore(measVal, stableRegion.Mean, stableRegion.Std) + pChange = getPercentageChange(measVal, stableRegion.Mean) } es := EnergyStats{ Benchmark: testname, - Measurement: measurement, + Measurement: measName, PatchVersion: rd.Info.Version, StableRegion: *stableRegion, - PatchValues: patchVal, + MeasurementVal: measVal, PercentChange: pChange, - EnergyStatistic: e, - TestStatistic: t, - HScore: h, - ZScore: z, + EnergyStatistic: estat, + TestStatistic: tstat, + HScore: hscore, + ZScore: zscore, } energyStats = append(energyStats, &es) } @@ -287,7 +292,7 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { for _, es := range energyStats { if math.Abs(es.ZScore) > 1.96 { testCount += 1 - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Med: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.PatchValues[0]) + fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Med: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) } } From 3fd0d39fee9adb599d18dcf630faaffd1c1e0f26 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 25 Jul 2025 16:27:40 -0400 Subject: [PATCH 27/35] Handle deferred errors and define findCtx in main --- internal/cmd/perfcomp/main.go | 36 +++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 734e9b3519..c881de011c 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -125,8 +125,16 @@ func main() { log.Fatalf("Error connecting client: %v", err) } + defer func() { // Defer disconnect client + err = client.Disconnect(context.Background()) + if err != nil { + log.Fatalf("Failed to disconnect client: %v", err) + } + }() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + err = client.Ping(ctx, nil) if err != nil { log.Fatalf("Error pinging MongoDB Analytics: %v", err) @@ -136,7 +144,11 @@ func main() { db := client.Database(expandedMetricsDB) // Get raw data, most recent stable region, and calculate energy stats - patchRawData, err := findRawData(version, db.Collection(rawResultsColl)) + + findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + patchRawData, err := findRawData(findCtx, version, db.Collection(rawResultsColl)) if err != nil { log.Fatalf("Error getting raw data: %v", err) } @@ -146,15 +158,9 @@ func main() { log.Fatalf("Error getting energy statistics: %v", err) } log.Println(generatePRComment(allEnergyStats, version)) - - // Disconnect client - err = client.Disconnect(context.Background()) - if err != nil { - log.Fatalf("Failed to disconnect client: %v", err) - } } -func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { +func findRawData(ctx context.Context, version string, coll *mongo.Collection) ([]RawData, error) { filter := bson.D{ {"info.project", "mongo-go-driver"}, {"info.version", version}, @@ -162,10 +168,7 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { {"info.task_name", "perf"}, } - findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - cursor, err := coll.Find(findCtx, filter) + cursor, err := coll.Find(ctx, filter) if err != nil { log.Fatalf( "Error retrieving raw data for version %q: %v", @@ -173,12 +176,17 @@ func findRawData(version string, coll *mongo.Collection) ([]RawData, error) { err, ) } - defer func() { err = cursor.Close(findCtx) }() + defer func() { + err = cursor.Close(ctx) + if err != nil { + log.Fatalf("Error closing cursor while retrieving raw data for version %q: %v", version, err) + } + }() log.Printf("Successfully retrieved %d docs from version %s.\n", cursor.RemainingBatchLength(), version) var rawData []RawData - err = cursor.All(findCtx, &rawData) + err = cursor.All(ctx, &rawData) if err != nil { log.Fatalf( "Error decoding raw data from version %q: %v", From 245e3b7942972704c6f9c36d59dcf91c31905f5f Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 25 Jul 2025 17:03:02 -0400 Subject: [PATCH 28/35] Add project flag to setup framework for other drivers --- internal/cmd/perfcomp/main.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index c881de011c..f5606eb958 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -11,6 +11,7 @@ package main import ( "context" + "flag" "fmt" "log" "math" @@ -91,6 +92,7 @@ type StableRegion struct { } type EnergyStats struct { + Project string Benchmark string Measurement string PatchVersion string @@ -119,6 +121,13 @@ func main() { log.Fatal("could not get VERSION_ID") } + // TODO (GODRIVER-3102): Map each project to a unique performance context, + // necessary for project switching to work since it's required for querying the stable region. + project := flag.String("project", "mongo-go-driver", "specify the name of an existing Evergreen project") + if project == nil { + log.Fatalf("must provide project") + } + // Connect to analytics node client, err := mongo.Connect(options.Client().ApplyURI(uri)) if err != nil { @@ -148,7 +157,7 @@ func main() { findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - patchRawData, err := findRawData(findCtx, version, db.Collection(rawResultsColl)) + patchRawData, err := findRawData(findCtx, *project, version, db.Collection(rawResultsColl)) if err != nil { log.Fatalf("Error getting raw data: %v", err) } @@ -160,9 +169,9 @@ func main() { log.Println(generatePRComment(allEnergyStats, version)) } -func findRawData(ctx context.Context, version string, coll *mongo.Collection) ([]RawData, error) { +func findRawData(ctx context.Context, project string, version string, coll *mongo.Collection) ([]RawData, error) { filter := bson.D{ - {"info.project", "mongo-go-driver"}, + {"info.project", project}, {"info.version", version}, {"info.variant", "perf"}, {"info.task_name", "perf"}, @@ -199,9 +208,9 @@ func findRawData(ctx context.Context, version string, coll *mongo.Collection) ([ } // Find the most recent stable region of the mainline version for a specific test/measurement -func findLastStableRegion(testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { +func findLastStableRegion(project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { filter := bson.D{ - {"time_series_info.project", "mongo-go-driver"}, + {"time_series_info.project", project}, {"time_series_info.variant", "perf"}, {"time_series_info.task", "perf"}, {"time_series_info.test", testname}, @@ -229,10 +238,11 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ var energyStats []*EnergyStats for i := range rd.Rollups.Stats { + project := rd.Info.Project measName := rd.Rollups.Stats[i].Name measVal := rd.Rollups.Stats[i].Val - stableRegion, err := findLastStableRegion(testname, measName, coll) + stableRegion, err := findLastStableRegion(project, testname, measName, coll) if err != nil { log.Fatalf( "Error finding last stable region for test %q, measurement %q: %v", @@ -244,10 +254,10 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ // The performance analyzer compares the measurement value from the patch to a stable region that succeeds the latest change point. // For example, if there were 5 measurements since the last change point, then the stable region is the 5 latest values for the measurement. - stabilityRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) + stableRegionVec := mat.NewDense(len(stableRegion.Values), 1, stableRegion.Values) measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton - estat, tstat, hscore, err := getEnergyStatistics(stabilityRegionVec, measValVec) + estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) var zscore float64 var pChange float64 if err != nil { @@ -261,6 +271,7 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ } es := EnergyStats{ + Project: project, Benchmark: testname, Measurement: measName, PatchVersion: rd.Info.Version, From 64a373f9172d6e5c0dc453bbfe9b8658ba8a491f Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 25 Jul 2025 17:14:42 -0400 Subject: [PATCH 29/35] Update shell script with project flag --- etc/perf-pr-comment.sh | 2 +- internal/cmd/perfcomp/main.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh index fd5dbfbad7..ab7b9b8d23 100755 --- a/etc/perf-pr-comment.sh +++ b/etc/perf-pr-comment.sh @@ -5,5 +5,5 @@ set -eux pushd ./internal/cmd/perfcomp >/dev/null || exist -GOWORK=off go run main.go ${VERSION_ID} +GOWORK=off go run main.go --project="mongo-go-driver" ${VERSION_ID} popd >/dev/null diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index f5606eb958..b3bd616df9 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -153,7 +153,6 @@ func main() { db := client.Database(expandedMetricsDB) // Get raw data, most recent stable region, and calculate energy stats - findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -267,7 +266,6 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ } else { zscore = getZScore(measVal, stableRegion.Mean, stableRegion.Std) pChange = getPercentageChange(measVal, stableRegion.Mean) - } es := EnergyStats{ @@ -348,6 +346,7 @@ func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { } else { A = 0 } + var B float64 // E|X-X'| if xrowsf > 0 { dist, err := getDistance(x, x) @@ -358,6 +357,7 @@ func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { } else { B = 0 } + var C float64 // E|Y-Y'| if yrowsf > 0 { dist, err := getDistance(y, y) From 4707e113d02f89d80e5ed7f77c386dc50ef72a43 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Fri, 25 Jul 2025 17:47:15 -0400 Subject: [PATCH 30/35] Use tabwriter to align table columns in output --- internal/cmd/perfcomp/main.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index b3bd616df9..a2375e8c2c 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -17,6 +17,7 @@ import ( "math" "os" "strings" + "text/tabwriter" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -303,15 +304,19 @@ func generatePRComment(energyStats []*EnergyStats, version string) string { var comment strings.Builder comment.WriteString("# đź‘‹GoDriver Performance\n") fmt.Fprintf(&comment, "The following benchmark tests for version %s had statistically significant changes (i.e., |z-score| > 1.96):\n", version) - comment.WriteString("| Benchmark | Measurement | H-Score | Z-Score | % Change | Stable Reg | Patch Value |\n| --- | --- | --- | --- | --- | --- | --- |\n") + + w := tabwriter.NewWriter(&comment, 0, 0, 1, ' ', 0) + fmt.Fprintln(w, "| Benchmark\t| Measurement\t| H-Score\t| Z-Score\t| % Change\t| Stable Reg\t| Patch Value\t|") + fmt.Fprintln(w, "| ---------\t| -----------\t| -------\t| -------\t| --------\t| ----------\t| -----------\t|") var testCount int64 for _, es := range energyStats { if math.Abs(es.ZScore) > 1.96 { testCount += 1 - fmt.Fprintf(&comment, "| %s | %s | %.4f | %.4f | %.4f | Avg: %.4f, Med: %.4f, Stdev: %.4f | %.4f |\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) + fmt.Fprintf(w, "| %s\t| %s\t| %.4f\t| %.4f\t| %.4f\t| Avg: %.4f, Med: %.4f, Stdev: %.4f\t| %.4f\t|\n", es.Benchmark, es.Measurement, es.HScore, es.ZScore, es.PercentChange, es.StableRegion.Mean, es.StableRegion.Median, es.StableRegion.Std, es.MeasurementVal) } } + w.Flush() if testCount == 0 { comment.Reset() From f670f334b5673f8dcffd3218f49e82dbb216a0b9 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 28 Jul 2025 09:41:52 -0400 Subject: [PATCH 31/35] add context to function signatures --- internal/cmd/perfcomp/main.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index a2375e8c2c..321fc929a4 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -162,7 +162,7 @@ func main() { log.Fatalf("Error getting raw data: %v", err) } - allEnergyStats, err := getEnergyStatsForAllBenchMarks(patchRawData, db.Collection(stableRegionsColl)) + allEnergyStats, err := getEnergyStatsForAllBenchMarks(findCtx, patchRawData, db.Collection(stableRegionsColl)) if err != nil { log.Fatalf("Error getting energy statistics: %v", err) } @@ -208,7 +208,7 @@ func findRawData(ctx context.Context, project string, version string, coll *mong } // Find the most recent stable region of the mainline version for a specific test/measurement -func findLastStableRegion(project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { +func findLastStableRegion(ctx context.Context, project string, testname string, measurement string, coll *mongo.Collection) (*StableRegion, error) { filter := bson.D{ {"time_series_info.project", project}, {"time_series_info.variant", "perf"}, @@ -221,11 +221,8 @@ func findLastStableRegion(project string, testname string, measurement string, c findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) - findCtx, cancel := context.WithTimeout(context.Background(), 180*time.Second) - defer cancel() - var sr *StableRegion - err := coll.FindOne(findCtx, filter, findOptions).Decode(&sr) + err := coll.FindOne(ctx, filter, findOptions).Decode(&sr) if err != nil { return nil, err } @@ -233,7 +230,7 @@ func findLastStableRegion(project string, testname string, measurement string, c } // For a specific test and measurement -func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { +func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo.Collection) ([]*EnergyStats, error) { testname := rd.Info.TestName var energyStats []*EnergyStats @@ -242,7 +239,7 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ measName := rd.Rollups.Stats[i].Name measVal := rd.Rollups.Stats[i].Val - stableRegion, err := findLastStableRegion(project, testname, measName, coll) + stableRegion, err := findLastStableRegion(ctx, project, testname, measName, coll) if err != nil { log.Fatalf( "Error finding last stable region for test %q, measurement %q: %v", @@ -288,10 +285,10 @@ func getEnergyStatsForOneBenchmark(rd RawData, coll *mongo.Collection) ([]*Energ return energyStats, nil } -func getEnergyStatsForAllBenchMarks(patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { +func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, coll *mongo.Collection) ([]*EnergyStats, error) { var allEnergyStats []*EnergyStats for _, rd := range patchRawData { - energyStats, err := getEnergyStatsForOneBenchmark(rd, coll) + energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) if err != nil { return nil, err } From 760e4c7f592a069d5cfda7288bbfb4420d6ad1e2 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 28 Jul 2025 09:57:44 -0400 Subject: [PATCH 32/35] Add check for empty inputs in getEnergyStats --- internal/cmd/perfcomp/energystatistics_test.go | 7 +++++++ internal/cmd/perfcomp/main.go | 3 +++ 2 files changed, 10 insertions(+) diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index c4c38eab2f..ba29f57ead 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -82,6 +82,13 @@ func TestEnergyStatistics(t *testing.T) { assert.NotEqual(t, nil, err) assert.ErrorContains(t, err, "both inputs must have the same number of columns") + x.Reset() + y = &mat.Dense{} + + _, _, _, err = getEnergyStatistics(x, y) + assert.NotEqual(t, nil, err) + assert.ErrorContains(t, err, "inputs cannot be empty") + x = mat.NewDense(2, 2, make([]float64, 4)) y = mat.NewDense(3, 2, make([]float64, 6)) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 321fc929a4..7eb48ecf4c 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -334,6 +334,9 @@ func getEnergyStatistics(x, y *mat.Dense) (float64, float64, float64, error) { if xcols != ycols { return 0, 0, 0, fmt.Errorf("both inputs must have the same number of columns") } + if xrows == 0 || yrows == 0 { + return 0, 0, 0, fmt.Errorf("inputs cannot be empty") + } xrowsf := float64(xrows) yrowsf := float64(yrows) From 9e48f45d78f401b8714c1767a67b2ac74da2fdf3 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 28 Jul 2025 10:21:00 -0400 Subject: [PATCH 33/35] Add todo for project switching --- internal/cmd/perfcomp/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 7eb48ecf4c..84a704f234 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -216,7 +216,7 @@ func findLastStableRegion(ctx context.Context, project string, testname string, {"time_series_info.test", testname}, {"time_series_info.measurement", measurement}, {"last", true}, - {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, + {"contexts", bson.D{{"$in", bson.A{"GoDriver perf task"}}}}, // TODO (GODRIVER-3102): Refactor perf context for project switching. } findOptions := options.FindOne().SetSort(bson.D{{"end", -1}}) From c3edd888e50ec701efe81b9a80cea62cc8b85ca1 Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Mon, 28 Jul 2025 10:27:56 -0400 Subject: [PATCH 34/35] Move single benchmark fail logging --- internal/cmd/perfcomp/main.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 84a704f234..165133e04c 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -255,17 +255,13 @@ func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo. measValVec := mat.NewDense(1, 1, []float64{measVal}) // singleton estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) - var zscore float64 - var pChange float64 if err != nil { - log.Printf("Could not calculate energy stats for test %q, measurement %q: %v", testname, measName, err) - zscore = 0 - pChange = 0 - } else { - zscore = getZScore(measVal, stableRegion.Mean, stableRegion.Std) - pChange = getPercentageChange(measVal, stableRegion.Mean) + return nil, fmt.Errorf("Could not calculate energy stats for test %q, measurement %q: %v", testname, measName, err) } + zscore := getZScore(measVal, stableRegion.Mean, stableRegion.Std) + pChange := getPercentageChange(measVal, stableRegion.Mean) + es := EnergyStats{ Project: project, Benchmark: testname, @@ -290,9 +286,10 @@ func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, for _, rd := range patchRawData { energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) if err != nil { - return nil, err + log.Printf("%v\n", err) // Even if energy calculation for one benchmark fails, we still want to show the ones that passed. + } else { + allEnergyStats = append(allEnergyStats, energyStats...) } - allEnergyStats = append(allEnergyStats, energyStats...) } return allEnergyStats, nil } From a6127dc5c115f22c6098067b921f83d9274be53c Mon Sep 17 00:00:00 2001 From: Selena Zhou Date: Tue, 29 Jul 2025 10:30:37 -0400 Subject: [PATCH 35/35] Log fatal if energy stats calculation fail --- internal/cmd/perfcomp/energystatistics_test.go | 2 -- internal/cmd/perfcomp/main.go | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/cmd/perfcomp/energystatistics_test.go b/internal/cmd/perfcomp/energystatistics_test.go index ba29f57ead..a41e24687d 100644 --- a/internal/cmd/perfcomp/energystatistics_test.go +++ b/internal/cmd/perfcomp/energystatistics_test.go @@ -31,7 +31,6 @@ func createTestVectors(start1 int, stop1 int, step1 int, start2 int, stop2 int, } func TestEnergyStatistics(t *testing.T) { - t.Run("similar distributions should have small e,t,h values ", func(t *testing.T) { x, y := createTestVectors(1, 100, 1, 1, 105, 1) e, tstat, h, _ := getEnergyStatistics(x, y) @@ -96,5 +95,4 @@ func TestEnergyStatistics(t *testing.T) { assert.NotEqual(t, nil, err) assert.ErrorContains(t, err, "both inputs must be column vectors") }) - } diff --git a/internal/cmd/perfcomp/main.go b/internal/cmd/perfcomp/main.go index 165133e04c..e9c55d6d1f 100644 --- a/internal/cmd/perfcomp/main.go +++ b/internal/cmd/perfcomp/main.go @@ -256,7 +256,12 @@ func getEnergyStatsForOneBenchmark(ctx context.Context, rd RawData, coll *mongo. estat, tstat, hscore, err := getEnergyStatistics(stableRegionVec, measValVec) if err != nil { - return nil, fmt.Errorf("Could not calculate energy stats for test %q, measurement %q: %v", testname, measName, err) + log.Fatalf( + "Could not calculate energy stats for test %q, measurement %q: %v", + testname, + measName, + err, + ) } zscore := getZScore(measVal, stableRegion.Mean, stableRegion.Std) @@ -286,7 +291,11 @@ func getEnergyStatsForAllBenchMarks(ctx context.Context, patchRawData []RawData, for _, rd := range patchRawData { energyStats, err := getEnergyStatsForOneBenchmark(ctx, rd, coll) if err != nil { - log.Printf("%v\n", err) // Even if energy calculation for one benchmark fails, we still want to show the ones that passed. + log.Fatalf( + "Could not get energy stats for %q: %v", + rd.Info.TestName, + err, + ) } else { allEnergyStats = append(allEnergyStats, energyStats...) }