diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 05832906d1..2855d4b2ad 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -285,6 +285,16 @@ functions: echo "Response Body: $response_body" echo "HTTP Status: $http_status" + send-perf-pr-comment: + - command: subprocess.exec + type: test + params: + binary: bash + env: + COMMIT: "${github_commit}" + include_expansions_in_env: [perf_uri_private_endpoint] + args: [*task-runner, perf-pr-comment] + run-enterprise-auth-tests: - command: ec2.assume_role params: @@ -676,6 +686,7 @@ tasks: binary: bash args: [*task-runner, driver-benchmark] - func: send-perf-data + - func: send-perf-pr-comment - name: test-standalone-noauth-nossl tags: ["test", "standalone"] diff --git a/Taskfile.yml b/Taskfile.yml index 3473cb4981..8b2c7df0e3 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -70,6 +70,8 @@ tasks: pr-task: bash etc/pr-task.sh + perf-pr-comment: bash etc/perf-pr-comment.sh + # Lint with various GOOS and GOARCH tasks to catch static analysis failures that may only affect # specific operating systems or architectures. For example, staticcheck will only check for 64-bit # alignment of atomically accessed variables on 32-bit architectures (see diff --git a/etc/perf-pr-comment.sh b/etc/perf-pr-comment.sh new file mode 100755 index 0000000000..1d2b3a1cfb --- /dev/null +++ b/etc/perf-pr-comment.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# perf-pr-comment +# Generates a report of Go Driver perf changes for the current branch. + +set -eux + +go run ./internal/cmd/perfnotif/main.go ./internal/cmd/perfnotif/generateperfbaronlink.go diff --git a/internal/cmd/perfnotif/generateperfbaronlink.go b/internal/cmd/perfnotif/generateperfbaronlink.go new file mode 100644 index 0000000000..0caf44d3d3 --- /dev/null +++ b/internal/cmd/perfnotif/generateperfbaronlink.go @@ -0,0 +1,119 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "net/url" +) + +const baseURL = "https://performance-monitoring-and-analysis.server-tig.prod.corp.mongodb.com/baron" + +type CPFilter struct { + Active bool `json:"active"` + Name string `json:"name"` + Operator string `json:"operator"` + Type string `json:"type"` + Value interface{} `json:"value,omitempty"` // Exclude if no value, following structure of change_point_filters +} + +func createBaseChangePointFilters() []CPFilter { + return []CPFilter{ + {Active: true, Name: "commit", Operator: "matches", Type: "regex"}, + {Active: true, Name: "commit_date", Operator: "after", Type: "date"}, + {Active: true, Name: "calculated_on", Operator: "after", Type: "date"}, + {Active: true, Name: "project", Operator: "matches", Type: "regex"}, + {Active: true, Name: "variant", Operator: "matches", Type: "regex"}, + {Active: true, Name: "task", Operator: "matches", Type: "regex"}, + {Active: true, Name: "test", Operator: "matches", Type: "regex"}, + {Active: true, Name: "measurement", Operator: "matches", Type: "regex"}, + {Active: true, Name: "args", Operator: "eq", Type: "json"}, + {Active: true, Name: "percent_change", Operator: "gt", Type: "number"}, + {Active: true, Name: "z_score_change", Operator: "gt", Type: "number"}, + {Active: true, Name: "h_score", Operator: "gt", Type: "number"}, + {Active: true, Name: "absolute_change", Operator: "gt", Type: "number"}, + {Active: true, Name: "build_failures", Operator: "matches", Type: "regex"}, + {Active: true, Name: "bf_suggestions", Operator: "inlist", Type: "listSelect"}, + {Active: true, Name: "triage_status", Operator: "inlist", Type: "listSelect"}, + {Active: true, Name: "changeType", Operator: "inlist", Type: "listSelect"}, + {Active: true, Name: "triage_contexts", Operator: "inlist", Type: "listSelect"}, + } +} + +func updateFilterString(filters []CPFilter, name string, val string) { + for i := range filters { + if filters[i].Name == name { + filters[i].Value = val + } + } +} + +func updateFilterList(filters []CPFilter, name string, val []string) { + for i := range filters { + if filters[i].Name == name { + filters[i].Value = val + } + } +} + +func createGoDriverContextFilters() []CPFilter { + baseFilters := createBaseChangePointFilters() + updateFilterString(baseFilters, "project", "mongo-go-driver") + updateFilterString(baseFilters, "variant", "perf") + updateFilterString(baseFilters, "task", "perf") + updateFilterList(baseFilters, "triage_contexts", []string{"GoDriver perf (h-score)"}) + return baseFilters +} + +func GeneratePerfBaronLink(commitName string, testName string) (string, error) { // If empty parameters, then doesn't filter for commit or test + cpFilters := createGoDriverContextFilters() + if commitName != "" { + updateFilterString(cpFilters, "commit", commitName) + } + if testName != "" { + updateFilterString(cpFilters, "test", testName) + } + + cpFiltersJSON, err := json.Marshal(cpFilters) + if err != nil { + return "", fmt.Errorf("failed to marshal change_point_filters to json: %v", err) + } + u, err := url.Parse(baseURL) + if err != nil { + return "", fmt.Errorf("failed to parse base URL: %v", err) + } + + q := u.Query() + q.Set("change_point_filters", string(cpFiltersJSON)) + u.RawQuery = q.Encode() + return u.String(), nil +} + +func TestGeneratePerfBaronLink() { + fmt.Println() + // just mongo-go-driver, perf, perf + link1, err1 := GeneratePerfBaronLink("", "") + if err1 == nil { + fmt.Println(link1) + } + // add commit + link2, err2 := GeneratePerfBaronLink("50cf0c20d228975074c0010bfb688917e25934a4", "") + if err2 == nil { + fmt.Println(link2) + } + // add test + link3, err3 := GeneratePerfBaronLink("", "BenchmarkMultiInsertLargeDocument") + if err3 == nil { + fmt.Println(link3) + } + // add both + link4, err4 := GeneratePerfBaronLink("50cf0c20d228975074c0010bfb688917e25934a4", "BenchmarkMultiInsertLargeDocument") + if err4 == nil { + fmt.Println(link4) + } +} diff --git a/internal/cmd/perfnotif/main.go b/internal/cmd/perfnotif/main.go new file mode 100644 index 0000000000..f3f1eab363 --- /dev/null +++ b/internal/cmd/perfnotif/main.go @@ -0,0 +1,149 @@ +// Copyright (C) MongoDB, Inc. 2025-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main + +import ( + "bytes" + "context" + "fmt" + "log" + "os" + "time" + + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +type ChangePoint struct { + TimeSeriesInfo struct { + Project string `bson:"project"` + Task string `bson:"task"` + Test string `bson:"test"` + Measurement string `bson:"measurement"` + } `bson:"time_series_info"` + TriageContexts []string `bson:"triage_contexts"` + HScore float64 `bson:"h_score"` +} + +func main() { + uri := os.Getenv("perf_uri_private_endpoint") + if uri == "" { + log.Panic("perf_uri_private_endpoint env variable is not set") + } + + client, err := mongo.Connect(options.Client().ApplyURI(uri)) + if err != nil { + log.Panicf("Error connecting client: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err = client.Ping(ctx, nil) + if err != nil { + log.Panicf("Error pinging MongoDB Analytics: %v", err) + } + fmt.Println("Successfully connected to MongoDB Analytics node.") + + commit := os.Getenv("COMMIT") + if commit == "" { + log.Panic("could not retrieve commit number") + } + + coll := client.Database("expanded_metrics").Collection("change_points") + var changePoints []ChangePoint + changePoints, err = getDocsWithContext(coll, commit) + if err != nil { + log.Panicf("Error retrieving and decoding documents from collection: %v.", err) + } + + var markdownComment = getMarkdownComment(changePoints, commit) + fmt.Print(markdownComment.String()) + + err = client.Disconnect(context.Background()) + if err != nil { + log.Panicf("Failed to disconnect client: %v", err) + } + +} + +func getDocsWithContext(coll *mongo.Collection, commit string) ([]ChangePoint, error) { + filter := bson.D{ + {"time_series_info.project", "mongo-go-driver"}, + {"time_series_info.variant", "perf"}, + {"time_series_info.task", "perf"}, + {"commit", commit}, + {"triage_contexts", bson.M{"$in": []string{"GoDriver perf (h-score)"}}}, + } + + projection := bson.D{ + {"time_series_info.project", 1}, + {"time_series_info.task", 1}, + {"time_series_info.test", 1}, + {"time_series_info.measurement", 1}, + {"triage_contexts", 1}, + {"h_score", 1}, + } + + findOptions := options.Find().SetProjection(projection) + + findCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + cursor, err := coll.Find(findCtx, filter, findOptions) + if err != nil { + return nil, err + } + defer cursor.Close(findCtx) + + fmt.Printf("Successfully retrieved %d documents from commit %s.\n", cursor.RemainingBatchLength(), commit) + + var changePoints []ChangePoint + for cursor.Next(findCtx) { + var cp ChangePoint + if err := cursor.Decode(&cp); err != nil { + return nil, err + } + changePoints = append(changePoints, cp) + } + + if err := cursor.Err(); err != nil { + return nil, err + } + + return changePoints, nil +} + +func getMarkdownComment(changePoints []ChangePoint, commit string) bytes.Buffer { + var buffer bytes.Buffer + + buffer.WriteString("# 👋 GoDriver Performance Notification\n") + + if len(changePoints) > 0 { + buffer.WriteString("The following benchmark tests had statistically significant changes (i.e., h-score > 0.6):\n") + buffer.WriteString("| Benchmark Test | Measurement | H-Score | Performance Baron |\n") + buffer.WriteString("|---|---|---|---|\n") + + for _, cp := range changePoints { + perfBaronLink, err := GeneratePerfBaronLink(commit, cp.TimeSeriesInfo.Test) + if err != nil { + perfBaronLink = "" + } + fmt.Fprintf(&buffer, "| %s | %s | %f | [linked here](%s) |\n", cp.TimeSeriesInfo.Test, cp.TimeSeriesInfo.Measurement, cp.HScore, perfBaronLink) + } + } else { + buffer.WriteString("There were no significant changes to the performance to report.\n") + } + + perfBaronLink, err := GeneratePerfBaronLink(commit, "") + if err != nil { + perfBaronLink = "" + } + buffer.WriteString("\n*For a comprehensive view of all microbenchmark results for this PR's commit, please visit [this link](" + perfBaronLink + ")*") + + return buffer +}