PMM-13602 Improve performance (#987)

* PMM-13602 Improve performance

* PMM-13602 Improve performance for callstats

* PMM-13602 enable indexDetails.

* PMM-13602 fix linters.

* PMM-13602 revert diagnostic data changes.

* PMM-13602 fix linters.

* Update .golangci.yml

Co-authored-by: Alex Demidoff <[email protected]>

---------

Co-authored-by: Alex Demidoff <[email protected]>

Author: BupycHuk

.golangci.yml

@@ -32,8 +32,9 @@ linters:
       - unused
       - testpackage
       - wsl
-      - exhaustivestruct
+      - exhaustruct
       - varnamelen
+      - gochecknoglobals # we need globals for better performance, we use them for mapping, and we don't mutate them, so it's safe.
       - maligned          #deprecated
       - scopelint         #deprecated
       - golint            #deprecated

exporter/collstats_collector.go

@@ -37,12 +37,12 @@ type collstatsCollector struct {
 }
 
 // newCollectionStatsCollector creates a collector for statistics about collections.
-func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, compatible, discovery bool, topology labelsGetter, collections []string) *collstatsCollector {
+func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, discovery bool, topology labelsGetter, collections []string) *collstatsCollector {
 	return &collstatsCollector{
 		ctx:  ctx,
 		base: newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "collstats"})),
 
-		compatibleMode:  compatible,
+		compatibleMode:  false, // there are no compatible metrics for this collector.
 		discoveringMode: discovery,
 		topologyInfo:    topology,
 
@@ -108,7 +108,9 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {
 			},
 		}
 
-		cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation})
+		pipeline := mongo.Pipeline{aggregation}
+
+		cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, pipeline)
 		if err != nil {
 			logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err)
 

exporter/collstats_collector_test.go

@@ -54,7 +54,7 @@ func TestCollStatsCollector(t *testing.T) {
 
 	collection := []string{"testdb.testcol_00", "testdb.testcol_01", "testdb.testcol_02"}
 	logger := logrus.New()
-	c := newCollectionStatsCollector(ctx, client, logger, false, false, ti, collection)
+	c := newCollectionStatsCollector(ctx, client, logger, false, ti, collection)
 
 	// The last \n at the end of this string is important
 	expected := strings.NewReader(`

exporter/exporter.go

@@ -153,6 +153,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 		e.getTotalCollectionsCount() <= e.opts.CollStatsLimit {
 		limitsOk = true
 	}
+	limitsOk = true
 
 	if e.opts.CollectAll {
 		if len(e.opts.CollStatsNamespaces) == 0 {
@@ -191,7 +192,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 	// If we manually set the collection names we want or auto discovery is set.
 	if (len(e.opts.CollStatsNamespaces) > 0 || e.opts.DiscoveringMode) && e.opts.EnableCollStats && limitsOk && requestOpts.EnableCollStats {
 		cc := newCollectionStatsCollector(ctx, client, e.opts.Logger,
-			e.opts.CompatibleMode, e.opts.DiscoveringMode,
+			e.opts.DiscoveringMode,
 			topologyInfo, e.opts.CollStatsNamespaces)
 		registry.MustRegister(cc)
 	}

exporter/metrics.go

@@ -335,7 +335,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo
 
 			metrics := []*rawMetric{rm}
 
-			if renamedMetrics := metricRenameAndLabel(rm, specialConversions()); renamedMetrics != nil {
+			if renamedMetrics := metricRenameAndLabel(rm, specialConversions); renamedMetrics != nil {
 				metrics = renamedMetrics
 			}
 
@@ -435,62 +435,60 @@ func metricRenameAndLabel(rm *rawMetric, convs []conversion) []*rawMetric {
 
 // specialConversions returns a list of special conversions we want to implement.
 // See: https://jira.percona.com/browse/PMM-6506
-func specialConversions() []conversion {
-	return []conversion{
-		{
-			oldName:     "mongodb_ss_opLatencies_ops",
-			prefix:      "mongodb_ss_opLatencies",
-			suffixLabel: "op_type",
-			suffixMapping: map[string]string{
-				"commands_ops":     "commands",
-				"reads_ops":        "reads",
-				"transactions_ops": "transactions",
-				"writes_ops":       "writes",
-			},
+var specialConversions = []conversion{ //nolint:gochecknoglobals
+	{
+		oldName:     "mongodb_ss_opLatencies_ops",
+		prefix:      "mongodb_ss_opLatencies",
+		suffixLabel: "op_type",
+		suffixMapping: map[string]string{
+			"commands_ops":     "commands",
+			"reads_ops":        "reads",
+			"transactions_ops": "transactions",
+			"writes_ops":       "writes",
 		},
-		{
-			oldName:     "mongodb_ss_opLatencies_latency",
-			prefix:      "mongodb_ss_opLatencies",
-			suffixLabel: "op_type",
-			suffixMapping: map[string]string{
-				"commands_latency":     "commands",
-				"reads_latency":        "reads",
-				"transactions_latency": "transactions",
-				"writes_latency":       "writes",
-			},
+	},
+	{
+		oldName:     "mongodb_ss_opLatencies_latency",
+		prefix:      "mongodb_ss_opLatencies",
+		suffixLabel: "op_type",
+		suffixMapping: map[string]string{
+			"commands_latency":     "commands",
+			"reads_latency":        "reads",
+			"transactions_latency": "transactions",
+			"writes_latency":       "writes",
 		},
-		// mongodb_ss_wt_concurrentTransactions_read_out
-		// mongodb_ss_wt_concurrentTransactions_write_out
-		{
-			oldName:     "mongodb_ss_wt_concurrentTransactions_out",
-			prefix:      "mongodb_ss_wt_concurrentTransactions",
-			suffixLabel: "txn_rw",
-			suffixMapping: map[string]string{
-				"read_out":  "read",
-				"write_out": "write",
-			},
+	},
+	// mongodb_ss_wt_concurrentTransactions_read_out
+	// mongodb_ss_wt_concurrentTransactions_write_out
+	{
+		oldName:     "mongodb_ss_wt_concurrentTransactions_out",
+		prefix:      "mongodb_ss_wt_concurrentTransactions",
+		suffixLabel: "txn_rw",
+		suffixMapping: map[string]string{
+			"read_out":  "read",
+			"write_out": "write",
 		},
-		// mongodb_ss_wt_concurrentTransactions_read_available
-		// mongodb_ss_wt_concurrentTransactions_write_available
-		{
-			oldName:     "mongodb_ss_wt_concurrentTransactions_available",
-			prefix:      "mongodb_ss_wt_concurrentTransactions",
-			suffixLabel: "txn_rw",
-			suffixMapping: map[string]string{
-				"read_available":  "read",
-				"write_available": "write",
-			},
+	},
+	// mongodb_ss_wt_concurrentTransactions_read_available
+	// mongodb_ss_wt_concurrentTransactions_write_available
+	{
+		oldName:     "mongodb_ss_wt_concurrentTransactions_available",
+		prefix:      "mongodb_ss_wt_concurrentTransactions",
+		suffixLabel: "txn_rw",
+		suffixMapping: map[string]string{
+			"read_available":  "read",
+			"write_available": "write",
 		},
-		// mongodb_ss_wt_concurrentTransactions_read_totalTickets
-		// mongodb_ss_wt_concurrentTransactions_write_totalTickets
-		{
-			oldName:     "mongodb_ss_wt_concurrentTransactions_totalTickets",
-			prefix:      "mongodb_ss_wt_concurrentTransactions",
-			suffixLabel: "txn_rw",
-			suffixMapping: map[string]string{
-				"read_totalTickets":  "read",
-				"write_totalTickets": "write",
-			},
+	},
+	// mongodb_ss_wt_concurrentTransactions_read_totalTickets
+	// mongodb_ss_wt_concurrentTransactions_write_totalTickets
+	{
+		oldName:     "mongodb_ss_wt_concurrentTransactions_totalTickets",
+		prefix:      "mongodb_ss_wt_concurrentTransactions",
+		suffixLabel: "txn_rw",
+		suffixMapping: map[string]string{
+			"read_totalTickets":  "read",
+			"write_totalTickets": "write",
 		},
-	}
+	},
 }

exporter/metrics_test.go

@@ -211,7 +211,7 @@ func TestRawToCompatibleRawMetric(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		m := metricRenameAndLabel(tc.in, specialConversions())
+		m := metricRenameAndLabel(tc.in, specialConversions)
 		assert.Equal(t, m[0], tc.want)
 	}
 }

exporter/top_collector.go

@@ -42,7 +42,7 @@ func newTopCollector(ctx context.Context, client *mongo.Client, logger *logrus.L
 	return &topCollector{
 		ctx:            ctx,
 		base:           newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "top"})),
-		compatibleMode: compatible,
+		compatibleMode: false, // there are no compatible metrics for this collector.
 		topologyInfo:   topology,
 	}
 }