Cody Rose
5581f083b4
We have several metrics that have captured job ID as a dimension. This is, in a strict sense, "wrong" because the cardinality is unbounded, but our job counts have been low enough that we've been able to get away with it. However, recent exploration of a new distributed job technique has caused job counts to substantially increase, and this extra, "incorrect" dimension is now harder to justify. We have been recently seeing some trouble with some of our Prometheus scrapes, and while I haven't drawn a direct connection between that trouble and the increased cardinality, I do know that we don't use the job ID dimension anywhere. We always sum it away! So this commit removes it. We're shouldn't be doing it, we're not using it, and it might be causing problems we're seeing.
116 lines
4.0 KiB
Go
116 lines
4.0 KiB
Go
package engine
|
|
|
|
import (
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
|
|
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
|
|
)
|
|
|
|
var (
|
|
decodeLatency = promauto.NewHistogramVec(
|
|
prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "decode_latency",
|
|
Help: "Time spent decoding a chunk in microseconds",
|
|
Buckets: prometheus.ExponentialBuckets(50, 2, 20),
|
|
},
|
|
[]string{"decoder_type", "source_name"},
|
|
)
|
|
|
|
// Detector metrics.
|
|
detectorExecutionCount = promauto.NewCounterVec(
|
|
prometheus.CounterOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "detector_execution_count",
|
|
Help: "Total number of times a detector has been executed.",
|
|
},
|
|
[]string{"detector_name", "job_id", "source_name"},
|
|
)
|
|
|
|
// Note this is the time taken to call FromData on each detector, not necessarily the time taken
|
|
// to verify a credential via an API call. If the regex match within FromData does not match, the
|
|
// detector will return early. For now this is a good proxy for the time taken to verify a credential.
|
|
// TODO (ahrav)
|
|
// We can work on a more fine-grained metric later.
|
|
detectorExecutionDuration = promauto.NewHistogramVec(
|
|
prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "detector_execution_duration",
|
|
Help: "Duration of detector execution in milliseconds.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 5, 6),
|
|
},
|
|
[]string{"detector_name"},
|
|
)
|
|
|
|
jobBytesScanned = promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "job_bytes_scanned",
|
|
Help: "Total number of bytes scanned for a job.",
|
|
},
|
|
[]string{"source_type", "source_name"},
|
|
)
|
|
|
|
scanBytesPerChunk = promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "scan_bytes_per_chunk",
|
|
Help: "Total number of bytes in a chunk.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 2, 18),
|
|
})
|
|
|
|
jobChunksScanned = promauto.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "job_chunks_scanned",
|
|
Help: "Total number of chunks scanned for a job.",
|
|
},
|
|
[]string{"source_type", "source_name"},
|
|
)
|
|
|
|
detectBytesPerMatch = promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "detect_bytes_per_match",
|
|
Help: "Total number of bytes used to detect a credential in a match per chunk.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 2, 18),
|
|
})
|
|
|
|
matchesPerChunk = promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "matches_per_chunk",
|
|
Help: "Total number of matches found in a chunk.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 2, 10),
|
|
})
|
|
|
|
// Metrics around latency for the different stages of the pipeline.
|
|
chunksScannedLatency = promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "chunk_scanned_latency",
|
|
Help: "Time taken to scan a chunk in microseconds.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 2, 22),
|
|
})
|
|
|
|
chunksDetectedLatency = promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "chunk_detected_latency",
|
|
Help: "Time taken to detect a chunk in microseconds.",
|
|
Buckets: prometheus.ExponentialBuckets(50, 2, 20),
|
|
})
|
|
|
|
chunksNotifiedLatency = promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: common.MetricsNamespace,
|
|
Subsystem: common.MetricsSubsystem,
|
|
Name: "chunk_notified_latency",
|
|
Help: "Time taken to notify a chunk in milliseconds.",
|
|
Buckets: prometheus.ExponentialBuckets(5, 2, 12),
|
|
})
|
|
)
|