diff --git a/.chloggen/starttime-fallback.yaml b/.chloggen/starttime-fallback.yaml index 59ee2665165b1..e15c815fa6928 100644 --- a/.chloggen/starttime-fallback.yaml +++ b/.chloggen/starttime-fallback.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: prometheusreceiver # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Add `UseCollectorStartTimeFallback` option for the start time metric adjuster to use the collector start time as an approximation of process start time as a fallback. +note: Add `UseCollectorStartTimeFallback` featuregate for the start time metric adjuster to use the collector start time as an approximation of process start time as a fallback. # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [36364] diff --git a/receiver/prometheusreceiver/README.md b/receiver/prometheusreceiver/README.md index 0cbe82c544562..da1c93551f0b4 100644 --- a/receiver/prometheusreceiver/README.md +++ b/receiver/prometheusreceiver/README.md @@ -67,7 +67,17 @@ prometheus --config.file=prom.yaml ```shell "--feature-gates=receiver.prometheusreceiver.UseCreatedMetric" ``` +- `receiver.prometheusreceiver.UseCollectorStartTimeFallback`: enables using + the collector start time as the metric start time if the + process_start_time_seconds metric yields no result (for example if targets + expose no process_start_time_seconds metric). This is useful when the collector + start time is a good approximation of the process start time - for example in + serverless workloads when the collector is deployed as a sidecar. To enable it, + use the following feature gate option: +```shell +"--feature-gates=receiver.prometheusreceiver.UseCollectorStartTimeFallback" +``` - `receiver.prometheusreceiver.EnableNativeHistograms`: process and turn native histogram metrics into OpenTelemetry exponential histograms. For more details consult the [Prometheus native histograms](#prometheus-native-histograms) section. ```shell @@ -119,7 +129,7 @@ The prometheus receiver also supports additional top-level options: - **trim_metric_suffixes**: [**Experimental**] When set to true, this enables trimming unit and some counter type suffixes from metric names. For example, it would cause `singing_duration_seconds_total` to be trimmed to `singing_duration`. This can be useful when trying to restore the original metric names used in OpenTelemetry instrumentation. Defaults to false. - **use_start_time_metric**: When set to true, this enables retrieving the start time of all counter metrics from the process_start_time_seconds metric. This is only correct if all counters on that endpoint started after the process start time, and the process is the only actor exporting the metric after the process started. It should not be used in "exporters" which export counters that may have started before the process itself. Use only if you know what you are doing, as this may result in incorrect rate calculations. Defaults to false. - **start_time_metric_regex**: The regular expression for the start time metric, and is only applied when use_start_time_metric is enabled. Defaults to process_start_time_seconds. -- **use_collector_start_time_fallback**: When set to true, this option enables using the collector start time as the metric start time if the process_start_time_seconds metric yields no result (for example if targets expose no process_start_time_seconds metric). This is useful when the collector start time is a good approximation of the process start time - for example in serverless workloads when the collector is deployed as a sidecar. This is only applied when use_start_time_metric is enabled. Defaults to false. + For example, ```yaml @@ -128,7 +138,6 @@ receivers: trim_metric_suffixes: true use_start_time_metric: true start_time_metric_regex: foo_bar_.* - use_collector_start_time_fallback: true config: scrape_configs: - job_name: 'otel-collector' diff --git a/receiver/prometheusreceiver/config.go b/receiver/prometheusreceiver/config.go index 3ad64974be120..8a07d5ff84cd3 100644 --- a/receiver/prometheusreceiver/config.go +++ b/receiver/prometheusreceiver/config.go @@ -24,17 +24,6 @@ type Config struct { PrometheusConfig *PromConfig `mapstructure:"config"` TrimMetricSuffixes bool `mapstructure:"trim_metric_suffixes"` - // Settings for adjusting metrics. Will default to using an InitialPointAdjuster - // which will use the first scraped point to define the start time for the timeseries. - AdjustOpts MetricAdjusterOpts `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct. - - // ReportExtraScrapeMetrics - enables reporting of additional metrics for Prometheus client like scrape_body_size_bytes - ReportExtraScrapeMetrics bool `mapstructure:"report_extra_scrape_metrics"` - - TargetAllocator *targetallocator.Config `mapstructure:"target_allocator"` -} - -type MetricAdjusterOpts struct { // UseStartTimeMetric enables retrieving the start time of all counter // metrics from the process_start_time_seconds metric. This is only correct // if all counters on that endpoint started after the process start time, @@ -45,19 +34,10 @@ type MetricAdjusterOpts struct { UseStartTimeMetric bool `mapstructure:"use_start_time_metric"` StartTimeMetricRegex string `mapstructure:"start_time_metric_regex"` - // UseCollectorStartTimeFallback enables using a fallback start time if a - // start time is otherwise unavailable when adjusting metrics. This would - // happen if the UseStartTimeMetric is used but the application doesn't emit - // a process_start_time_seconds metric or a metric that matches the - // StartTimeMetricRegex provided. - // - // If enabled, the fallback start time used for adjusted metrics is an - // approximation of the collector start time. - // - // This option should be used when the collector start time is a good - // approximation of the process start time - for example in serverless - // workloads when the collector is deployed as a sidecar. - UseCollectorStartTimeFallback bool `mapstructure:"use_collector_start_time_fallback"` + // ReportExtraScrapeMetrics - enables reporting of additional metrics for Prometheus client like scrape_body_size_bytes + ReportExtraScrapeMetrics bool `mapstructure:"report_extra_scrape_metrics"` + + TargetAllocator *targetallocator.Config `mapstructure:"target_allocator"` } // Validate checks the receiver configuration is valid. diff --git a/receiver/prometheusreceiver/config_test.go b/receiver/prometheusreceiver/config_test.go index f675b1adc8c2c..2c0fe018270a6 100644 --- a/receiver/prometheusreceiver/config_test.go +++ b/receiver/prometheusreceiver/config_test.go @@ -43,9 +43,9 @@ func TestLoadConfig(t *testing.T) { r1 := cfg.(*Config) assert.Equal(t, "demo", r1.PrometheusConfig.ScrapeConfigs[0].JobName) assert.Equal(t, 5*time.Second, time.Duration(r1.PrometheusConfig.ScrapeConfigs[0].ScrapeInterval)) - assert.True(t, r1.AdjustOpts.UseStartTimeMetric) + assert.True(t, r1.UseStartTimeMetric) assert.True(t, r1.TrimMetricSuffixes) - assert.Equal(t, "^(.+_)*process_start_time_seconds$", r1.AdjustOpts.StartTimeMetricRegex) + assert.Equal(t, "^(.+_)*process_start_time_seconds$", r1.StartTimeMetricRegex) assert.True(t, r1.ReportExtraScrapeMetrics) assert.Equal(t, "http://my-targetallocator-service", r1.TargetAllocator.Endpoint) diff --git a/receiver/prometheusreceiver/factory.go b/receiver/prometheusreceiver/factory.go index 91f71308b9c79..f726e6bfc5144 100644 --- a/receiver/prometheusreceiver/factory.go +++ b/receiver/prometheusreceiver/factory.go @@ -33,6 +33,14 @@ var enableNativeHistogramsGate = featuregate.GlobalRegistry().MustRegister( " those Prometheus classic histograms that have a native histogram alternative"), ) +var useCollectorStartTimeFallbackGate = featuregate.GlobalRegistry().MustRegister( + "receiver.prometheusreceiver.UseCollectorStartTimeFallback", + featuregate.StageAlpha, + featuregate.WithRegisterDescription("When enabled, the Prometheus receiver's"+ + " start time metric adjuster will fallback to using the collector start time"+ + " when a start time is not available"), +) + // NewFactory creates a new Prometheus receiver factory. func NewFactory() receiver.Factory { return receiver.NewFactory( diff --git a/receiver/prometheusreceiver/internal/starttimemetricadjuster.go b/receiver/prometheusreceiver/internal/starttimemetricadjuster.go index 9e62e30c19292..f47df61d4d732 100644 --- a/receiver/prometheusreceiver/internal/starttimemetricadjuster.go +++ b/receiver/prometheusreceiver/internal/starttimemetricadjuster.go @@ -65,7 +65,7 @@ func (stma *startTimeMetricAdjuster) AdjustMetrics(metrics pmetric.Metrics) erro if stma.fallbackStartTime == nil { return err } - stma.logger.Info("Couldn't get start time for metrics. Using fallback start time.", zap.Error(err)) + stma.logger.Info("Couldn't get start time for metrics. Using fallback start time.", zap.Error(err), zap.Time("fallback_start_time", *stma.fallbackStartTime)) startTime = float64(stma.fallbackStartTime.Unix()) } diff --git a/receiver/prometheusreceiver/metrics_receiver.go b/receiver/prometheusreceiver/metrics_receiver.go index fdde0f5c76093..0fa3a305253fd 100644 --- a/receiver/prometheusreceiver/metrics_receiver.go +++ b/receiver/prometheusreceiver/metrics_receiver.go @@ -123,8 +123,8 @@ func (r *pReceiver) initPrometheusComponents(ctx context.Context, logger log.Log }() var startTimeMetricRegex *regexp.Regexp - if r.cfg.AdjustOpts.StartTimeMetricRegex != "" { - startTimeMetricRegex, err = regexp.Compile(r.cfg.AdjustOpts.StartTimeMetricRegex) + if r.cfg.StartTimeMetricRegex != "" { + startTimeMetricRegex, err = regexp.Compile(r.cfg.StartTimeMetricRegex) if err != nil { return err } @@ -134,10 +134,10 @@ func (r *pReceiver) initPrometheusComponents(ctx context.Context, logger log.Log r.consumer, r.settings, gcInterval(r.cfg.PrometheusConfig), - r.cfg.AdjustOpts.UseStartTimeMetric, + r.cfg.UseStartTimeMetric, startTimeMetricRegex, useCreatedMetricGate.IsEnabled(), - r.cfg.AdjustOpts.UseCollectorStartTimeFallback, + useCollectorStartTimeFallbackGate.IsEnabled(), enableNativeHistogramsGate.IsEnabled(), r.cfg.PrometheusConfig.GlobalConfig.ExternalLabels, r.cfg.TrimMetricSuffixes, diff --git a/receiver/prometheusreceiver/metrics_receiver_helper_test.go b/receiver/prometheusreceiver/metrics_receiver_helper_test.go index b1b006d57a2a6..1bda7ac42e905 100644 --- a/receiver/prometheusreceiver/metrics_receiver_helper_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_helper_test.go @@ -687,8 +687,8 @@ func testComponent(t *testing.T, targets []*testData, alterConfig func(*Config), defer mp.Close() config := &Config{ - PrometheusConfig: cfg, - AdjustOpts: MetricAdjusterOpts{StartTimeMetricRegex: ""}, + PrometheusConfig: cfg, + StartTimeMetricRegex: "", } if alterConfig != nil { alterConfig(config) diff --git a/receiver/prometheusreceiver/metrics_receiver_report_extra_scrape_metrics_test.go b/receiver/prometheusreceiver/metrics_receiver_report_extra_scrape_metrics_test.go index d176cbbeef0bb..9b9fbc5a04c1d 100644 --- a/receiver/prometheusreceiver/metrics_receiver_report_extra_scrape_metrics_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_report_extra_scrape_metrics_test.go @@ -52,11 +52,9 @@ func testScraperMetrics(t *testing.T, targets []*testData, reportExtraScrapeMetr cms := new(consumertest.MetricsSink) receiver := newPrometheusReceiver(receivertest.NewNopSettings(), &Config{ - PrometheusConfig: cfg, - AdjustOpts: MetricAdjusterOpts{ - UseStartTimeMetric: false, - StartTimeMetricRegex: "", - }, + PrometheusConfig: cfg, + UseStartTimeMetric: false, + StartTimeMetricRegex: "", ReportExtraScrapeMetrics: reportExtraScrapeMetrics, }, cms) diff --git a/receiver/prometheusreceiver/metrics_receiver_test.go b/receiver/prometheusreceiver/metrics_receiver_test.go index 31c69d75b4790..b0c1ed784ce18 100644 --- a/receiver/prometheusreceiver/metrics_receiver_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_test.go @@ -1424,7 +1424,7 @@ func TestStartTimeMetric(t *testing.T) { }, } testComponent(t, targets, func(c *Config) { - c.AdjustOpts.UseStartTimeMetric = true + c.UseStartTimeMetric = true }) } @@ -1475,8 +1475,8 @@ func TestStartTimeMetricRegex(t *testing.T) { }, } testComponent(t, targets, func(c *Config) { - c.AdjustOpts.StartTimeMetricRegex = "^(.+_)*process_start_time_seconds$" - c.AdjustOpts.UseStartTimeMetric = true + c.StartTimeMetricRegex = "^(.+_)*process_start_time_seconds$" + c.UseStartTimeMetric = true }) } diff --git a/receiver/purefbreceiver/go.mod b/receiver/purefbreceiver/go.mod index ade7adfd320f8..6f0ef65a1a103 100644 --- a/receiver/purefbreceiver/go.mod +++ b/receiver/purefbreceiver/go.mod @@ -48,6 +48,7 @@ require ( github.com/docker/docker v27.3.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/ebitengine/purego v0.8.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/envoyproxy/go-control-plane v0.13.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect @@ -58,6 +59,7 @@ require ( github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/swag v0.22.9 // indirect @@ -107,6 +109,7 @@ require ( github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/linode/linodego v1.37.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -130,14 +133,19 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rs/cors v1.11.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.29 // indirect + github.com/shirou/gopsutil/v4 v4.24.12 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/collector/client v1.23.1-0.20250119231113-f07ebc3afb51 // indirect go.opentelemetry.io/collector/component/componentstatus v0.117.1-0.20250119231113-f07ebc3afb51 // indirect diff --git a/receiver/purefbreceiver/go.sum b/receiver/purefbreceiver/go.sum index 27f73a80ab2f5..118486046b914 100644 --- a/receiver/purefbreceiver/go.sum +++ b/receiver/purefbreceiver/go.sum @@ -231,6 +231,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -842,6 +843,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -867,6 +869,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -884,6 +887,7 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=