Skip to content

Commit

Permalink
fix config key collision: un-squash client config (issue 37332)
Browse files Browse the repository at this point in the history
  • Loading branch information
vigr committed Jan 30, 2025
1 parent 1b93d01 commit de2388c
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 17 deletions.
21 changes: 14 additions & 7 deletions exporter/prometheusremotewriteexporter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,16 @@ how this exporter works.

The following settings are required:

- `endpoint` (no default): The remote write URL to send remote write samples.
- `prw_client`
- `endpoint` (no default): The remote write URL to send remote write samples.
- `endpoint`: DEPRECATED

By default, TLS is enabled and must be configured under `tls:`:
By default, TLS is enabled and must be configured under `prw_client.tls:`:

- `insecure` (default = `false`): whether to enable client transport security for
the exporter's connection.

As a result, the following parameters are also required under `tls:`:
As a result, the following parameters are also required under `prw_client.tls:`:

- `cert_file` (no default): path to the TLS cert to use for TLS required connections. Should
only be used if `insecure` is set to false.
Expand All @@ -46,8 +48,11 @@ As a result, the following parameters are also required under `tls:`:
The following settings can be optionally configured:

- `external_labels`: map of labels names and values to be attached to each metric data point
- `headers`: additional headers attached to each HTTP request.
- *Note the following headers cannot be changed: `Content-Encoding`, `Content-Type`, `X-Prometheus-Remote-Write-Version`, and `User-Agent`.*
- `prw_client`
- `headers`: additional headers attached to each HTTP request.
- *Note the following headers cannot be changed: `Content-Encoding`, `Content-Type`, `X-Prometheus-Remote-Write-Version`, and `User-Agent`.*
- `timeout`: sets timeout for HTTP requests to Prometheus remote write endpoint (default 5s).
- `headers`: DEPRECATED
- `namespace`: prefix attached to each exported metric name.
- `add_metric_suffixes`: If set to false, type and unit suffixes will not be added to metrics. Default: true.
- `send_metadata`: If set to true, prometheus metadata will be generated and sent. Default: false.
Expand All @@ -73,7 +78,8 @@ Example:
```yaml
exporters:
prometheusremotewrite:
endpoint: "https://my-cortex:7900/api/v1/push"
prw_client:
endpoint: "https://my-cortex:7900/api/v1/push"
wal: # Enabling the Write-Ahead-Log for the exporter.
directory: ./prom_rw # The directory to store the WAL in
buffer_size: 100 # Optional count of elements to be read from the WAL before truncating; default of 300
Expand All @@ -87,7 +93,8 @@ Example:
```yaml
exporters:
prometheusremotewrite:
endpoint: "https://my-cortex:7900/api/v1/push"
prw_client:
endpoint: "https://my-cortex:7900/api/v1/push"
external_labels:
label_name1: label_value1
label_name2: label_value2
Expand Down
43 changes: 43 additions & 0 deletions exporter/prometheusremotewriteexporter/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/config/confighttp"
"go.opentelemetry.io/collector/config/configretry"
"go.opentelemetry.io/collector/confmap"
"go.opentelemetry.io/collector/exporter/exporterhelper"

"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry"
Expand All @@ -31,6 +32,8 @@ type Config struct {
ExternalLabels map[string]string `mapstructure:"external_labels"`

ClientConfig confighttp.ClientConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
// PRWClient configures HTTP client for Prometheus remote write
PRWClient *confighttp.ClientConfig `mapstructure:"prw_client,omitempty"`

// maximum size in bytes of time series batch sent to remote storage
MaxBatchSizeBytes int `mapstructure:"max_batch_size_bytes"`
Expand Down Expand Up @@ -59,6 +62,16 @@ type Config struct {
SendMetadata bool `mapstructure:"send_metadata"`
}

func (cfg *Config) Unmarshal(c *confmap.Conf) error {
if err := c.Unmarshal(cfg); err != nil {
return err
}
if cfg.PRWClient != nil {
setDefaultsPRWClientConfig(cfg.PRWClient)
}
return nil
}

type CreatedMetric struct {
// Enabled if true the _created metrics could be exported
Enabled bool `mapstructure:"enabled"`
Expand Down Expand Up @@ -126,3 +139,33 @@ func (cfg *Config) Validate() error {

return nil
}

func setDefaultsPRWClientConfig(cfg *confighttp.ClientConfig) {
if cfg.Endpoint == "" {
cfg.Endpoint = "http://some.url:9411/api/prom/push"
}
// We almost read 0 bytes, so no need to tune ReadBufferSize.
if cfg.WriteBufferSize == 0 {
cfg.WriteBufferSize = 512 * 1024
}
if cfg.Timeout == 0 {
cfg.Timeout = exporterhelper.NewDefaultTimeoutConfig().Timeout
}

defaultCfg := confighttp.NewDefaultClientConfig()
if cfg.Headers == nil {
cfg.Headers = defaultCfg.Headers
}
if cfg.MaxIdleConns == nil {
cfg.MaxIdleConns = defaultCfg.MaxIdleConns
}
if cfg.MaxIdleConnsPerHost == nil {
cfg.MaxIdleConnsPerHost = defaultCfg.MaxIdleConnsPerHost
}
if cfg.MaxConnsPerHost == nil {
cfg.MaxConnsPerHost = defaultCfg.MaxConnsPerHost
}
if cfg.IdleConnTimeout == nil {
cfg.IdleConnTimeout = defaultCfg.IdleConnTimeout
}
}
34 changes: 34 additions & 0 deletions exporter/prometheusremotewriteexporter/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ func TestLoadConfig(t *testing.T) {
"Prometheus-Remote-Write-Version": "0.1.0",
"X-Scope-OrgID": "234",
}
prwClientConfig := newDefaultPRWClientConfig()
prwClientConfig.Endpoint = "localhost:8888"
prwClientConfig.Timeout = 11 * time.Second
tests := []struct {
id component.ID
expected component.Config
Expand Down Expand Up @@ -76,13 +79,44 @@ func TestLoadConfig(t *testing.T) {
Namespace: "test-space",
ExternalLabels: map[string]string{"key1": "value1", "key2": "value2"},
ClientConfig: clientConfig,
PRWClient: nil,
ResourceToTelemetrySettings: resourcetotelemetry.Settings{Enabled: true},
TargetInfo: &TargetInfo{
Enabled: true,
},
CreatedMetric: &CreatedMetric{Enabled: true},
},
},
{
id: component.NewIDWithName(metadata.Type, "prw_client"),
expected: &Config{
MaxBatchSizeBytes: 3000000,
TimeoutSettings: exporterhelper.NewDefaultTimeoutConfig(),
BackOffConfig: configretry.BackOffConfig{
Enabled: true,
InitialInterval: 50 * time.Millisecond,
MaxInterval: 30 * time.Second,
MaxElapsedTime: 5 * time.Minute,
RandomizationFactor: backoff.DefaultRandomizationFactor,
Multiplier: backoff.DefaultMultiplier,
},
RemoteWriteQueue: RemoteWriteQueue{
Enabled: true,
QueueSize: 10000,
NumConsumers: 5,
},
AddMetricSuffixes: true,
Namespace: "",
ExternalLabels: make(map[string]string),
ClientConfig: newDefaultPRWClientConfig(),
PRWClient: &prwClientConfig,
ResourceToTelemetrySettings: resourcetotelemetry.Settings{Enabled: false},
TargetInfo: &TargetInfo{
Enabled: true,
},
CreatedMetric: &CreatedMetric{Enabled: false},
},
},
{
id: component.NewIDWithName(metadata.Type, "negative_queue_size"),
errorMessage: "remote write queue size can't be negative",
Expand Down
9 changes: 7 additions & 2 deletions exporter/prometheusremotewriteexporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,12 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
return nil, err
}

endpointURL, err := url.ParseRequestURI(cfg.ClientConfig.Endpoint)
clientSettings := &cfg.ClientConfig
if cfg.PRWClient != nil {
clientSettings = cfg.PRWClient
}

endpointURL, err := url.ParseRequestURI(clientSettings.Endpoint)
if err != nil {
return nil, errors.New("invalid endpoint")
}
Expand All @@ -139,7 +144,7 @@ func newPRWExporter(cfg *Config, set exporter.Settings) (*prwExporter, error) {
userAgentHeader: userAgentHeader,
maxBatchSizeBytes: cfg.MaxBatchSizeBytes,
concurrency: concurrency,
clientSettings: &cfg.ClientConfig,
clientSettings: clientSettings,
settings: set.TelemetrySettings,
retrySettings: cfg.BackOffConfig,
retryOnHTTP429: retryOn429FeatureGate.IsEnabled(),
Expand Down
30 changes: 30 additions & 0 deletions exporter/prometheusremotewriteexporter/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,36 @@ func Test_NewPRWExporter(t *testing.T) {
}
}

func Test_NewPRWExporter_ClientConfig(t *testing.T) {
prwClientConfig := confighttp.NewDefaultClientConfig()
prwClientConfig.Endpoint = "overridden.endpoint:8080"
cfg := &Config{
TimeoutSettings: exporterhelper.TimeoutConfig{},
BackOffConfig: configretry.BackOffConfig{},
Namespace: "",
ExternalLabels: map[string]string{},
ClientConfig: confighttp.NewDefaultClientConfig(),
PRWClient: &prwClientConfig,
TargetInfo: &TargetInfo{
Enabled: true,
},
CreatedMetric: &CreatedMetric{
Enabled: false,
},
}
buildInfo := component.BuildInfo{
Description: "OpenTelemetry Collector",
Version: "1.0",
}
set := exportertest.NewNopSettings()
set.BuildInfo = buildInfo

prwe, err := newPRWExporter(cfg, set)
assert.NoError(t, err)

assert.Equal(t, "overridden.endpoint:8080", prwe.endpointURL.String())
}

// Test_Start checks if the client is properly created as expected.
func Test_Start(t *testing.T) {
cfg := &Config{
Expand Down
16 changes: 8 additions & 8 deletions exporter/prometheusremotewriteexporter/factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,16 +82,15 @@ func createMetricsExporter(ctx context.Context, set exporter.Settings,
return resourcetotelemetry.WrapMetricsExporter(prwCfg.ResourceToTelemetrySettings, exporter), nil
}

func newDefaultPRWClientConfig() confighttp.ClientConfig {
cfg := confighttp.NewDefaultClientConfig()
setDefaultsPRWClientConfig(&cfg)
return cfg
}

func createDefaultConfig() component.Config {
retrySettings := configretry.NewDefaultBackOffConfig()
retrySettings.InitialInterval = 50 * time.Millisecond
clientConfig := confighttp.NewDefaultClientConfig()
clientConfig.Endpoint = "http://some.url:9411/api/prom/push"
// We almost read 0 bytes, so no need to tune ReadBufferSize.
clientConfig.ReadBufferSize = 0
clientConfig.WriteBufferSize = 512 * 1024
clientConfig.Timeout = exporterhelper.NewDefaultTimeoutConfig().Timeout

numConsumers := 5
if enableMultipleWorkersFeatureGate.IsEnabled() {
numConsumers = 1
Expand All @@ -106,7 +105,8 @@ func createDefaultConfig() component.Config {
BackOffConfig: retrySettings,
AddMetricSuffixes: true,
SendMetadata: false,
ClientConfig: clientConfig,
ClientConfig: newDefaultPRWClientConfig(),
PRWClient: nil,
// TODO(jbd): Adjust the default queue size.
RemoteWriteQueue: RemoteWriteQueue{
Enabled: true,
Expand Down
5 changes: 5 additions & 0 deletions exporter/prometheusremotewriteexporter/testdata/config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ prometheusremotewrite/2:
queue_size: 2000
num_consumers: 10

prometheusremotewrite/prw_client:
prw_client:
endpoint: "localhost:8888"
timeout: "11s"

prometheusremotewrite/negative_queue_size:
endpoint: "localhost:8888"
remote_write_queue:
Expand Down

0 comments on commit de2388c

Please sign in to comment.