diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index fde18e4ab6..91d8201a33 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -158,6 +158,36 @@ func InitializeMetrics() { utils.InterfaceName, ) + ConntrackPacketsForward = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackPacketsForwardGaugeName, + ConntrackPacketForwardDescription, + ) + + ConntrackPacketsReply = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackPacketsReplyGaugeName, + ConntrackPacketReplyDescription, + ) + + ConntrackBytesForward = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackBytesForwardGaugeName, + ConntrackBytesForwardDescription, + ) + + ConntrackBytesReply = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackBytesReplyGaugeName, + ConntrackBytesReplyDescription, + ) + + ConntrackTotalConnections = exporter.CreatePrometheusGaugeVecForMetric( + exporter.DefaultRegistry, + utils.ConntrackTotalConnectionsName, + ConntrackTotalConnectionsDescription, + ) + isInitialized = true metricsLogger.Info("Metrics initialized") } diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index cff5c28884..d934d0338e 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -43,6 +43,13 @@ const ( // Control plane metrics pluginManagerFailedToReconcileCounterDescription = "Number of times the plugin manager failed to reconcile the plugins" lostEventsCounterDescription = "Number of events lost in control plane" + + // Conntrack metrics + ConntrackPacketForwardDescription = "Number of forward packets" + ConntrackPacketReplyDescription = "Number of reply packets" + ConntrackBytesForwardDescription = "Number of forward bytes" + ConntrackBytesReplyDescription = "Number of reply bytes" + ConntrackTotalConnectionsDescription = "Total number of connections" ) // Metric Counters @@ -89,6 +96,13 @@ var ( InfinibandStatsGauge GaugeVec InfinibandStatusParamsGauge GaugeVec + + // Conntrack + ConntrackPacketsForward GaugeVec + ConntrackPacketsReply GaugeVec + ConntrackBytesForward GaugeVec + ConntrackBytesReply GaugeVec + ConntrackTotalConnections GaugeVec ) func ToPrometheusType(metric interface{}) prometheus.Collector { diff --git a/pkg/plugin/conntrack/conntrack_linux.go b/pkg/plugin/conntrack/conntrack_linux.go index dc2fb0c449..fb61f33f9b 100644 --- a/pkg/plugin/conntrack/conntrack_linux.go +++ b/pkg/plugin/conntrack/conntrack_linux.go @@ -16,6 +16,7 @@ import ( "github.com/microsoft/retina/internal/ktime" "github.com/microsoft/retina/pkg/loader" "github.com/microsoft/retina/pkg/log" + "github.com/microsoft/retina/pkg/metrics" plugincommon "github.com/microsoft/retina/pkg/plugin/common" _ "github.com/microsoft/retina/pkg/plugin/conntrack/_cprog" // nolint // This is needed so cprog is included when vendoring "github.com/microsoft/retina/pkg/utils" @@ -23,6 +24,8 @@ import ( "go.uber.org/zap" ) +var conntrackMetricsEnabled = false // conntrack metrics global variable + //go:generate go run github.com/cilium/ebpf/cmd/bpf2go@master -cflags "-g -O2 -Wall -D__TARGET_ARCH_${GOARCH} -Wall" -target ${GOARCH} -type ct_v4_key conntrack ./_cprog/conntrack.c -- -I../lib/_${GOARCH} -I../lib/common/libbpf/_src -I../lib/common/libbpf/_include/linux -I../lib/common/libbpf/_include/uapi/linux -I../lib/common/libbpf/_include/asm // Init initializes the conntrack eBPF map in the kernel for the first time. @@ -88,6 +91,10 @@ func GenerateDynamic(ctx context.Context, dynamicHeaderPath string, conntrackMet if err != nil { return errors.Wrap(err, "failed to write conntrack dynamic header") } + // set a global variable + if conntrackMetrics == 1 { + conntrackMetricsEnabled = true + } return nil } @@ -118,6 +125,10 @@ func (ct *Conntrack) Run(ctx context.Context) error { // List of keys to be deleted var keysToDelete []conntrackCtV4Key + // metrics counters + var packetsCountForward, packetsCountReply, totConnections uint32 + var bytesCountForward, bytesCountReply uint64 + iter := ct.ctMap.Iterate() for iter.Next(&key, &value) { noOfCtEntries++ @@ -133,6 +144,18 @@ func (ct *Conntrack) Run(ctx context.Context) error { dstIP := utils.Int2ip(key.DstIp).To4() sourcePortShort := uint32(utils.HostToNetShort(key.SrcPort)) destinationPortShort := uint32(utils.HostToNetShort(key.DstPort)) + + // Add conntrack metrics. + if conntrackMetricsEnabled { + // Basic metrics, node-level + ctMeta := value.ConntrackMetadata + totConnections++ + packetsCountForward += ctMeta.PacketsForwardCount + packetsCountReply += ctMeta.PacketsReplyCount + bytesCountForward += ctMeta.BytesForwardCount + bytesCountReply += ctMeta.BytesReplyCount + } + ct.l.Debug("conntrack entry", zap.String("src_ip", srcIP.String()), zap.Uint32("src_port", sourcePortShort), @@ -151,6 +174,16 @@ func (ct *Conntrack) Run(ctx context.Context) error { if err := iter.Err(); err != nil { ct.l.Error("Iterate failed", zap.Error(err)) } + + // create metrics + if conntrackMetricsEnabled { + metrics.ConntrackPacketsForward.WithLabelValues().Set(float64(packetsCountForward)) + metrics.ConntrackBytesForward.WithLabelValues().Set(float64(bytesCountForward)) + metrics.ConntrackPacketsReply.WithLabelValues().Set(float64(packetsCountReply)) + metrics.ConntrackBytesReply.WithLabelValues().Set(float64(bytesCountReply)) + metrics.ConntrackTotalConnections.WithLabelValues().Set(float64(totConnections)) + } + // Delete the conntrack entries for _, key := range keysToDelete { if err := ct.ctMap.Delete(key); err != nil { diff --git a/pkg/utils/metric_names.go b/pkg/utils/metric_names.go index 8060b05243..6d949dc8a0 100644 --- a/pkg/utils/metric_names.go +++ b/pkg/utils/metric_names.go @@ -34,6 +34,13 @@ const ( // Common Gauges across os distributions NodeConnectivityStatusName = "node_connectivity_status" NodeConnectivityLatencySecondsName = "node_connectivity_latency_seconds" + + // Conntrack + ConntrackPacketsForwardGaugeName = "conntrack_packets_forward" + ConntrackPacketsReplyGaugeName = "conntrack_packets_reply" + ConntrackBytesForwardGaugeName = "conntrack_bytes_forward" + ConntrackBytesReplyGaugeName = "conntrack_bytes_reply" + ConntrackTotalConnectionsName = "conntrack_total_connections" ) // IsAdvancedMetric is a helper function to determine if a name is an advanced metric