feat: several changes (#53)

- introduced e2e test framework
- fixed error in CRD: when RequireAuthentication is false AuthenticationMethod is optional
- improved handling of configmaps
- improved readme

README.md

@@ -63,6 +63,8 @@ spec:
 ```
 If the field `metricType` is set to `cost`, then the API in `url` must expose a FOCUS report in a CSV file. Otherwise, if set to `resource`, it must expose usage metrics according to the JSON/OPENAPI schema in the folder resources and the field `additionalVariables` must contain a field `ResourceId` with the identifier of the resources to be used in the database as external key to reference the cost metric from the usage metric (i.e., the same as the field `resourceId` of the focusConfig CR).
 
+The field `spec.scraperConfig.url` can be left empty if the exporter and scraper are both configured. The operator will compile this field automatically.
+
 The CR can be configured to include a `provider`, which is an object reference to a set of CRs that identify, for a given provider, which resources and which additional metrics should be exported and scraped. For example, for the CPU usage of virtual machines on Azure:
 ```yaml
 apiVersion: finops.krateo.io/v1
@@ -105,6 +107,11 @@ To start the exporting process, see the examples section. The configuration samp
 
 The exporter container is created in the namespace of the CR. The exporter container looks for a secret in the CR namespace called `registry-credentials`, configurable in the HELM chart.
 
+The FOCUS data needs to be in the CSV format and the `Tags` column has to use the following format:
+```
+{"CostCenter": "1234","Cost department": "Marketing","env": "prod","org": "trey","Project": "Foo"}
+```
+
 ### Prerequisites
 - go version v1.21.0+
 - docker version 17.03+.

config/crd/bases/finops.krateo.io_exporterscraperconfigs.yaml

@@ -46,7 +46,10 @@ spec:
                       type: string
                     type: object
                   authenticationMethod:
-                    pattern: ^(\bcert-file\b)|(\bbearer-token\b)$
+                    enum:
+                    - cert-file
+                    - bearer-token
+                    - ""
                     type: string
                   bearerToken:
                     description: A SecretKeySelector is a reference to a secret key
@@ -93,6 +96,11 @@ spec:
                 - requireAuthentication
                 - url
                 type: object
+                x-kubernetes-validations:
+                - message: AuthenticationMethod must not be empty when RequireAuthentication
+                    is true.
+                  rule: '!self.requireAuthentication || self.authenticationMethod
+                    != '''''
               scraperConfig:
                 properties:
                   metricType:

go.mod

@@ -5,17 +5,24 @@ go 1.23.0
 toolchain go1.23.2
 
 require (
-	github.com/krateoplatformops/finops-data-types v0.0.0-20241206125906-b334d2f82ff2
+	github.com/krateoplatformops/finops-data-types v0.0.0-20241212193013-e4acc1b215ca
 	github.com/onsi/ginkgo/v2 v2.20.0
 	github.com/onsi/gomega v1.34.1
 	k8s.io/apimachinery v0.31.3
-	k8s.io/client-go v0.31.0
+	k8s.io/client-go v0.31.1
 	sigs.k8s.io/controller-runtime v0.19.0
 )
 
 require (
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/moby/spdystream v0.4.0 // indirect
+	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
+	github.com/vladimirvivien/gexe v0.3.0 // indirect
+	k8s.io/component-base v0.31.1 // indirect
+	sigs.k8s.io/e2e-framework v0.5.0 // indirect
 )
 
 require (
@@ -46,7 +53,7 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -1,5 +1,7 @@
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -48,8 +50,12 @@ github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQu
 github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM=
+github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -70,6 +76,8 @@ github.com/krateoplatformops/finops-data-types v0.0.0-20241206125201-5221ea29211
 github.com/krateoplatformops/finops-data-types v0.0.0-20241206125201-5221ea292114/go.mod h1:T1nMm8Neu4S9ljh7jSip9bqWR4g6zlkheCYgF1ZU/18=
 github.com/krateoplatformops/finops-data-types v0.0.0-20241206125906-b334d2f82ff2 h1:EQRYODQYLBYqHvMbkJkDuft+vWr86/Ld5F4Arhbs3gs=
 github.com/krateoplatformops/finops-data-types v0.0.0-20241206125906-b334d2f82ff2/go.mod h1:T1nMm8Neu4S9ljh7jSip9bqWR4g6zlkheCYgF1ZU/18=
+github.com/krateoplatformops/finops-data-types v0.0.0-20241212193013-e4acc1b215ca h1:Vo/SeLu5l26lqrN+D/PP5/rcCeQqeD2CnNmjFefobf0=
+github.com/krateoplatformops/finops-data-types v0.0.0-20241212193013-e4acc1b215ca/go.mod h1:T1nMm8Neu4S9ljh7jSip9bqWR4g6zlkheCYgF1ZU/18=
 github.com/krateoplatformops/provider-runtime v0.9.0 h1:ZvgJbfmv4Zx+Z/a4sat6xF884dJa4BtUGZ+HUk4UeEg=
 github.com/krateoplatformops/provider-runtime v0.9.0/go.mod h1:A0OKDAXE9KnX1GyhZH0UpZhpn15xQANoc4KVYLsfZM0=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
@@ -82,13 +90,17 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8=
+github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/onsi/ginkgo/v2 v2.20.0 h1:PE84V2mHqoT1sglvHc8ZdQtPcwmvvt29WLEEO3xmdZw=
 github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI=
 github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
@@ -117,6 +129,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/vladimirvivien/gexe v0.3.0 h1:4xwiOwGrDob5OMR6E92B9olDXYDglXdHhzR1ggYtWJM=
+github.com/vladimirvivien/gexe v0.3.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -199,6 +213,10 @@ k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4=
 k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
 k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=
 k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=
+k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
+k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
+k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8=
+k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/kube-openapi v0.0.0-20240822171749-76de80e0abd9 h1:y+4z/s0h3R97P/o/098DSjlpyNpHzGirNPlTL+GHdqY=
@@ -207,6 +225,8 @@ k8s.io/utils v0.0.0-20240821151609-f90d01438635 h1:2wThSvJoW/Ncn9TmQEYXRnevZXi2d
 k8s.io/utils v0.0.0-20240821151609-f90d01438635/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=
 sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
+sigs.k8s.io/e2e-framework v0.5.0 h1:YLhk8R7EHuTFQAe6Fxy5eBzn5Vb+yamR5u8MH1Rq3cE=
+sigs.k8s.io/e2e-framework v0.5.0/go.mod h1:jJSH8u2RNmruekUZgHAtmRjb5Wj67GErli9UjLSY7Zc=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=

internal/helpers/kube/comparators/comparators.go

@@ -8,6 +8,7 @@ import (
 	"gopkg.in/yaml.v2"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func CheckService(service corev1.Service, exporterScraperConfig finopsv1.ExporterScraperConfig) bool {
@@ -17,6 +18,7 @@ func CheckService(service corev1.Service, exporterScraperConfig finopsv1.Exporte
 
 	ownerReferencesLive := service.OwnerReferences
 	if len(ownerReferencesLive) != 1 {
+		log.Logger.Debug().Msg("Owner reference length not one")
 		return false
 	}
 
@@ -25,10 +27,12 @@ func CheckService(service corev1.Service, exporterScraperConfig finopsv1.Exporte
 	}
 
 	if service.Spec.Type != corev1.ServiceTypeNodePort {
+		log.Logger.Debug().Msg("service type wrong")
 		return false
 	}
 
 	if len(service.Spec.Ports) == 0 {
+		log.Logger.Debug().Msg("service ports number wrong")
 		return false
 	} else {
 		found := false
@@ -38,6 +42,7 @@ func CheckService(service corev1.Service, exporterScraperConfig finopsv1.Exporte
 			}
 		}
 		if !found {
+			log.Logger.Debug().Msg("service port range wrong")
 			return false
 		}
 	}
@@ -46,44 +51,57 @@ func CheckService(service corev1.Service, exporterScraperConfig finopsv1.Exporte
 		ownerReferencesLive[0].Name != exporterScraperConfig.Name ||
 		ownerReferencesLive[0].UID != exporterScraperConfig.UID ||
 		ownerReferencesLive[0].APIVersion != exporterScraperConfig.APIVersion {
+		log.Logger.Debug().Msg("Owner reference wrong")
 		return false
 	}
 
 	return true
 }
 
-func CheckConfigMap(configMap corev1.ConfigMap, exporterScraperConfig finopsv1.ExporterScraperConfig) bool {
-	if configMap.Name != exporterScraperConfig.Name+"-configmap" {
-		log.Logger.Info().Msg("name wrong")
+func CheckConfigMap(configMap corev1.ConfigMap, exporterScraperConfigComplete finopsv1.ExporterScraperConfig) bool {
+	if configMap.Name != exporterScraperConfigComplete.Name+"-configmap" {
+		log.Logger.Debug().Msg("name wrong")
 		return false
 	}
 
+	exporterScraperConfig := &finopsv1.ExporterScraperConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      exporterScraperConfigComplete.ObjectMeta.Name,
+			Namespace: exporterScraperConfigComplete.ObjectMeta.Namespace,
+			UID:       exporterScraperConfigComplete.ObjectMeta.UID,
+		},
+		TypeMeta: exporterScraperConfigComplete.TypeMeta,
+		Spec:     exporterScraperConfigComplete.Spec,
+	}
+
 	yamlData, err := yaml.Marshal(exporterScraperConfig)
 	if err != nil {
 		return false
 	}
 
 	if yamlDataFromLive, ok := configMap.BinaryData["config.yaml"]; ok {
 		if strings.Replace(string(yamlData), " ", "", -1) != strings.Replace(string(yamlDataFromLive), " ", "", -1) {
-			log.Logger.Info().Msg("bytes different")
+			log.Logger.Debug().Msg("bytes different")
+			log.Logger.Debug().Msg(string(yamlDataFromLive))
+			log.Logger.Debug().Msg(string(yamlData))
 			return false
 		}
 	} else {
-		log.Logger.Info().Msg("config.yaml not in map")
+		log.Logger.Debug().Msg("config.yaml not in map")
 		return false
 	}
 
 	ownerReferencesLive := configMap.OwnerReferences
 	if len(ownerReferencesLive) != 1 {
-		log.Logger.Info().Msg("owner len not 1")
+		log.Logger.Debug().Msg("Owner reference length not one")
 		return false
 	}
 
 	if ownerReferencesLive[0].Kind != exporterScraperConfig.Kind ||
 		ownerReferencesLive[0].Name != exporterScraperConfig.Name ||
 		ownerReferencesLive[0].UID != exporterScraperConfig.UID ||
 		ownerReferencesLive[0].APIVersion != exporterScraperConfig.APIVersion {
-		log.Logger.Info().Msg("owner wrong")
+		log.Logger.Debug().Msg("owner reference wrong")
 		return false
 	}
 

internal/utils/utils.go

@@ -94,7 +94,16 @@ func GetGenericExporterDeployment(exporterScraperConfig *finopsv1.ExporterScrape
 	return deployment, nil
 }
 
-func GetGenericExporterConfigMap(exporterScraperConfig *finopsv1.ExporterScraperConfig) (*corev1.ConfigMap, error) {
+func GetGenericExporterConfigMap(exporterScraperConfigComplete *finopsv1.ExporterScraperConfig) (*corev1.ConfigMap, error) {
+	exporterScraperConfig := &finopsv1.ExporterScraperConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      exporterScraperConfigComplete.ObjectMeta.Name,
+			Namespace: exporterScraperConfigComplete.ObjectMeta.Namespace,
+			UID:       exporterScraperConfigComplete.ObjectMeta.UID,
+		},
+		TypeMeta: exporterScraperConfigComplete.TypeMeta,
+		Spec:     exporterScraperConfigComplete.Spec,
+	}
 	yamlData, err := yaml.Marshal(exporterScraperConfig)
 	if err != nil {
 		return &corev1.ConfigMap{}, err