3.0.0-beta.1 release (#1134)

Signed-off-by: Sriram <153843+yesoreyeram@users.noreply.github.com>
Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
Co-authored-by: Ward Bekker <ward@equanimity.nl>

CHANGELOG.md

@@ -1,8 +1,13 @@
 # Change Log
 
-## 2.12.2
+## 3.0.0-beta.1
+
+🚀 **New Feature**: Support for passing grafana meta data such as user id, datasource uid to the underlying API as headers / query params via datasource settings
+🚀 **Improvements**: Added support for gzip compression for outgoing requests by default. Fixes [#1003](https://github.com/grafana/grafana-infinity-datasource/issues/1003)
+🚀 **Improvements**: Added frame type to dataplane compliant numeric data frames. This will help us to handle the results correctly in alerts, recorded queries, SSE etc.
+🎉 **Chore**: BREAKING: Plugin now requires Grafana 10.4.8 or newer
 
-### Patch Changes
+## 2.12.2
 
 🐛 Build and publish pipelines uses latest go lang version `1.23.5` which includes security fixes to the `crypto/x509` and `net/http` packages ( CVE-2024-45341 and CVE-2024-45336 ). More details can be found [here](https://groups.google.com/g/golang-announce/c/sSaUhLA-2SI)
 

cspell.config.json

@@ -42,6 +42,7 @@
     "countif",
     "csvframer",
     "dataframe",
+    "dataplane",
     "datapoints",
     "dataproxy",
     "datasource",

docs/sources/references/url.md

@@ -58,6 +58,24 @@ You can configure the headers required for the URL in the datasource config and
 
 Note: We suggest adding secure headers only via configuration and not in query.
 
+## Forwarding Grafana meta data as headers / query params
+
+From Infinity plugin version 3.0.0, You will be able to forward grafana meta data such as user id, datasource uid to the outgoing requests via **Custom HTTP Headers** / **URL Query parameters\*** from the datasource settings page. In the datasource **URL** section, you can add any number of custom headers / query parameters with their own values. The values can include following macros which will be interpolated into actual value from the request context.
+
+| Macro name            | Description                                                         |
+| --------------------- | ------------------------------------------------------------------- |
+| `${__org.id}`         | This will be replaced by grafana org id where the request came from |
+| `${__plugin.id}`      | This will be replaced by the plugin id                              |
+| `${__plugin.version}` | This will be replaced by the plugin version                         |
+| `${__ds.uid}`         | This will be replaced by the datasource uid                         |
+| `${__ds.name}`        | This will be replaced by the datasource name                        |
+| `${__ds.id}`          | This will be replaced by the datasource id (deprecated)             |
+| `${__user.login}`     | This will be replaced by the user login id                          |
+| `${__user.email}`     | This will be replaced by the user login email                       |
+| `${__user.name}`      | This will be replaced by the user name                              |
+
+> Note: Certain macros such as `${__user.login}` won't be available in the context of alerts, recorded queries, public dashboards etc.
+
 ## Allowed Hosts
 
 Leaving blank will allow all the hosts. This is by default.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "grafana-infinity-datasource",
-  "version": "2.12.2",
+  "version": "3.0.0-beta.1",
   "description": "JSON, CSV, XML, GraphQL, HTML and REST API datasource for Grafana. Do infinite things with Grafana. Transform data with UQL/GROQ. Visualize data from many apis, RSS/ATOM feeds directly",
   "keywords": [
     "grafana",
@@ -56,10 +56,10 @@
   },
   "dependencies": {
     "@emotion/css": "11.10.6",
-    "@grafana/data": "10.3.3",
-    "@grafana/runtime": "10.3.3",
-    "@grafana/schema": "10.3.3",
-    "@grafana/ui": "10.3.3",
+    "@grafana/data": "10.4.8",
+    "@grafana/runtime": "10.4.8",
+    "@grafana/schema": "10.4.8",
+    "@grafana/ui": "10.4.8",
     "cheerio": "^1.0.0-rc.10",
     "csv-parse": "^4.12.0",
     "groq-js": "1.1.8",
@@ -94,6 +94,7 @@
     "@types/lodash": "^4.14.194",
     "@types/mathjs": "^6.0.5",
     "@types/node": "^20.8.7",
+    "@types/react": "18.2.0",
     "@types/react-router-dom": "^5.2.0",
     "@types/testing-library__jest-dom": "5.14.8",
     "@types/xml2js": "^0.4.6",

pkg/dataplane/dataplane.go

@@ -0,0 +1,102 @@
+package dataplane
+
+import "github.com/grafana/grafana-plugin-sdk-go/data"
+
+type fieldTypeCount struct {
+	nullableFields    int
+	nonNullableFields int
+	unknownFields     int
+	numericFields     int
+	boolFields        int
+	stringFields      int
+	timeFields        int
+	jsonFields        int
+	enumFields        int
+}
+
+func getFieldTypesCount(frame *data.Frame) fieldTypeCount {
+	res := fieldTypeCount{
+		nullableFields:    0,
+		nonNullableFields: 0,
+		unknownFields:     0,
+		numericFields:     0,
+		boolFields:        0,
+		stringFields:      0,
+		timeFields:        0,
+		jsonFields:        0,
+	}
+	for _, field := range frame.Fields {
+		if field == nil {
+			continue
+		}
+		if field.Nullable() {
+			res.nullableFields++
+		}
+		if !field.Nullable() {
+			res.nonNullableFields++
+		}
+		if field.Type().Numeric() {
+			res.numericFields++
+			continue
+		}
+		if field.Type().Time() {
+			res.timeFields++
+			continue
+		}
+		if field.Type().JSON() {
+			res.jsonFields++
+			continue
+		}
+		switch field.Type() {
+		case data.FieldTypeBool,
+			data.FieldTypeNullableBool:
+			res.boolFields++
+		case data.FieldTypeString,
+			data.FieldTypeNullableString:
+			res.stringFields++
+		case data.FieldTypeEnum,
+			data.FieldTypeNullableEnum:
+			res.enumFields++
+		default:
+			res.unknownFields++
+		}
+	}
+	return res
+}
+
+// CanBeNumericWide asserts if the data frame comply with numeric wide type
+// https://grafana.com/developers/dataplane/numeric#numeric-wide-format-numericwide
+func CanBeNumericWide(frame *data.Frame) bool {
+	if frame == nil {
+		return false
+	}
+	ftCount := getFieldTypesCount(frame)
+	rowLen, err := frame.RowLen()
+	if err != nil {
+		return false
+	}
+	if rowLen <= 1 && (ftCount.numericFields+ftCount.boolFields) > 0 {
+		return true
+	}
+	return false
+}
+
+// CanBeNumericLong asserts if the data frame comply with numeric long type
+// https://grafana.com/developers/dataplane/numeric#numeric-long-format-numericlong-sql-table-like
+func CanBeNumericLong(frame *data.Frame) bool {
+	if frame == nil {
+		return false
+	}
+	ftCount := getFieldTypesCount(frame)
+	rowLen, err := frame.RowLen()
+	if err != nil {
+		return false
+	}
+	if rowLen == 1 && ftCount.numericFields > 0 && ftCount.stringFields == 0 {
+		return true
+	}
+	if rowLen > 1 && ftCount.numericFields > 0 && ftCount.stringFields > 0 {
+		return true
+	}
+	return false
+}

pkg/infinity/client.go

@@ -2,6 +2,7 @@ package infinity
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"encoding/json"
 	"errors"
@@ -98,11 +99,11 @@ func replaceSect(input string, settings models.InfinitySettings, includeSect boo
 	return input
 }
 
-func (client *Client) req(ctx context.Context, url string, body io.Reader, settings models.InfinitySettings, query models.Query, requestHeaders map[string]string) (obj any, statusCode int, duration time.Duration, err error) {
+func (client *Client) req(ctx context.Context, pCtx *backend.PluginContext, url string, body io.Reader, settings models.InfinitySettings, query models.Query, requestHeaders map[string]string) (obj any, statusCode int, duration time.Duration, err error) {
 	ctx, span := tracing.DefaultTracer().Start(ctx, "client.req")
 	logger := backend.Logger.FromContext(ctx)
 	defer span.End()
-	req, err := GetRequest(ctx, settings, body, query, requestHeaders, true)
+	req, err := GetRequest(ctx, pCtx, settings, body, query, requestHeaders, true)
 	if err != nil {
 		return nil, http.StatusInternalServerError, 0, backend.DownstreamError(fmt.Errorf("error preparing request. %w", err))
 	}
@@ -145,7 +146,7 @@ func (client *Client) req(ctx context.Context, url string, body io.Reader, setti
 		// therefore any incoming error is considered downstream
 		return nil, res.StatusCode, duration, backend.DownstreamError(err)
 	}
-	bodyBytes, err := io.ReadAll(res.Body)
+	bodyBytes, err := getBodyBytes(res)
 	if err != nil {
 		logger.Debug("error reading response body", "url", url, "error", err.Error())
 		return nil, res.StatusCode, duration, backend.DownstreamError(err)
@@ -164,12 +165,24 @@ func (client *Client) req(ctx context.Context, url string, body io.Reader, setti
 	return string(bodyBytes), res.StatusCode, duration, err
 }
 
+func getBodyBytes(res *http.Response) ([]byte, error) {
+	if strings.EqualFold(res.Header.Get("Content-Encoding"), "gzip") {
+		reader, err := gzip.NewReader(res.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
+		return io.ReadAll(reader)
+	}
+	return io.ReadAll(res.Body)
+}
+
 // https://stackoverflow.com/questions/31398044/got-error-invalid-character-%C3%AF-looking-for-beginning-of-value-from-json-unmar
 func removeBOMContent(input []byte) []byte {
 	return bytes.TrimPrefix(input, []byte("\xef\xbb\xbf"))
 }
 
-func (client *Client) GetResults(ctx context.Context, query models.Query, requestHeaders map[string]string) (o any, statusCode int, duration time.Duration, err error) {
+func (client *Client) GetResults(ctx context.Context, pCtx *backend.PluginContext, query models.Query, requestHeaders map[string]string) (o any, statusCode int, duration time.Duration, err error) {
 	logger := backend.Logger.FromContext(ctx)
 	if query.Source == "azure-blob" {
 		if strings.TrimSpace(query.AzBlobContainerName) == "" || strings.TrimSpace(query.AzBlobName) == "" {
@@ -202,9 +215,9 @@ func (client *Client) GetResults(ctx context.Context, query models.Query, reques
 	switch strings.ToUpper(query.URLOptions.Method) {
 	case http.MethodPost:
 		body := GetQueryBody(ctx, query)
-		return client.req(ctx, query.URL, body, client.Settings, query, requestHeaders)
+		return client.req(ctx, pCtx, query.URL, body, client.Settings, query, requestHeaders)
 	default:
-		return client.req(ctx, query.URL, nil, client.Settings, query, requestHeaders)
+		return client.req(ctx, pCtx, query.URL, nil, client.Settings, query, requestHeaders)
 	}
 }
 

pkg/infinity/client_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/grafana/grafana-infinity-datasource/pkg/infinity"
 	"github.com/grafana/grafana-infinity-datasource/pkg/models"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -65,7 +66,8 @@ func TestInfinityClient_GetResults(t *testing.T) {
 				Settings:   tt.settings,
 				HttpClient: &http.Client{},
 			}
-			gotO, statusCode, duration, err := client.GetResults(context.Background(), tt.query, tt.requestHeaders)
+			pluginContext := &backend.PluginContext{}
+			gotO, statusCode, duration, err := client.GetResults(context.Background(), pluginContext, tt.query, tt.requestHeaders)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("GetResults() error = %v, wantErr %v", err, tt.wantErr)
 				return

pkg/infinity/headers.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/grafana/grafana-infinity-datasource/pkg/models"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/propagation"
 )
@@ -22,10 +23,11 @@ const (
 )
 
 const (
-	headerKeyAccept        = "Accept"
-	headerKeyContentType   = "Content-Type"
-	HeaderKeyAuthorization = "Authorization"
-	HeaderKeyIdToken       = "X-Id-Token"
+	headerKeyAccept         = "Accept"
+	headerKeyContentType    = "Content-Type"
+	headerKeyAcceptEncoding = "Accept-Encoding"
+	HeaderKeyAuthorization  = "Authorization"
+	HeaderKeyIdToken        = "X-Id-Token"
 )
 
 func ApplyAcceptHeader(_ context.Context, query models.Query, settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
@@ -68,17 +70,20 @@ func ApplyContentTypeHeader(_ context.Context, query models.Query, settings mode
 	return req
 }
 
-func ApplyHeadersFromSettings(_ context.Context, settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
+func ApplyAcceptEncodingHeader(_ context.Context, query models.Query, settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
+	req.Header.Set(headerKeyAcceptEncoding, "gzip")
+	return req
+}
+
+func ApplyHeadersFromSettings(_ context.Context, pCtx *backend.PluginContext, requestHeaders map[string]string, settings models.InfinitySettings, req *http.Request, includeSect bool) *http.Request {
 	for key, value := range settings.CustomHeaders {
-		val := dummyHeader
+		headerValue := dummyHeader
 		if includeSect {
-			val = value
+			headerValue = value
 		}
+		headerValue = interpolateGrafanaMetaDataMacros(headerValue, pCtx)
 		if key != "" {
-			req.Header.Add(key, val)
-			if strings.EqualFold(key, headerKeyAccept) || strings.EqualFold(key, headerKeyContentType) {
-				req.Header.Set(key, val)
-			}
+			req.Header.Set(key, headerValue)
 		}
 	}
 	return req