Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mindev: Add utility to generate a data source definition from a Swagger doc #5283

Merged
merged 3 commits into from
Jan 14, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions cmd/dev/app/datasource/datasource.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// SPDX-FileCopyrightText: Copyright 2025 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

// Package datasource provides the root command for the datasource subcommands
package datasource

import "github.com/spf13/cobra"

// CmdDataSource is the root command for the datasource subcommands
func CmdDataSource() *cobra.Command {
var rtCmd = &cobra.Command{
Use: "datasource",
Short: "datasource provides utilities for testing and working with data sources",
}

rtCmd.AddCommand(CmdGenerate())

return rtCmd
}
233 changes: 233 additions & 0 deletions cmd/dev/app/datasource/generate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
// SPDX-FileCopyrightText: Copyright 2025 The Minder Authors
// SPDX-License-Identifier: Apache-2.0

package datasource

import (
"fmt"
"net/http"
"net/url"
"os"
"regexp"
"strings"

"buf.build/go/protoyaml"
"github.com/go-openapi/loads"
"github.com/go-openapi/spec"
"github.com/spf13/cobra"
"google.golang.org/protobuf/types/known/structpb"

minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1"
)

// CmdGenerate returns a cobra command for the 'datasource generate' subcommand.
func CmdGenerate() *cobra.Command {
var generateCmd = &cobra.Command{
Use: "generate",
Aliases: []string{"gen"},
Short: "generate datasource code from an OpenAPI specification",
Long: `The 'datasource generate' subcommand allows you to generate datasource code from an OpenAPI
specification`,
RunE: generateCmdRun,
Args: cobra.ExactArgs(1),
SilenceUsage: true,
}

return generateCmd
}

// parseOpenAPI parses an OpenAPI specification from a byte slice.
func parseOpenAPI(filepath string) (*spec.Swagger, error) {
doc, err := loads.Spec(filepath)
if err != nil {
return nil, fmt.Errorf("failed to parse OpenAPI spec: %w", err)
}

return doc.Spec(), nil
}

func initDataSourceStruct(name string) *minderv1.DataSource {
return &minderv1.DataSource{
Version: minderv1.VersionV1,
Type: "data-source",
Name: name,
Context: &minderv1.ContextV2{},
}
}

func initDriverStruct() *minderv1.RestDataSource {
return &minderv1.RestDataSource{
Def: make(map[string]*minderv1.RestDataSource_Def),
}
}

// conver the title to a valid datasource name. It should only contain alphanumeric characters and dashes.
func swaggerTitleToDataSourceName(title string) string {
re := regexp.MustCompile("^[a-z][-_[:word:]]*$")
return re.ReplaceAllString(title, "-")
}

// swaggerToDataSource generates datasource code from an OpenAPI specification.
func swaggerToDataSource(cmd *cobra.Command, swagger *spec.Swagger) error {
if swagger.Info == nil {
return fmt.Errorf("info section is required in OpenAPI spec")
}

ds := initDataSourceStruct(swaggerTitleToDataSourceName(swagger.Info.Title))
drv := initDriverStruct()
ds.Driver = &minderv1.DataSource_Rest{Rest: drv}

// Add the OpenAPI specification to the DataSource
basepath := swagger.BasePath
if basepath == "" {
return fmt.Errorf("base path is required in OpenAPI spec")
}

for path, pathItem := range swagger.Paths.Paths {
p, err := url.JoinPath(basepath, path)
if err != nil {
cmd.PrintErrf("error joining path %s and basepath %s: %v\n Skipping", path, basepath, err)
continue
}

for method, op := range operations(pathItem) {
opName := generateOpName(method, path)
// Create a new REST DataSource definition
def := &minderv1.RestDataSource_Def{
Method: method,
Endpoint: p,
// TODO: Make this configurable
Parse: "json",
}

is := paramsToInputSchema(op.Parameters)

if requiresMsgBody(method) {
def.Body = &minderv1.RestDataSource_Def_BodyFromField{
BodyFromField: "body",
}

// Add the `body` field to the input schema
is = inputSchemaForBody(is)
}

pbs, err := structpb.NewStruct(is)
if err != nil {
return fmt.Errorf("error creating input schema: %w", err)
}

def.InputSchema = pbs

// Add the operation to the DataSource
drv.Def[opName] = def
}
}

return writeDataSourceToFile(ds)
}

// Generates an operation name for a data source. Note that these names
// must be unique within a data source. They also should be only alphanumeric
// characters and underscores
func generateOpName(method, path string) string {
// Replace all non-alphanumeric characters with underscores
re := regexp.MustCompile("[^a-zA-Z0-9]+")
return re.ReplaceAllString(fmt.Sprintf("%s_%s", strings.ToLower(method), strings.ToLower(path)), "_")
}

func operations(p spec.PathItem) map[string]*spec.Operation {
out := make(map[string]*spec.Operation)
for mstr, op := range map[string]*spec.Operation{
http.MethodGet: p.Get,
http.MethodPut: p.Put,
http.MethodPost: p.Post,
http.MethodDelete: p.Delete,
http.MethodOptions: p.Options,
http.MethodHead: p.Head,
http.MethodPatch: p.Patch,
} {
if op != nil {
out[mstr] = op
}
}

return out
}

func requiresMsgBody(method string) bool {
return method == http.MethodPost || method == http.MethodPut || method == http.MethodPatch
}

func paramsToInputSchema(params []spec.Parameter) map[string]any {
if len(params) == 0 {
return nil
}

is := map[string]any{
"type": "object",
"properties": make(map[string]any),
}

for _, p := range params {
is["properties"].(map[string]any)[p.Name] = map[string]any{
// TODO: Add support for more types
"type": "string",
}

if p.Required {
if _, ok := is["required"]; !ok {
is["required"] = make([]string, 0)
}

is["required"] = append(is["required"].([]string), p.Name)
}
}

return is
}

func inputSchemaForBody(is map[string]any) map[string]any {
if is == nil {
is = map[string]any{
"type": "object",
"properties": make(map[string]any),
}
}

is["properties"].(map[string]any)["body"] = map[string]any{
"type": "object",
}

return is
}

func writeDataSourceToFile(ds *minderv1.DataSource) error {
// Convert the DataSource to YAML
dsYAML, err := protoyaml.MarshalOptions{
Indent: 2,
}.Marshal(ds)
if err != nil {
return fmt.Errorf("error marshalling DataSource to YAML: %w", err)
}

// Write the YAML to a file
if _, err := os.Stdout.Write(dsYAML); err != nil {
return fmt.Errorf("error writing DataSource to file: %w", err)
}

return nil
}

// generateCmdRun is the entry point for the 'datasource generate' command.
func generateCmdRun(cmd *cobra.Command, args []string) error {
// We've already validated that there is exactly one argument via the cobra.ExactArgs(1) call
filePath := args[0]

// Parse the OpenAPI specification
swagger, err := parseOpenAPI(filePath)
if err != nil {
return fmt.Errorf("error parsing OpenAPI spec: %w", err)
}

return swaggerToDataSource(cmd, swagger)
}
2 changes: 2 additions & 0 deletions cmd/dev/app/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/spf13/cobra"

"github.com/mindersec/minder/cmd/dev/app/bundles"
"github.com/mindersec/minder/cmd/dev/app/datasource"
"github.com/mindersec/minder/cmd/dev/app/image"
"github.com/mindersec/minder/cmd/dev/app/rule_type"
"github.com/mindersec/minder/cmd/dev/app/testserver"
Expand All @@ -29,6 +30,7 @@ https://docs.stacklok.com/minder`,
cmd.AddCommand(image.CmdImage())
cmd.AddCommand(testserver.CmdTestServer())
cmd.AddCommand(bundles.CmdBundle())
cmd.AddCommand(datasource.CmdDataSource())

return cmd
}
Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.23.4

require (
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1
buf.build/go/protoyaml v0.3.1
github.com/ThreeDotsLabs/watermill v1.4.2
github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0
github.com/alexdrl/zerowater v0.0.3
Expand Down Expand Up @@ -306,9 +307,9 @@ require (
github.com/go-openapi/errors v0.22.0 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/loads v0.22.0 // indirect
github.com/go-openapi/loads v0.22.0
github.com/go-openapi/runtime v0.28.0 // indirect
github.com/go-openapi/spec v0.21.0 // indirect
github.com/go-openapi/spec v0.21.0
github.com/go-openapi/strfmt v0.23.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-openapi/validate v0.24.0 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1 h1:BICM6du/XzvEgeorNo4xgohK3nMTmEPViGyd5t7xVqk=
buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.2-20241127180247-a33202765966.1/go.mod h1:JnMVLi3qrNYPODVpEKG7UjHLl/d2zR221e66YCSmP2Q=
buf.build/go/protoyaml v0.3.1 h1:ucyzE7DRnjX+mQ6AH4JzN0Kg50ByHHu+yrSKbgQn2D4=
buf.build/go/protoyaml v0.3.1/go.mod h1:0TzNpFQDXhwbkXb/ajLvxIijqbve+vMQvWY/b3/Dzxg=
cel.dev/expr v0.18.0 h1:CJ6drgk+Hf96lkLikr4rFf19WrU0BOWEihyZnI2TAzo=
cel.dev/expr v0.18.0/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
Expand Down
Loading