Skip to content

Commit

Permalink
Allow passing extra service args (#99)
Browse files Browse the repository at this point in the history 
* Allow passing extra service args

The CAPI provider already allows passing extra kube-api args.

We'll do the same for the other k8s services (e.g. kubelet,
kube-proxy, containerd, dqlite, etc), exposing the corresponding
k8s-snap settings.

Fixes: #91

* Fix formatting
  • Loading branch information
@petrutlucian94
petrutlucian94 authored Jan 31, 2025
1 parent 7b816b3 commit 91e4fe7
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 11 deletions.
30 changes: 29 additions & 1 deletion bootstrap/api/v1beta2/ck8sconfig_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,22 @@ type CK8sConfigSpec struct {
// typically set in Jinja template form, e.g."{{ ds.meta_data.local_hostname }}".
// +optional
NodeName string `json:"nodeName,omitempty"`

// ExtraKubeProxyArgs - extra arguments to add to kube-proxy.
// +optional
ExtraKubeProxyArgs map[string]*string `json:"extraKubeProxyArgs,omitempty"`

// ExtraKubeletArgs - extra arguments to add to kubelet.
// +optional
ExtraKubeletArgs map[string]*string `json:"extraKubeletArgs,omitempty"`

// ExtraContainerdArgs - extra arguments to add to containerd.
// +optional
ExtraContainerdArgs map[string]*string `json:"extraContainerdArgs,omitempty"`

// ExtraK8sAPIServerProxyArgs - extra arguments to add to k8s-api-server-proxy.
// +optional
ExtraK8sAPIServerProxyArgs map[string]*string `json:"ExtraK8sAPIServerProxyArgs,omitempty"`
}

// IsEtcdManaged returns true if the control plane is using k8s-dqlite.
Expand Down Expand Up @@ -152,9 +168,21 @@ type CK8sControlPlaneConfig struct {
// +optional
MicroclusterPort *int `json:"microclusterPort,omitempty"`

// ExtraKubeAPIServerArgs is extra arguments to add to kube-apiserver.
// ExtraKubeAPIServerArgs - extra arguments to add to kube-apiserver.
// +optional
ExtraKubeAPIServerArgs map[string]*string `json:"extraKubeAPIServerArgs,omitempty"`

// ExtraKubeControllerManagerArgs - extra arguments to add to kube-controller-manager.
// +optional
ExtraKubeControllerManagerArgs map[string]*string `json:"extraKubeControllerManagerArgs,omitempty"`

// ExtraKubeSchedulerArgs - extra arguments to add to kube-scheduler.
// +optional
ExtraKubeSchedulerArgs map[string]*string `json:"extraKubeSchedulerArgs,omitempty"`

// ExtraK8sDqliteArgs - extra arguments to add to k8s-dqlite.
// +optional
ExtraK8sDqliteArgs map[string]*string `json:"ExtraK8sDqliteArgs,omitempty"`
}

// GetMicroclusterPort returns the port to use for microcluster.
Expand Down
42 changes: 40 additions & 2 deletions bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ spec:
spec:
description: CK8sConfigSpec defines the desired state of CK8sConfig.
properties:
ExtraK8sAPIServerProxyArgs:
additionalProperties:
type: string
description: ExtraK8sAPIServerProxyArgs - extra arguments to add to
k8s-api-server-proxy.
type: object
airGapped:
description: |-
AirGapped is used to signal that we are deploying to an airgap environment. In this case,
Expand Down Expand Up @@ -91,6 +97,11 @@ spec:
description: CK8sControlPlaneConfig is configuration for the control
plane node.
properties:
ExtraK8sDqliteArgs:
additionalProperties:
type: string
description: ExtraK8sDqliteArgs - extra arguments to add to k8s-dqlite.
type: object
cloudProvider:
description: CloudProvider is the cloud-provider configuration
option to set.
Expand All @@ -117,8 +128,20 @@ spec:
extraKubeAPIServerArgs:
additionalProperties:
type: string
description: ExtraKubeAPIServerArgs is extra arguments to add
to kube-apiserver.
description: ExtraKubeAPIServerArgs - extra arguments to add to
kube-apiserver.
type: object
extraKubeControllerManagerArgs:
additionalProperties:
type: string
description: ExtraKubeControllerManagerArgs - extra arguments
to add to kube-controller-manager.
type: object
extraKubeSchedulerArgs:
additionalProperties:
type: string
description: ExtraKubeSchedulerArgs - extra arguments to add to
kube-scheduler.
type: object
extraSANs:
description: ExtraSANs is a list of SANs to include in the server
Expand All @@ -145,6 +168,21 @@ spec:
type: string
type: array
type: object
extraContainerdArgs:
additionalProperties:
type: string
description: ExtraContainerdArgs - extra arguments to add to containerd.
type: object
extraKubeProxyArgs:
additionalProperties:
type: string
description: ExtraKubeProxyArgs - extra arguments to add to kube-proxy.
type: object
extraKubeletArgs:
additionalProperties:
type: string
description: ExtraKubeletArgs - extra arguments to add to kubelet.
type: object
files:
description: Files specifies extra files to be passed to user_data
upon creation.
Expand Down
44 changes: 43 additions & 1 deletion bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_ck8sconfigtemplates.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ spec:
spec:
description: CK8sConfigSpec defines the desired state of CK8sConfig.
properties:
ExtraK8sAPIServerProxyArgs:
additionalProperties:
type: string
description: ExtraK8sAPIServerProxyArgs - extra arguments
to add to k8s-api-server-proxy.
type: object
airGapped:
description: |-
AirGapped is used to signal that we are deploying to an airgap environment. In this case,
Expand Down Expand Up @@ -98,6 +104,12 @@ spec:
description: CK8sControlPlaneConfig is configuration for the
control plane node.
properties:
ExtraK8sDqliteArgs:
additionalProperties:
type: string
description: ExtraK8sDqliteArgs - extra arguments to add
to k8s-dqlite.
type: object
cloudProvider:
description: CloudProvider is the cloud-provider configuration
option to set.
Expand All @@ -124,9 +136,21 @@ spec:
extraKubeAPIServerArgs:
additionalProperties:
type: string
description: ExtraKubeAPIServerArgs is extra arguments
description: ExtraKubeAPIServerArgs - extra arguments
to add to kube-apiserver.
type: object
extraKubeControllerManagerArgs:
additionalProperties:
type: string
description: ExtraKubeControllerManagerArgs - extra arguments
to add to kube-controller-manager.
type: object
extraKubeSchedulerArgs:
additionalProperties:
type: string
description: ExtraKubeSchedulerArgs - extra arguments
to add to kube-scheduler.
type: object
extraSANs:
description: ExtraSANs is a list of SANs to include in
the server certificates.
Expand All @@ -153,6 +177,24 @@ spec:
type: string
type: array
type: object
extraContainerdArgs:
additionalProperties:
type: string
description: ExtraContainerdArgs - extra arguments to add
to containerd.
type: object
extraKubeProxyArgs:
additionalProperties:
type: string
description: ExtraKubeProxyArgs - extra arguments to add to
kube-proxy.
type: object
extraKubeletArgs:
additionalProperties:
type: string
description: ExtraKubeletArgs - extra arguments to add to
kubelet.
type: object
files:
description: Files specifies extra files to be passed to user_data
upon creation.
Expand Down
16 changes: 15 additions & 1 deletion bootstrap/controllers/ck8sconfig_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,10 @@ func (r *CK8sConfigReconciler) joinControlplane(ctx context.Context, scope *Scop
configStruct := ck8s.GenerateJoinControlPlaneConfig(ck8s.JoinControlPlaneConfig{
ControlPlaneEndpoint: scope.Cluster.Spec.ControlPlaneEndpoint.Host,
ControlPlaneConfig: controlPlaneConfig,

ExtraKubeProxyArgs: scope.Config.Spec.ExtraKubeProxyArgs,
ExtraKubeletArgs: scope.Config.Spec.ExtraKubeletArgs,
ExtraContainerdArgs: scope.Config.Spec.ExtraContainerdArgs,
})
joinConfig, err := kubeyaml.Marshal(configStruct)
if err != nil {
Expand Down Expand Up @@ -344,7 +348,12 @@ func (r *CK8sConfigReconciler) joinWorker(ctx context.Context, scope *Scope) err
return fmt.Errorf("failed to request join token: %w", err)
}

configStruct := ck8s.GenerateJoinWorkerConfig(ck8s.JoinWorkerConfig{})
configStruct := ck8s.GenerateJoinWorkerConfig(ck8s.JoinWorkerConfig{
ExtraKubeProxyArgs: scope.Config.Spec.ExtraKubeProxyArgs,
ExtraKubeletArgs: scope.Config.Spec.ExtraKubeletArgs,
ExtraContainerdArgs: scope.Config.Spec.ExtraContainerdArgs,
ExtraK8sAPIServerProxyArgs: scope.Config.Spec.ExtraK8sAPIServerProxyArgs,
})
joinConfig, err := kubeyaml.Marshal(configStruct)
if err != nil {
return err
Expand Down Expand Up @@ -637,6 +646,11 @@ func (r *CK8sConfigReconciler) handleClusterNotInitialized(ctx context.Context,
InitConfig: scope.Config.Spec.InitConfig,

ClusterNetwork: scope.Cluster.Spec.ClusterNetwork,

ExtraKubeProxyArgs: scope.Config.Spec.ExtraKubeProxyArgs,
ExtraKubeletArgs: scope.Config.Spec.ExtraKubeletArgs,
ExtraContainerdArgs: scope.Config.Spec.ExtraContainerdArgs,
ExtraK8sAPIServerProxyArgs: scope.Config.Spec.ExtraK8sAPIServerProxyArgs,
}

if !scope.Config.Spec.IsEtcdManaged() {
Expand Down
43 changes: 41 additions & 2 deletions controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanes.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,12 @@ spec:
CK8sConfigSpec is a CK8sConfigSpec
to use for initializing and joining machines to the control plane.
properties:
ExtraK8sAPIServerProxyArgs:
additionalProperties:
type: string
description: ExtraK8sAPIServerProxyArgs - extra arguments to add
to k8s-api-server-proxy.
type: object
airGapped:
description: |-
AirGapped is used to signal that we are deploying to an airgap environment. In this case,
Expand Down Expand Up @@ -286,6 +292,12 @@ spec:
description: CK8sControlPlaneConfig is configuration for the control
plane node.
properties:
ExtraK8sDqliteArgs:
additionalProperties:
type: string
description: ExtraK8sDqliteArgs - extra arguments to add to
k8s-dqlite.
type: object
cloudProvider:
description: CloudProvider is the cloud-provider configuration
option to set.
Expand All @@ -312,8 +324,20 @@ spec:
extraKubeAPIServerArgs:
additionalProperties:
type: string
description: ExtraKubeAPIServerArgs is extra arguments to
add to kube-apiserver.
description: ExtraKubeAPIServerArgs - extra arguments to add
to kube-apiserver.
type: object
extraKubeControllerManagerArgs:
additionalProperties:
type: string
description: ExtraKubeControllerManagerArgs - extra arguments
to add to kube-controller-manager.
type: object
extraKubeSchedulerArgs:
additionalProperties:
type: string
description: ExtraKubeSchedulerArgs - extra arguments to add
to kube-scheduler.
type: object
extraSANs:
description: ExtraSANs is a list of SANs to include in the
Expand Down Expand Up @@ -341,6 +365,21 @@ spec:
type: string
type: array
type: object
extraContainerdArgs:
additionalProperties:
type: string
description: ExtraContainerdArgs - extra arguments to add to containerd.
type: object
extraKubeProxyArgs:
additionalProperties:
type: string
description: ExtraKubeProxyArgs - extra arguments to add to kube-proxy.
type: object
extraKubeletArgs:
additionalProperties:
type: string
description: ExtraKubeletArgs - extra arguments to add to kubelet.
type: object
files:
description: Files specifies extra files to be passed to user_data
upon creation.
Expand Down
44 changes: 43 additions & 1 deletion controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_ck8scontrolplanetemplates.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,12 @@ spec:
CK8sConfigSpec is a CK8sConfigSpec
to use for initializing and joining machines to the control plane.
properties:
ExtraK8sAPIServerProxyArgs:
additionalProperties:
type: string
description: ExtraK8sAPIServerProxyArgs - extra arguments
to add to k8s-api-server-proxy.
type: object
airGapped:
description: |-
AirGapped is used to signal that we are deploying to an airgap environment. In this case,
Expand Down Expand Up @@ -262,6 +268,12 @@ spec:
description: CK8sControlPlaneConfig is configuration for
the control plane node.
properties:
ExtraK8sDqliteArgs:
additionalProperties:
type: string
description: ExtraK8sDqliteArgs - extra arguments
to add to k8s-dqlite.
type: object
cloudProvider:
description: CloudProvider is the cloud-provider configuration
option to set.
Expand All @@ -288,9 +300,21 @@ spec:
extraKubeAPIServerArgs:
additionalProperties:
type: string
description: ExtraKubeAPIServerArgs is extra arguments
description: ExtraKubeAPIServerArgs - extra arguments
to add to kube-apiserver.
type: object
extraKubeControllerManagerArgs:
additionalProperties:
type: string
description: ExtraKubeControllerManagerArgs - extra
arguments to add to kube-controller-manager.
type: object
extraKubeSchedulerArgs:
additionalProperties:
type: string
description: ExtraKubeSchedulerArgs - extra arguments
to add to kube-scheduler.
type: object
extraSANs:
description: ExtraSANs is a list of SANs to include
in the server certificates.
Expand Down Expand Up @@ -318,6 +342,24 @@ spec:
type: string
type: array
type: object
extraContainerdArgs:
additionalProperties:
type: string
description: ExtraContainerdArgs - extra arguments to
add to containerd.
type: object
extraKubeProxyArgs:
additionalProperties:
type: string
description: ExtraKubeProxyArgs - extra arguments to add
to kube-proxy.
type: object
extraKubeletArgs:
additionalProperties:
type: string
description: ExtraKubeletArgs - extra arguments to add
to kubelet.
type: object
files:
description: Files specifies extra files to be passed
to user_data upon creation.
Expand Down
12 changes: 12 additions & 0 deletions pkg/ck8s/config_init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@ type InitControlPlaneConfig struct {
DatastoreServers []string

ClusterNetwork *clusterv1.ClusterNetwork

ExtraKubeProxyArgs map[string]*string
ExtraKubeletArgs map[string]*string
ExtraContainerdArgs map[string]*string
ExtraK8sAPIServerProxyArgs map[string]*string
}

func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.BootstrapConfig, error) {
Expand Down Expand Up @@ -131,6 +136,13 @@ func GenerateInitControlPlaneConfig(cfg InitControlPlaneConfig) (apiv1.Bootstrap
}

out.ExtraNodeKubeAPIServerArgs = cfg.ControlPlaneConfig.ExtraKubeAPIServerArgs
out.ExtraNodeKubeControllerManagerArgs = cfg.ControlPlaneConfig.ExtraKubeControllerManagerArgs
out.ExtraNodeKubeSchedulerArgs = cfg.ControlPlaneConfig.ExtraKubeSchedulerArgs
out.ExtraNodeK8sDqliteArgs = cfg.ControlPlaneConfig.ExtraK8sDqliteArgs

out.ExtraNodeKubeProxyArgs = cfg.ExtraKubeProxyArgs
out.ExtraNodeKubeletArgs = cfg.ExtraKubeletArgs
out.ExtraNodeContainerdArgs = cfg.ExtraContainerdArgs

return out, nil
}
Loading

0 comments on commit 91e4fe7

Please sign in to comment.