From 3cf82cb947ffcc5e001d6e3cade1947a6ebba944 Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Wed, 11 Sep 2024 08:54:39 +0000 Subject: [PATCH] Updated resource creation: * StatefulSet * PVC * Service * Ingress (TODO: SSH port forwarding) --- api/v1/devcontainerapp_types.go | 8 +- ...container.devstar.cn_devcontainerapps.yaml | 22 ++--- .../devcontainer_v1_devcontainerapp.yaml | 10 +- go.mod | 2 +- .../controller/devcontainerapp_controller.go | 99 ++++++++++++++++++- internal/controller/templates/ingress.yaml | 8 +- internal/controller/templates/service.yaml | 6 +- .../{stateful_set.yaml => statefulset.yaml} | 25 +++-- internal/controller/utils/template_utils.go | 2 +- 9 files changed, 142 insertions(+), 40 deletions(-) rename internal/controller/templates/{stateful_set.yaml => statefulset.yaml} (63%) diff --git a/api/v1/devcontainerapp_types.go b/api/v1/devcontainerapp_types.go index e073cb0..487435f 100644 --- a/api/v1/devcontainerapp_types.go +++ b/api/v1/devcontainerapp_types.go @@ -28,26 +28,24 @@ type DevcontainerAppSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - StatefulSet StatefulSetSpec `json:"stateful_set"` + StatefulSet StatefulSetSpec `json:"statefulset"` Service ServiceSpec `json:"service"` Ingress IngressSpec `json:"ingress"` } // StatefulSetSpec specifies StatefulSet for DevContainer type StatefulSetSpec struct { - Name string `json:"name"` Image string `json:"image"` - PVC string `json:"pvc"` } // ServiceSpec specifies Service for DevContainer type ServiceSpec struct { - Name string `json:"name"` + ContainerPort uint16 `json:"containerPort"` } // IngressSpec specifies Ingress Controller access point for DevContainer type IngressSpec struct { - Name string `json:"name"` + Port uint16 `json:"port"` } // DevcontainerAppStatus defines the observed state of DevcontainerApp diff --git a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml index b1dcadb..b58a7a2 100644 --- a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml +++ b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml @@ -43,37 +43,31 @@ spec: description: IngressSpec specifies Ingress Controller access point for DevContainer properties: - name: - type: string + port: + type: integer required: - - name + - port type: object service: description: ServiceSpec specifies Service for DevContainer properties: - name: - type: string + containerPort: + type: integer required: - - name + - containerPort type: object - stateful_set: + statefulset: description: StatefulSetSpec specifies StatefulSet for DevContainer properties: image: type: string - name: - type: string - pvc: - type: string required: - image - - name - - pvc type: object required: - ingress - service - - stateful_set + - statefulset type: object status: description: DevcontainerAppStatus defines the observed state of DevcontainerApp diff --git a/config/samples/devcontainer_v1_devcontainerapp.yaml b/config/samples/devcontainer_v1_devcontainerapp.yaml index 108b4b0..5718092 100644 --- a/config/samples/devcontainer_v1_devcontainerapp.yaml +++ b/config/samples/devcontainer_v1_devcontainerapp.yaml @@ -1,9 +1,15 @@ apiVersion: devcontainer.devstar.cn/v1 kind: DevcontainerApp metadata: + name: daimingchen-devstar-beef092a69c011ef9c00000c2952a362 + namespace: devstar-devcontainer-ns labels: app.kubernetes.io/name: devstar-devcontainer-kubebuilder-scaffold app.kubernetes.io/managed-by: kustomize - name: devcontainerapp-sample spec: - # TODO(user): Add fields here + statefulset: + image: nginx:latest + service: + containerPort: 2222 + ingress: + port: 22 diff --git a/go.mod b/go.mod index ba7356d..fc64f3f 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.0 // indirect + k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/apiserver v0.31.0 // indirect k8s.io/component-base v0.31.0 // indirect diff --git a/internal/controller/devcontainerapp_controller.go b/internal/controller/devcontainerapp_controller.go index 0b1cf30..ccc7f09 100644 --- a/internal/controller/devcontainerapp_controller.go +++ b/internal/controller/devcontainerapp_controller.go @@ -19,12 +19,20 @@ package controller import ( "context" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - devcontainerv1 "devstar.cn/DevcontainerApp/api/v1" + devcontainer_v1 "devstar.cn/DevcontainerApp/api/v1" + devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils" + app_v1 "k8s.io/api/apps/v1" + core_v1 "k8s.io/api/core/v1" + networking_v1 "k8s.io/api/networking/v1" + k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // DevcontainerAppReconciler reconciles a DevcontainerApp object @@ -39,7 +47,7 @@ type DevcontainerAppReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by +// Modify the Reconcile function to compare the state specified by // the DevcontainerApp object against the actual cluster state, and then // perform operations to make the cluster state reflect the state specified by // the user. @@ -47,16 +55,97 @@ type DevcontainerAppReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + logger := log.FromContext(ctx) - // TODO(user): your logic here + logger.Info("[Reconciler] ENTER controller.Reconcile(...)") + var err error + + // 1. 读取缓存中的 app + app := &devcontainer_v1.DevcontainerApp{} + err = r.Get(ctx, req.NamespacedName, app) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // 2. 根据 DevcontainerApp 配置信息进行处理 + // 2.1 StatefulSet 处理 + statefulSet := devcontainer_controller_utils.NewStatefulSet(app) + err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, statefulSet, r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + + // 2.2 查找 集群中同名称的 StatefulSet + statefulSetInNamespace := &app_v1.StatefulSet{} + err = r.Get(ctx, req.NamespacedName, statefulSetInNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + err = r.Create(ctx, statefulSet) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "Failed to create StatefulSet") + return ctrl.Result{}, err + } + } + // 这里会反复触发更新 + // 原因：在 SetupWithManager方法中，监听了 StatefulSet ，所以只要更新 StatefulSet 就会触发 + // 此处更新和 controllerManager 更新 StatefulSet 都会触发更新事件，导致循环触发 + //修复方法：加上判断条件，仅在 app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image 时才更新 StatefulSet + if app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image { + if err := r.Update(ctx, statefulSet); err != nil { + return ctrl.Result{}, err + } + } + + // 2.2 Service 处理 + service := devcontainer_controller_utils.NewService(app) + if err := k8s_sigs_controller_runtime_utils.SetControllerReference(app, service, r.Scheme); err != nil { + return ctrl.Result{}, err + } + serviceInNamespace := &core_v1.Service{} + err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, serviceInNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + err = r.Create(ctx, service) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "create service failed") + return ctrl.Result{}, err + } + } + + // 2.3 Ingress 处理 + ingress := devcontainer_controller_utils.NewIngress(app) + err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, ingress, r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + ingressInNamespace := &networking_v1.Ingress{} + err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, ingressInNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + err := r.Create(ctx, ingress) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "failed to create ingress") + return ctrl.Result{}, err + } + } + + logger.Info("[Reconciler] LEAVE controller.Reconcile(...)") return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&devcontainerv1.DevcontainerApp{}). + For(&devcontainer_v1.DevcontainerApp{}). + Owns(&app_v1.StatefulSet{}). + Owns(&core_v1.Service{}). + Owns(&networking_v1.Ingress{}). Complete(r) } diff --git a/internal/controller/templates/ingress.yaml b/internal/controller/templates/ingress.yaml index 4ec55fc..b2af8f1 100644 --- a/internal/controller/templates/ingress.yaml +++ b/internal/controller/templates/ingress.yaml @@ -1,10 +1,12 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{.ObjectMeta.Name}} + name: nginx-{{.ObjectMeta.Name}}-ingress namespace: {{.ObjectMeta.Namespace}} spec: ingressClassName: nginx + # TODO: create port-forwarding rule for SSH + # - https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/ rules: - host: {{.ObjectMeta.Name}}.devcontainer.devstar.cn http: @@ -13,6 +15,6 @@ spec: pathType: Prefix backend: service: - name: {{.ObjectMeta.Name}} + name: {{.ObjectMeta.Name}}-svc port: - number: 22 + number: {{.Spec.Ingress.Port}} diff --git a/internal/controller/templates/service.yaml b/internal/controller/templates/service.yaml index bbb9d76..a3d1487 100644 --- a/internal/controller/templates/service.yaml +++ b/internal/controller/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{.ObjectMeta.Name}} + name: {{.ObjectMeta.Name}}-svc namespace: {{.ObjectMeta.Namespace}} spec: selector: app: {{.ObjectMeta.Name}} ports: - name: ssh + port: {{.Spec.Ingress.Port}} + targetPort: ssh-port protocol: TCP - port: 22 - targetPort: 22 diff --git a/internal/controller/templates/stateful_set.yaml b/internal/controller/templates/statefulset.yaml similarity index 63% rename from internal/controller/templates/stateful_set.yaml rename to internal/controller/templates/statefulset.yaml index 9a7154c..ce77098 100644 --- a/internal/controller/templates/stateful_set.yaml +++ b/internal/controller/templates/statefulset.yaml @@ -18,20 +18,20 @@ spec: spec: containers: - name: {{.ObjectMeta.Name}} - image: {{.Spec.Image}} + image: {{.Spec.StatefulSet.Image}} imagePullPolicy: IfNotPresent ports: - - name: tcp-ssh - protocol: tcp - containerPort: 22 + - name: ssh-port + protocol: TCP + containerPort: {{.Spec.Service.ContainerPort}} livenessProbe: exec: command: - /bin/sh - -c - - exec ping localhost + - exec ls ~ failureThreshold: 6 - initialDelaySeconds: 30 + initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 @@ -44,3 +44,16 @@ spec: cpu: 100m ephemeral-storage: 50Mi memory: 128Mi + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: devstar-devcontainer-pvc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: local + volumeMode: Filesystem diff --git a/internal/controller/utils/template_utils.go b/internal/controller/utils/template_utils.go index e57c3c3..41f246b 100644 --- a/internal/controller/utils/template_utils.go +++ b/internal/controller/utils/template_utils.go @@ -28,7 +28,7 @@ func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerAp // NewStatefulSet 根据创建 StatefulSet func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet { statefulSet := &app_v1.StatefulSet{} - err := yaml_util.Unmarshal(parseTemplate("stateful_set", app), statefulSet) + err := yaml_util.Unmarshal(parseTemplate("statefulset", app), statefulSet) if err != nil { panic(err) }