From 1a42cda60aed8fc6aee79ef6c1b58fbd5e2793f0 Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Tue, 10 Sep 2024 08:28:12 +0000 Subject: [PATCH 1/8] Added .editorconfig --- .editorconfig | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ae0df50 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +indent_style = space +indent_size = 2 +tab_width = 2 +end_of_line = crlf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_style = tab From dd2ff561906fe51f33047ee00ffe1cc2de39d08c Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Wed, 11 Sep 2024 04:10:47 +0000 Subject: [PATCH 2/8] Added resource creation --- internal/controller/templates/ingress.yaml | 18 ++++++ internal/controller/templates/service.yaml | 13 +++++ .../controller/templates/stateful_set.yaml | 46 +++++++++++++++ internal/controller/utils/template_utils.go | 56 +++++++++++++++++++ 4 files changed, 133 insertions(+) create mode 100644 internal/controller/templates/ingress.yaml create mode 100644 internal/controller/templates/service.yaml create mode 100644 internal/controller/templates/stateful_set.yaml create mode 100644 internal/controller/utils/template_utils.go diff --git a/internal/controller/templates/ingress.yaml b/internal/controller/templates/ingress.yaml new file mode 100644 index 0000000..4ec55fc --- /dev/null +++ b/internal/controller/templates/ingress.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{.ObjectMeta.Name}} + namespace: {{.ObjectMeta.Namespace}} +spec: + ingressClassName: nginx + rules: + - host: {{.ObjectMeta.Name}}.devcontainer.devstar.cn + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{.ObjectMeta.Name}} + port: + number: 22 diff --git a/internal/controller/templates/service.yaml b/internal/controller/templates/service.yaml new file mode 100644 index 0000000..bbb9d76 --- /dev/null +++ b/internal/controller/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.ObjectMeta.Name}} + namespace: {{.ObjectMeta.Namespace}} +spec: + selector: + app: {{.ObjectMeta.Name}} + ports: + - name: ssh + protocol: TCP + port: 22 + targetPort: 22 diff --git a/internal/controller/templates/stateful_set.yaml b/internal/controller/templates/stateful_set.yaml new file mode 100644 index 0000000..9a7154c --- /dev/null +++ b/internal/controller/templates/stateful_set.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{.ObjectMeta.Name}} + namespace: {{.ObjectMeta.Namespace}} + labels: + app: {{.ObjectMeta.Name}} +spec: + podManagementPolicy: OrderedReady + replicas: 1 + selector: + matchLabels: + app: {{.ObjectMeta.Name}} + template: + metadata: + labels: + app: {{.ObjectMeta.Name}} + spec: + containers: + - name: {{.ObjectMeta.Name}} + image: {{.Spec.Image}} + imagePullPolicy: IfNotPresent + ports: + - name: tcp-ssh + protocol: tcp + containerPort: 22 + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec ping localhost + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi diff --git a/internal/controller/utils/template_utils.go b/internal/controller/utils/template_utils.go new file mode 100644 index 0000000..e57c3c3 --- /dev/null +++ b/internal/controller/utils/template_utils.go @@ -0,0 +1,56 @@ +package utils + +import ( + "bytes" + "text/template" + + devcontainer_apps_v1 "devstar.cn/DevcontainerApp/api/v1" + app_v1 "k8s.io/api/apps/v1" + core_v1 "k8s.io/api/core/v1" + networking_v1 "k8s.io/api/networking/v1" + yaml_util "k8s.io/apimachinery/pkg/util/yaml" +) + +// parseTemplate 解析 Go Template 模板文件 +func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerApp) []byte { + tmpl, err := template.ParseFiles("internal/controller/templates/" + templateName + ".yaml") + if err != nil { + panic(err) + } + b := new(bytes.Buffer) + err = tmpl.Execute(b, app) + if err != nil { + panic(err) + } + return b.Bytes() +} + +// NewStatefulSet 根据创建 StatefulSet +func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet { + statefulSet := &app_v1.StatefulSet{} + err := yaml_util.Unmarshal(parseTemplate("stateful_set", app), statefulSet) + if err != nil { + panic(err) + } + return statefulSet +} + +// NewService 创建 Service +func NewService(app *devcontainer_apps_v1.DevcontainerApp) *core_v1.Service { + service := &core_v1.Service{} + err := yaml_util.Unmarshal(parseTemplate("service", app), service) + if err != nil { + panic(err) + } + return service +} + +// NewIngress 创建新的 Ingress Controller 规则 +func NewIngress(app *devcontainer_apps_v1.DevcontainerApp) *networking_v1.Ingress { + ingress := &networking_v1.Ingress{} + err := yaml_util.Unmarshal(parseTemplate("ingress", app), ingress) + if err != nil { + panic(err) + } + return ingress +} From 3cf82cb947ffcc5e001d6e3cade1947a6ebba944 Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Wed, 11 Sep 2024 08:54:39 +0000 Subject: [PATCH 3/8] Updated resource creation: * StatefulSet * PVC * Service * Ingress (TODO: SSH port forwarding) --- api/v1/devcontainerapp_types.go | 8 +- ...container.devstar.cn_devcontainerapps.yaml | 22 ++--- .../devcontainer_v1_devcontainerapp.yaml | 10 +- go.mod | 2 +- .../controller/devcontainerapp_controller.go | 99 ++++++++++++++++++- internal/controller/templates/ingress.yaml | 8 +- internal/controller/templates/service.yaml | 6 +- .../{stateful_set.yaml => statefulset.yaml} | 25 +++-- internal/controller/utils/template_utils.go | 2 +- 9 files changed, 142 insertions(+), 40 deletions(-) rename internal/controller/templates/{stateful_set.yaml => statefulset.yaml} (63%) diff --git a/api/v1/devcontainerapp_types.go b/api/v1/devcontainerapp_types.go index e073cb0..487435f 100644 --- a/api/v1/devcontainerapp_types.go +++ b/api/v1/devcontainerapp_types.go @@ -28,26 +28,24 @@ type DevcontainerAppSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - StatefulSet StatefulSetSpec `json:"stateful_set"` + StatefulSet StatefulSetSpec `json:"statefulset"` Service ServiceSpec `json:"service"` Ingress IngressSpec `json:"ingress"` } // StatefulSetSpec specifies StatefulSet for DevContainer type StatefulSetSpec struct { - Name string `json:"name"` Image string `json:"image"` - PVC string `json:"pvc"` } // ServiceSpec specifies Service for DevContainer type ServiceSpec struct { - Name string `json:"name"` + ContainerPort uint16 `json:"containerPort"` } // IngressSpec specifies Ingress Controller access point for DevContainer type IngressSpec struct { - Name string `json:"name"` + Port uint16 `json:"port"` } // DevcontainerAppStatus defines the observed state of DevcontainerApp diff --git a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml index b1dcadb..b58a7a2 100644 --- a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml +++ b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml @@ -43,37 +43,31 @@ spec: description: IngressSpec specifies Ingress Controller access point for DevContainer properties: - name: - type: string + port: + type: integer required: - - name + - port type: object service: description: ServiceSpec specifies Service for DevContainer properties: - name: - type: string + containerPort: + type: integer required: - - name + - containerPort type: object - stateful_set: + statefulset: description: StatefulSetSpec specifies StatefulSet for DevContainer properties: image: type: string - name: - type: string - pvc: - type: string required: - image - - name - - pvc type: object required: - ingress - service - - stateful_set + - statefulset type: object status: description: DevcontainerAppStatus defines the observed state of DevcontainerApp diff --git a/config/samples/devcontainer_v1_devcontainerapp.yaml b/config/samples/devcontainer_v1_devcontainerapp.yaml index 108b4b0..5718092 100644 --- a/config/samples/devcontainer_v1_devcontainerapp.yaml +++ b/config/samples/devcontainer_v1_devcontainerapp.yaml @@ -1,9 +1,15 @@ apiVersion: devcontainer.devstar.cn/v1 kind: DevcontainerApp metadata: + name: daimingchen-devstar-beef092a69c011ef9c00000c2952a362 + namespace: devstar-devcontainer-ns labels: app.kubernetes.io/name: devstar-devcontainer-kubebuilder-scaffold app.kubernetes.io/managed-by: kustomize - name: devcontainerapp-sample spec: - # TODO(user): Add fields here + statefulset: + image: nginx:latest + service: + containerPort: 2222 + ingress: + port: 22 diff --git a/go.mod b/go.mod index ba7356d..fc64f3f 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.31.0 // indirect + k8s.io/api v0.31.0 k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/apiserver v0.31.0 // indirect k8s.io/component-base v0.31.0 // indirect diff --git a/internal/controller/devcontainerapp_controller.go b/internal/controller/devcontainerapp_controller.go index 0b1cf30..ccc7f09 100644 --- a/internal/controller/devcontainerapp_controller.go +++ b/internal/controller/devcontainerapp_controller.go @@ -19,12 +19,20 @@ package controller import ( "context" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" - devcontainerv1 "devstar.cn/DevcontainerApp/api/v1" + devcontainer_v1 "devstar.cn/DevcontainerApp/api/v1" + devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils" + app_v1 "k8s.io/api/apps/v1" + core_v1 "k8s.io/api/core/v1" + networking_v1 "k8s.io/api/networking/v1" + k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // DevcontainerAppReconciler reconciles a DevcontainerApp object @@ -39,7 +47,7 @@ type DevcontainerAppReconciler struct { // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. -// TODO(user): Modify the Reconcile function to compare the state specified by +// Modify the Reconcile function to compare the state specified by // the DevcontainerApp object against the actual cluster state, and then // perform operations to make the cluster state reflect the state specified by // the user. @@ -47,16 +55,97 @@ type DevcontainerAppReconciler struct { // For more details, check Reconcile and its Result here: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - _ = log.FromContext(ctx) + logger := log.FromContext(ctx) - // TODO(user): your logic here + logger.Info("[Reconciler] ENTER controller.Reconcile(...)") + var err error + + // 1. 读取缓存中的 app + app := &devcontainer_v1.DevcontainerApp{} + err = r.Get(ctx, req.NamespacedName, app) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // 2. 根据 DevcontainerApp 配置信息进行处理 + // 2.1 StatefulSet 处理 + statefulSet := devcontainer_controller_utils.NewStatefulSet(app) + err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, statefulSet, r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + + // 2.2 查找 集群中同名称的 StatefulSet + statefulSetInNamespace := &app_v1.StatefulSet{} + err = r.Get(ctx, req.NamespacedName, statefulSetInNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + err = r.Create(ctx, statefulSet) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "Failed to create StatefulSet") + return ctrl.Result{}, err + } + } + // 这里会反复触发更新 + // 原因：在 SetupWithManager方法中，监听了 StatefulSet ，所以只要更新 StatefulSet 就会触发 + // 此处更新和 controllerManager 更新 StatefulSet 都会触发更新事件，导致循环触发 + //修复方法：加上判断条件，仅在 app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image 时才更新 StatefulSet + if app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image { + if err := r.Update(ctx, statefulSet); err != nil { + return ctrl.Result{}, err + } + } + + // 2.2 Service 处理 + service := devcontainer_controller_utils.NewService(app) + if err := k8s_sigs_controller_runtime_utils.SetControllerReference(app, service, r.Scheme); err != nil { + return ctrl.Result{}, err + } + serviceInNamespace := &core_v1.Service{} + err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, serviceInNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + err = r.Create(ctx, service) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "create service failed") + return ctrl.Result{}, err + } + } + + // 2.3 Ingress 处理 + ingress := devcontainer_controller_utils.NewIngress(app) + err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, ingress, r.Scheme) + if err != nil { + return ctrl.Result{}, err + } + ingressInNamespace := &networking_v1.Ingress{} + err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, ingressInNamespace) + if err != nil { + if !errors.IsNotFound(err) { + return ctrl.Result{}, err + } + err := r.Create(ctx, ingress) + if err != nil && !errors.IsAlreadyExists(err) { + logger.Error(err, "failed to create ingress") + return ctrl.Result{}, err + } + } + + logger.Info("[Reconciler] LEAVE controller.Reconcile(...)") return ctrl.Result{}, nil } // SetupWithManager sets up the controller with the Manager. func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). - For(&devcontainerv1.DevcontainerApp{}). + For(&devcontainer_v1.DevcontainerApp{}). + Owns(&app_v1.StatefulSet{}). + Owns(&core_v1.Service{}). + Owns(&networking_v1.Ingress{}). Complete(r) } diff --git a/internal/controller/templates/ingress.yaml b/internal/controller/templates/ingress.yaml index 4ec55fc..b2af8f1 100644 --- a/internal/controller/templates/ingress.yaml +++ b/internal/controller/templates/ingress.yaml @@ -1,10 +1,12 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: {{.ObjectMeta.Name}} + name: nginx-{{.ObjectMeta.Name}}-ingress namespace: {{.ObjectMeta.Namespace}} spec: ingressClassName: nginx + # TODO: create port-forwarding rule for SSH + # - https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/ rules: - host: {{.ObjectMeta.Name}}.devcontainer.devstar.cn http: @@ -13,6 +15,6 @@ spec: pathType: Prefix backend: service: - name: {{.ObjectMeta.Name}} + name: {{.ObjectMeta.Name}}-svc port: - number: 22 + number: {{.Spec.Ingress.Port}} diff --git a/internal/controller/templates/service.yaml b/internal/controller/templates/service.yaml index bbb9d76..a3d1487 100644 --- a/internal/controller/templates/service.yaml +++ b/internal/controller/templates/service.yaml @@ -1,13 +1,13 @@ apiVersion: v1 kind: Service metadata: - name: {{.ObjectMeta.Name}} + name: {{.ObjectMeta.Name}}-svc namespace: {{.ObjectMeta.Namespace}} spec: selector: app: {{.ObjectMeta.Name}} ports: - name: ssh + port: {{.Spec.Ingress.Port}} + targetPort: ssh-port protocol: TCP - port: 22 - targetPort: 22 diff --git a/internal/controller/templates/stateful_set.yaml b/internal/controller/templates/statefulset.yaml similarity index 63% rename from internal/controller/templates/stateful_set.yaml rename to internal/controller/templates/statefulset.yaml index 9a7154c..ce77098 100644 --- a/internal/controller/templates/stateful_set.yaml +++ b/internal/controller/templates/statefulset.yaml @@ -18,20 +18,20 @@ spec: spec: containers: - name: {{.ObjectMeta.Name}} - image: {{.Spec.Image}} + image: {{.Spec.StatefulSet.Image}} imagePullPolicy: IfNotPresent ports: - - name: tcp-ssh - protocol: tcp - containerPort: 22 + - name: ssh-port + protocol: TCP + containerPort: {{.Spec.Service.ContainerPort}} livenessProbe: exec: command: - /bin/sh - -c - - exec ping localhost + - exec ls ~ failureThreshold: 6 - initialDelaySeconds: 30 + initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 @@ -44,3 +44,16 @@ spec: cpu: 100m ephemeral-storage: 50Mi memory: 128Mi + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: devstar-devcontainer-pvc + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + storageClassName: local + volumeMode: Filesystem diff --git a/internal/controller/utils/template_utils.go b/internal/controller/utils/template_utils.go index e57c3c3..41f246b 100644 --- a/internal/controller/utils/template_utils.go +++ b/internal/controller/utils/template_utils.go @@ -28,7 +28,7 @@ func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerAp // NewStatefulSet 根据创建 StatefulSet func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet { statefulSet := &app_v1.StatefulSet{} - err := yaml_util.Unmarshal(parseTemplate("stateful_set", app), statefulSet) + err := yaml_util.Unmarshal(parseTemplate("statefulset", app), statefulSet) if err != nil { panic(err) } From d6319c9814a6c40f12339a2a6ac291f17c1d3aaf Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Tue, 17 Sep 2024 10:54:03 +0000 Subject: [PATCH 4/8] Replaced Nginx Ingress Controller with NodePort Service --- api/v1/devcontainerapp_types.go | 58 ++++++++-- api/v1/zz_generated.deepcopy.go | 57 ++++++---- ...container.devstar.cn_devcontainerapps.yaml | 107 +++++++++++++++--- .../devcontainer_v1_devcontainerapp.yaml | 28 ++++- .../controller/devcontainerapp_controller.go | 25 ---- internal/controller/templates/ingress.yaml | 20 ---- internal/controller/templates/service.yaml | 18 ++- .../controller/templates/statefulset.yaml | 22 ++-- internal/controller/utils/template_utils.go | 24 ++-- 9 files changed, 247 insertions(+), 112 deletions(-) delete mode 100644 internal/controller/templates/ingress.yaml diff --git a/api/v1/devcontainerapp_types.go b/api/v1/devcontainerapp_types.go index 487435f..49cb554 100644 --- a/api/v1/devcontainerapp_types.go +++ b/api/v1/devcontainerapp_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -29,29 +30,70 @@ type DevcontainerAppSpec struct { // Important: Run "make" to regenerate code after modifying this file StatefulSet StatefulSetSpec `json:"statefulset"` - Service ServiceSpec `json:"service"` - Ingress IngressSpec `json:"ingress"` + // +optional + Service ServiceSpec `json:"service"` + + // +kubebuilder:validation:Minimum=0 + + // Optional deadline in seconds for starting the job if it misses scheduled + // time for any reason. Missed jobs executions will be counted as failed ones. + // +optional + StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"` + + // This flag tells the controller to suspend subsequent executions, it does + // not apply to already started executions. Defaults to false. + // +optional + Suspend *bool `json:"suspend,omitempty"` + + // +kubebuilder:validation:Minimum=0 + + // The number of successful finished jobs to retain. + // This is a pointer to distinguish between explicit zero and not specified. + // +optional + SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty"` + + // +kubebuilder:validation:Minimum=0 + + // The number of failed finished jobs to retain. + // This is a pointer to distinguish between explicit zero and not specified. + // +optional + FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"` } // StatefulSetSpec specifies StatefulSet for DevContainer type StatefulSetSpec struct { - Image string `json:"image"` + Image string `json:"image"` + Command []string `json:"command"` + + // +kubebuilder:validation:Minimum=0 + // +optional + ContainerPort uint16 `json:"containerPort,omitempty"` } // ServiceSpec specifies Service for DevContainer type ServiceSpec struct { - ContainerPort uint16 `json:"containerPort"` -} + // +kubebuilder:validation:Minimum=30000 + // +kubebuilder:validation:Maximum=32767 + // +optional + NodePort uint16 `json:"nodePort,omitempty"` -// IngressSpec specifies Ingress Controller access point for DevContainer -type IngressSpec struct { - Port uint16 `json:"port"` + // +kubebuilder:validation:Minimum=0 + // +optional + ServicePort uint16 `json:"servicePort,omitempty"` } // DevcontainerAppStatus defines the observed state of DevcontainerApp type DevcontainerAppStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file + + // A list of pointers to currently running jobs. + // +optional + Active []corev1.ObjectReference `json:"active,omitempty"` + + // Information when was the last time the job was successfully scheduled. + // +optional + LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index b4055a2..89ca8ce 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1 import ( + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -29,8 +30,8 @@ func (in *DevcontainerApp) DeepCopyInto(out *DevcontainerApp) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerApp. @@ -86,9 +87,28 @@ func (in *DevcontainerAppList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DevcontainerAppSpec) DeepCopyInto(out *DevcontainerAppSpec) { *out = *in - out.StatefulSet = in.StatefulSet + in.StatefulSet.DeepCopyInto(&out.StatefulSet) out.Service = in.Service - out.Ingress = in.Ingress + if in.StartingDeadlineSeconds != nil { + in, out := &in.StartingDeadlineSeconds, &out.StartingDeadlineSeconds + *out = new(int64) + **out = **in + } + if in.Suspend != nil { + in, out := &in.Suspend, &out.Suspend + *out = new(bool) + **out = **in + } + if in.SuccessfulJobsHistoryLimit != nil { + in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.FailedJobsHistoryLimit != nil { + in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppSpec. @@ -104,6 +124,15 @@ func (in *DevcontainerAppSpec) DeepCopy() *DevcontainerAppSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DevcontainerAppStatus) DeepCopyInto(out *DevcontainerAppStatus) { *out = *in + if in.Active != nil { + in, out := &in.Active, &out.Active + *out = make([]corev1.ObjectReference, len(*in)) + copy(*out, *in) + } + if in.LastScheduleTime != nil { + in, out := &in.LastScheduleTime, &out.LastScheduleTime + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppStatus. @@ -116,21 +145,6 @@ func (in *DevcontainerAppStatus) DeepCopy() *DevcontainerAppStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IngressSpec) DeepCopyInto(out *IngressSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec. -func (in *IngressSpec) DeepCopy() *IngressSpec { - if in == nil { - return nil - } - out := new(IngressSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = *in @@ -149,6 +163,11 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec. diff --git a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml index b58a7a2..4dfc850 100644 --- a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml +++ b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml @@ -39,38 +39,117 @@ spec: spec: description: DevcontainerAppSpec defines the desired state of DevcontainerApp properties: - ingress: - description: IngressSpec specifies Ingress Controller access point - for DevContainer - properties: - port: - type: integer - required: - - port - type: object + failedJobsHistoryLimit: + description: |- + The number of failed finished jobs to retain. + This is a pointer to distinguish between explicit zero and not specified. + format: int32 + minimum: 0 + type: integer service: description: ServiceSpec specifies Service for DevContainer properties: - containerPort: + nodePort: + maximum: 32767 + minimum: 30000 + type: integer + servicePort: + minimum: 0 type: integer - required: - - containerPort type: object + startingDeadlineSeconds: + description: |- + Optional deadline in seconds for starting the job if it misses scheduled + time for any reason. Missed jobs executions will be counted as failed ones. + format: int64 + minimum: 0 + type: integer statefulset: description: StatefulSetSpec specifies StatefulSet for DevContainer properties: + command: + items: + type: string + type: array + containerPort: + minimum: 0 + type: integer image: type: string required: + - command - image type: object + successfulJobsHistoryLimit: + description: |- + The number of successful finished jobs to retain. + This is a pointer to distinguish between explicit zero and not specified. + format: int32 + minimum: 0 + type: integer + suspend: + description: |- + This flag tells the controller to suspend subsequent executions, it does + not apply to already started executions. Defaults to false. + type: boolean required: - - ingress - - service - statefulset type: object status: description: DevcontainerAppStatus defines the observed state of DevcontainerApp + properties: + active: + description: A list of pointers to currently running jobs. + items: + description: ObjectReference contains enough information to let + you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + lastScheduleTime: + description: Information when was the last time the job was successfully + scheduled. + format: date-time + type: string type: object type: object served: true diff --git a/config/samples/devcontainer_v1_devcontainerapp.yaml b/config/samples/devcontainer_v1_devcontainerapp.yaml index 5718092..d5f5c67 100644 --- a/config/samples/devcontainer_v1_devcontainerapp.yaml +++ b/config/samples/devcontainer_v1_devcontainerapp.yaml @@ -8,8 +8,28 @@ metadata: app.kubernetes.io/managed-by: kustomize spec: statefulset: - image: nginx:latest + image: mcr.microsoft.com/devcontainers/base:dev-ubuntu-20.04 + command: + - /bin/bash + - -c + - echo 'root:root' | chpasswd && useradd -m -s /bin/bash username && echo 'username:password' | chpasswd && usermod -aG sudo username && apt-get update && apt-get install -y openssh-server && service ssh start && apt-get clean && tail -f /dev/null + containerPort: 22 service: - containerPort: 2222 - ingress: - port: 22 + servicePort: 22 + # nodePort: 30000 # 建议动态分配，不建议写入固定 NodePort 值 + + +###################################################################################################################################### +# 后记：SSH连接方式 + +# ```bash +# >>>>> minikube ip +# # 192.168.49.2 +# +# >>>>> minikube service list +# # |-------------------------|----------------------------------------------------------|--------------|---------------------------| +# # | NAMESPACE | NAME | TARGET PORT | URL | +# # |-------------------------|----------------------------------------------------------|--------------|---------------------------| +# # | devstar-devcontainer-ns | daimingchen-devstar-beef092a69c011ef9c00000c2952a362-svc | ssh-port/22 | http://192.168.49.2:32598 | +# +# >>>>> ssh -p 32598 username@192.168.49.2 diff --git a/internal/controller/devcontainerapp_controller.go b/internal/controller/devcontainerapp_controller.go index ccc7f09..686b89a 100644 --- a/internal/controller/devcontainerapp_controller.go +++ b/internal/controller/devcontainerapp_controller.go @@ -31,7 +31,6 @@ import ( devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils" app_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" - networking_v1 "k8s.io/api/networking/v1" k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -56,9 +55,6 @@ type DevcontainerAppReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - - logger.Info("[Reconciler] ENTER controller.Reconcile(...)") - var err error // 1. 读取缓存中的 app @@ -117,26 +113,6 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - // 2.3 Ingress 处理 - ingress := devcontainer_controller_utils.NewIngress(app) - err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, ingress, r.Scheme) - if err != nil { - return ctrl.Result{}, err - } - ingressInNamespace := &networking_v1.Ingress{} - err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, ingressInNamespace) - if err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{}, err - } - err := r.Create(ctx, ingress) - if err != nil && !errors.IsAlreadyExists(err) { - logger.Error(err, "failed to create ingress") - return ctrl.Result{}, err - } - } - - logger.Info("[Reconciler] LEAVE controller.Reconcile(...)") return ctrl.Result{}, nil } @@ -146,6 +122,5 @@ func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&devcontainer_v1.DevcontainerApp{}). Owns(&app_v1.StatefulSet{}). Owns(&core_v1.Service{}). - Owns(&networking_v1.Ingress{}). Complete(r) } diff --git a/internal/controller/templates/ingress.yaml b/internal/controller/templates/ingress.yaml deleted file mode 100644 index b2af8f1..0000000 --- a/internal/controller/templates/ingress.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: nginx-{{.ObjectMeta.Name}}-ingress - namespace: {{.ObjectMeta.Namespace}} -spec: - ingressClassName: nginx - # TODO: create port-forwarding rule for SSH - # - https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/ - rules: - - host: {{.ObjectMeta.Name}}.devcontainer.devstar.cn - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{.ObjectMeta.Name}}-svc - port: - number: {{.Spec.Ingress.Port}} diff --git a/internal/controller/templates/service.yaml b/internal/controller/templates/service.yaml index a3d1487..046dabe 100644 --- a/internal/controller/templates/service.yaml +++ b/internal/controller/templates/service.yaml @@ -6,8 +6,18 @@ metadata: spec: selector: app: {{.ObjectMeta.Name}} + sessionAffinity: None + type: NodePort + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 ports: - - name: ssh - port: {{.Spec.Ingress.Port}} - targetPort: ssh-port - protocol: TCP + - name: ssh-port + protocol: TCP + port: 22 + targetPort: {{.Spec.StatefulSet.ContainerPort}} + {{ if .Spec.Service.NodePort}} + nodePort: {{.Spec.Service.NodePort}} + {{ end }} diff --git a/internal/controller/templates/statefulset.yaml b/internal/controller/templates/statefulset.yaml index ce77098..a9e1554 100644 --- a/internal/controller/templates/statefulset.yaml +++ b/internal/controller/templates/statefulset.yaml @@ -19,11 +19,18 @@ spec: containers: - name: {{.ObjectMeta.Name}} image: {{.Spec.StatefulSet.Image}} + command: + {{range .Spec.StatefulSet.Command}} + - {{.}} + {{end}} imagePullPolicy: IfNotPresent ports: - name: ssh-port protocol: TCP - containerPort: {{.Spec.Service.ContainerPort}} + containerPort: {{.Spec.StatefulSet.ContainerPort}} + volumeMounts: + - name: pvc-devcontainer + mountPath: /data livenessProbe: exec: command: @@ -37,9 +44,9 @@ spec: timeoutSeconds: 5 resources: limits: - cpu: 150m - ephemeral-storage: 2Gi - memory: 192Mi + cpu: 300m + ephemeral-storage: 8Gi + memory: 512Mi requests: cpu: 100m ephemeral-storage: 50Mi @@ -48,12 +55,11 @@ spec: - apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: devstar-devcontainer-pvc + name: pvc-devcontainer spec: + storageClassName: openebs-hostpath accessModes: - ReadWriteOnce resources: requests: - storage: 8Gi - storageClassName: local - volumeMode: Filesystem + storage: 10Gi diff --git a/internal/controller/utils/template_utils.go b/internal/controller/utils/template_utils.go index 41f246b..70940ad 100644 --- a/internal/controller/utils/template_utils.go +++ b/internal/controller/utils/template_utils.go @@ -7,13 +7,19 @@ import ( devcontainer_apps_v1 "devstar.cn/DevcontainerApp/api/v1" app_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" - networking_v1 "k8s.io/api/networking/v1" yaml_util "k8s.io/apimachinery/pkg/util/yaml" ) +const ( + TemplatePath = "internal/controller/templates/" +) + // parseTemplate 解析 Go Template 模板文件 func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerApp) []byte { - tmpl, err := template.ParseFiles("internal/controller/templates/" + templateName + ".yaml") + tmpl, err := template. + New(templateName + ".yaml"). + Funcs(template.FuncMap{"default": DefaultFunc}). + ParseFiles(TemplatePath + templateName + ".yaml") if err != nil { panic(err) } @@ -25,7 +31,7 @@ func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerAp return b.Bytes() } -// NewStatefulSet 根据创建 StatefulSet +// NewStatefulSet 创建 StatefulSet func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet { statefulSet := &app_v1.StatefulSet{} err := yaml_util.Unmarshal(parseTemplate("statefulset", app), statefulSet) @@ -45,12 +51,10 @@ func NewService(app *devcontainer_apps_v1.DevcontainerApp) *core_v1.Service { return service } -// NewIngress 创建新的 Ingress Controller 规则 -func NewIngress(app *devcontainer_apps_v1.DevcontainerApp) *networking_v1.Ingress { - ingress := &networking_v1.Ingress{} - err := yaml_util.Unmarshal(parseTemplate("ingress", app), ingress) - if err != nil { - panic(err) +// DefaultFunc 函数用于实现默认值 +func DefaultFunc(value interface{}, defaultValue interface{}) interface{} { + if value == nil || value == "" { + return defaultValue } - return ingress + return value } From 8f3902a8bfb883faa03422a827819651d132a81d Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Tue, 17 Sep 2024 11:00:02 +0000 Subject: [PATCH 5/8] Updated Quickstart Doc --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index fe5e910..80a4f4e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,29 @@ DevStar DevContainer CRD 脚手架工程 - docker version 17.03+. - kubectl version v1.11.3+. - Access to a Kubernetes v1.11.3+ cluster. +- OpenEBS + +### 使用 Helm Chart 安装 OpenEBS + +DevStar DevContainer 需要 [OpenEBS](https://openebs.io/) 进行动态存储分配 + +```bash +helm repo add openebs https://openebs.github.io/openebs +helm repo update +helm install openebs --namespace openebs openebs/openebs --create-namespace + +helm ls -n openebs +# NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION +# openebs openebs 1 2024-09-13 02:18:28.88814725 +0000 UTC deployed openebs-4.1.0 4.1.0 + +kubectl get sc +# NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE +# mayastor-etcd-localpv openebs.io/local Delete WaitForFirstConsumer false 3m48s +# mayastor-loki-localpv openebs.io/local Delete WaitForFirstConsumer false 3m48s +# openebs-hostpath openebs.io/local Delete WaitForFirstConsumer false 3m48s +# openebs-single-replica io.openebs.csi-mayastor Delete Immediate true 3m48s +# standard (default) k8s.io/minikube-hostpath Delete Immediate false 8d +``` ### To Deploy on the cluster **Build and push your image to the location specified by `IMG`:** From 9e3fe2b00a70106613a33b827a68df9ef1097339 Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Wed, 18 Sep 2024 01:36:00 +0000 Subject: [PATCH 6/8] Updated port num validation --- api/v1/devcontainerapp_types.go | 4 ++-- .../crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/v1/devcontainerapp_types.go b/api/v1/devcontainerapp_types.go index 49cb554..7ed34e6 100644 --- a/api/v1/devcontainerapp_types.go +++ b/api/v1/devcontainerapp_types.go @@ -65,7 +65,7 @@ type StatefulSetSpec struct { Image string `json:"image"` Command []string `json:"command"` - // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Minimum=1 // +optional ContainerPort uint16 `json:"containerPort,omitempty"` } @@ -77,7 +77,7 @@ type ServiceSpec struct { // +optional NodePort uint16 `json:"nodePort,omitempty"` - // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Minimum=1 // +optional ServicePort uint16 `json:"servicePort,omitempty"` } diff --git a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml index 4dfc850..dc352e8 100644 --- a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml +++ b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml @@ -54,7 +54,7 @@ spec: minimum: 30000 type: integer servicePort: - minimum: 0 + minimum: 1 type: integer type: object startingDeadlineSeconds: @@ -72,7 +72,7 @@ spec: type: string type: array containerPort: - minimum: 0 + minimum: 1 type: integer image: type: string From 75919d52cad36798d5d456fa080ba54d4ed5f11d Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Wed, 18 Sep 2024 01:36:23 +0000 Subject: [PATCH 7/8] Added Readiness Probing --- internal/controller/templates/statefulset.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/controller/templates/statefulset.yaml b/internal/controller/templates/statefulset.yaml index a9e1554..f4ee4b1 100644 --- a/internal/controller/templates/statefulset.yaml +++ b/internal/controller/templates/statefulset.yaml @@ -42,6 +42,12 @@ spec: periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - exec cat /etc/ssh/ssh_host*.pub resources: limits: cpu: 300m From 7aa665a31454431c76549df56ee45e7f5406d69f Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Wed, 25 Sep 2024 01:32:43 +0000 Subject: [PATCH 8/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=20DevcontainerApp=20Reco?= =?UTF-8?q?nciler=20=E9=80=BB=E8=BE=91=EF=BC=9A=20-=20=E5=B0=86=20NodePort?= =?UTF-8?q?=20=E8=B0=83=E5=BA=A6=E7=BB=93=E6=9E=9C=E6=94=BE=E5=9C=A8=20app?= =?UTF-8?q?.Status=20=E5=9F=9F=EF=BC=8C=E9=81=BF=E5=85=8D=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=20Spec=20=E5=AF=BC=E8=87=B4=20NodePort=20Service=20?= =?UTF-8?q?=E9=87=8D=E5=A4=8D=E5=88=9B=E5=BB=BA=20-=20=E5=B0=86=20namespac?= =?UTF-8?q?e=20=E4=BF=AE=E6=94=B9=E6=88=90=20devstar-studio-ns=EF=BC=8C?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E5=90=8E=E6=9C=9F=E6=95=B4=E5=90=88=20RBAC?= =?UTF-8?q?=20=E9=81=87=E5=88=B0=E9=97=AE=E9=A2=98=20-=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=20label=20devstar-resource-type=3Ddevstar-devcontaine?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/v1/devcontainerapp_types.go | 4 ++ ...container.devstar.cn_devcontainerapps.yaml | 3 ++ .../devcontainer_v1_devcontainerapp.yaml | 2 +- .../controller/devcontainerapp_controller.go | 44 ++++++++++++------- internal/controller/templates/service.yaml | 1 + .../controller/templates/statefulset.yaml | 3 ++ 6 files changed, 39 insertions(+), 18 deletions(-) diff --git a/api/v1/devcontainerapp_types.go b/api/v1/devcontainerapp_types.go index 7ed34e6..0aac191 100644 --- a/api/v1/devcontainerapp_types.go +++ b/api/v1/devcontainerapp_types.go @@ -94,6 +94,10 @@ type DevcontainerAppStatus struct { // Information when was the last time the job was successfully scheduled. // +optional LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"` + + // NodePortAssigned 存储 DevcontainerApp CRD调度后集群分配的 NodePort + // +optional + NodePortAssigned uint16 `json:"nodePortAssigned"` } // +kubebuilder:object:root=true diff --git a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml index dc352e8..41453fa 100644 --- a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml +++ b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml @@ -150,6 +150,9 @@ spec: scheduled. format: date-time type: string + nodePortAssigned: + description: NodePortAssigned 存储 DevcontainerApp CRD调度后集群分配的 NodePort + type: integer type: object type: object served: true diff --git a/config/samples/devcontainer_v1_devcontainerapp.yaml b/config/samples/devcontainer_v1_devcontainerapp.yaml index d5f5c67..c60982d 100644 --- a/config/samples/devcontainer_v1_devcontainerapp.yaml +++ b/config/samples/devcontainer_v1_devcontainerapp.yaml @@ -2,7 +2,7 @@ apiVersion: devcontainer.devstar.cn/v1 kind: DevcontainerApp metadata: name: daimingchen-devstar-beef092a69c011ef9c00000c2952a362 - namespace: devstar-devcontainer-ns + namespace: devstar-studio-ns labels: app.kubernetes.io/name: devstar-devcontainer-kubebuilder-scaffold app.kubernetes.io/managed-by: kustomize diff --git a/internal/controller/devcontainerapp_controller.go b/internal/controller/devcontainerapp_controller.go index 686b89a..021c142 100644 --- a/internal/controller/devcontainerapp_controller.go +++ b/internal/controller/devcontainerapp_controller.go @@ -29,7 +29,7 @@ import ( devcontainer_v1 "devstar.cn/DevcontainerApp/api/v1" devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils" - app_v1 "k8s.io/api/apps/v1" + apps_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -57,10 +57,11 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ logger := log.FromContext(ctx) var err error - // 1. 读取缓存中的 app + // 1. 读取缓存中的 DevcontainerApp app := &devcontainer_v1.DevcontainerApp{} err = r.Get(ctx, req.NamespacedName, app) if err != nil { + // 当 CRD 资源 “DevcontainerApp” 被删除后，直接返回空结果，跳过剩下步骤 return ctrl.Result{}, client.IgnoreNotFound(err) } @@ -73,7 +74,7 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ } // 2.2 查找 集群中同名称的 StatefulSet - statefulSetInNamespace := &app_v1.StatefulSet{} + statefulSetInNamespace := &apps_v1.StatefulSet{} err = r.Get(ctx, req.NamespacedName, statefulSetInNamespace) if err != nil { if !errors.IsNotFound(err) { @@ -84,14 +85,15 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ logger.Error(err, "Failed to create StatefulSet") return ctrl.Result{}, err } - } - // 这里会反复触发更新 - // 原因：在 SetupWithManager方法中，监听了 StatefulSet ，所以只要更新 StatefulSet 就会触发 - // 此处更新和 controllerManager 更新 StatefulSet 都会触发更新事件，导致循环触发 - //修复方法：加上判断条件，仅在 app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image 时才更新 StatefulSet - if app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image { - if err := r.Update(ctx, statefulSet); err != nil { - return ctrl.Result{}, err + } else { + // 这里会反复触发更新 + // 原因：在 SetupWithManager方法中，监听了 StatefulSet ，所以只要更新 StatefulSet 就会触发 + // 此处更新和 controllerManager 更新 StatefulSet 都会触发更新事件，导致循环触发 + //修复方法：加上判断条件，仅在 app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image 时才更新 StatefulSet + if app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image { + if err := r.Update(ctx, statefulSet); err != nil { + return ctrl.Result{}, err + } } } @@ -100,19 +102,27 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ if err := k8s_sigs_controller_runtime_utils.SetControllerReference(app, service, r.Scheme); err != nil { return ctrl.Result{}, err } - serviceInNamespace := &core_v1.Service{} - err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, serviceInNamespace) + serviceInCluster := &core_v1.Service{} + err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, serviceInCluster) if err != nil { if !errors.IsNotFound(err) { return ctrl.Result{}, err } err = r.Create(ctx, service) - if err != nil && !errors.IsAlreadyExists(err) { - logger.Error(err, "create service failed") + if err == nil { + // 创建 NodePort Service 成功只执行一次 ==> 将NodePort 端口分配信息更新到 app.Status + logger.Info("[DevStar][DevContainer] NodePort Assigned", "nodePortAssigned", service.Spec.Ports[0].NodePort) + + app.Status.NodePortAssigned = uint16(service.Spec.Ports[0].NodePort) + if err := r.Status().Update(ctx, app); err != nil { + logger.Error(err, "Failed to update NodePort of DevcontainerApp", "nodePortAssigned", service.Spec.Ports[0].NodePort) + return ctrl.Result{}, err + } + } else if !errors.IsAlreadyExists(err) { + logger.Error(err, "Failed to create DevcontainerApp NodePort Service", "nodePortServiceName", service.Name) return ctrl.Result{}, err } } - return ctrl.Result{}, nil } @@ -120,7 +130,7 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&devcontainer_v1.DevcontainerApp{}). - Owns(&app_v1.StatefulSet{}). + Owns(&apps_v1.StatefulSet{}). Owns(&core_v1.Service{}). Complete(r) } diff --git a/internal/controller/templates/service.yaml b/internal/controller/templates/service.yaml index 046dabe..973ce89 100644 --- a/internal/controller/templates/service.yaml +++ b/internal/controller/templates/service.yaml @@ -6,6 +6,7 @@ metadata: spec: selector: app: {{.ObjectMeta.Name}} + devstar-resource-type: devstar-devcontainer sessionAffinity: None type: NodePort externalTrafficPolicy: Cluster diff --git a/internal/controller/templates/statefulset.yaml b/internal/controller/templates/statefulset.yaml index f4ee4b1..cd99347 100644 --- a/internal/controller/templates/statefulset.yaml +++ b/internal/controller/templates/statefulset.yaml @@ -5,16 +5,19 @@ metadata: namespace: {{.ObjectMeta.Namespace}} labels: app: {{.ObjectMeta.Name}} + devstar-resource-type: devstar-devcontainer spec: podManagementPolicy: OrderedReady replicas: 1 selector: matchLabels: app: {{.ObjectMeta.Name}} + devstar-resource-type: devstar-devcontainer template: metadata: labels: app: {{.ObjectMeta.Name}} + devstar-resource-type: devstar-devcontainer spec: containers: - name: {{.ObjectMeta.Name}}