From d6319c9814a6c40f12339a2a6ac291f17c1d3aaf Mon Sep 17 00:00:00 2001 From: Mingchen Dai Date: Tue, 17 Sep 2024 10:54:03 +0000 Subject: [PATCH] Replaced Nginx Ingress Controller with NodePort Service --- api/v1/devcontainerapp_types.go | 58 ++++++++-- api/v1/zz_generated.deepcopy.go | 57 ++++++---- ...container.devstar.cn_devcontainerapps.yaml | 107 +++++++++++++++--- .../devcontainer_v1_devcontainerapp.yaml | 28 ++++- .../controller/devcontainerapp_controller.go | 25 ---- internal/controller/templates/ingress.yaml | 20 ---- internal/controller/templates/service.yaml | 18 ++- .../controller/templates/statefulset.yaml | 22 ++-- internal/controller/utils/template_utils.go | 24 ++-- 9 files changed, 247 insertions(+), 112 deletions(-) delete mode 100644 internal/controller/templates/ingress.yaml diff --git a/api/v1/devcontainerapp_types.go b/api/v1/devcontainerapp_types.go index 487435f..49cb554 100644 --- a/api/v1/devcontainerapp_types.go +++ b/api/v1/devcontainerapp_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -29,29 +30,70 @@ type DevcontainerAppSpec struct { // Important: Run "make" to regenerate code after modifying this file StatefulSet StatefulSetSpec `json:"statefulset"` - Service ServiceSpec `json:"service"` - Ingress IngressSpec `json:"ingress"` + // +optional + Service ServiceSpec `json:"service"` + + // +kubebuilder:validation:Minimum=0 + + // Optional deadline in seconds for starting the job if it misses scheduled + // time for any reason. Missed jobs executions will be counted as failed ones. + // +optional + StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"` + + // This flag tells the controller to suspend subsequent executions, it does + // not apply to already started executions. Defaults to false. + // +optional + Suspend *bool `json:"suspend,omitempty"` + + // +kubebuilder:validation:Minimum=0 + + // The number of successful finished jobs to retain. + // This is a pointer to distinguish between explicit zero and not specified. + // +optional + SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty"` + + // +kubebuilder:validation:Minimum=0 + + // The number of failed finished jobs to retain. + // This is a pointer to distinguish between explicit zero and not specified. + // +optional + FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"` } // StatefulSetSpec specifies StatefulSet for DevContainer type StatefulSetSpec struct { - Image string `json:"image"` + Image string `json:"image"` + Command []string `json:"command"` + + // +kubebuilder:validation:Minimum=0 + // +optional + ContainerPort uint16 `json:"containerPort,omitempty"` } // ServiceSpec specifies Service for DevContainer type ServiceSpec struct { - ContainerPort uint16 `json:"containerPort"` -} + // +kubebuilder:validation:Minimum=30000 + // +kubebuilder:validation:Maximum=32767 + // +optional + NodePort uint16 `json:"nodePort,omitempty"` -// IngressSpec specifies Ingress Controller access point for DevContainer -type IngressSpec struct { - Port uint16 `json:"port"` + // +kubebuilder:validation:Minimum=0 + // +optional + ServicePort uint16 `json:"servicePort,omitempty"` } // DevcontainerAppStatus defines the observed state of DevcontainerApp type DevcontainerAppStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file + + // A list of pointers to currently running jobs. + // +optional + Active []corev1.ObjectReference `json:"active,omitempty"` + + // Information when was the last time the job was successfully scheduled. + // +optional + LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"` } // +kubebuilder:object:root=true diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index b4055a2..89ca8ce 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1 import ( + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -29,8 +30,8 @@ func (in *DevcontainerApp) DeepCopyInto(out *DevcontainerApp) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerApp. @@ -86,9 +87,28 @@ func (in *DevcontainerAppList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DevcontainerAppSpec) DeepCopyInto(out *DevcontainerAppSpec) { *out = *in - out.StatefulSet = in.StatefulSet + in.StatefulSet.DeepCopyInto(&out.StatefulSet) out.Service = in.Service - out.Ingress = in.Ingress + if in.StartingDeadlineSeconds != nil { + in, out := &in.StartingDeadlineSeconds, &out.StartingDeadlineSeconds + *out = new(int64) + **out = **in + } + if in.Suspend != nil { + in, out := &in.Suspend, &out.Suspend + *out = new(bool) + **out = **in + } + if in.SuccessfulJobsHistoryLimit != nil { + in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit + *out = new(int32) + **out = **in + } + if in.FailedJobsHistoryLimit != nil { + in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppSpec. @@ -104,6 +124,15 @@ func (in *DevcontainerAppSpec) DeepCopy() *DevcontainerAppSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DevcontainerAppStatus) DeepCopyInto(out *DevcontainerAppStatus) { *out = *in + if in.Active != nil { + in, out := &in.Active, &out.Active + *out = make([]corev1.ObjectReference, len(*in)) + copy(*out, *in) + } + if in.LastScheduleTime != nil { + in, out := &in.LastScheduleTime, &out.LastScheduleTime + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppStatus. @@ -116,21 +145,6 @@ func (in *DevcontainerAppStatus) DeepCopy() *DevcontainerAppStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *IngressSpec) DeepCopyInto(out *IngressSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec. -func (in *IngressSpec) DeepCopy() *IngressSpec { - if in == nil { - return nil - } - out := new(IngressSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) { *out = *in @@ -149,6 +163,11 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) { *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec. diff --git a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml index b58a7a2..4dfc850 100644 --- a/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml +++ b/config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml @@ -39,38 +39,117 @@ spec: spec: description: DevcontainerAppSpec defines the desired state of DevcontainerApp properties: - ingress: - description: IngressSpec specifies Ingress Controller access point - for DevContainer - properties: - port: - type: integer - required: - - port - type: object + failedJobsHistoryLimit: + description: |- + The number of failed finished jobs to retain. + This is a pointer to distinguish between explicit zero and not specified. + format: int32 + minimum: 0 + type: integer service: description: ServiceSpec specifies Service for DevContainer properties: - containerPort: + nodePort: + maximum: 32767 + minimum: 30000 + type: integer + servicePort: + minimum: 0 type: integer - required: - - containerPort type: object + startingDeadlineSeconds: + description: |- + Optional deadline in seconds for starting the job if it misses scheduled + time for any reason. Missed jobs executions will be counted as failed ones. + format: int64 + minimum: 0 + type: integer statefulset: description: StatefulSetSpec specifies StatefulSet for DevContainer properties: + command: + items: + type: string + type: array + containerPort: + minimum: 0 + type: integer image: type: string required: + - command - image type: object + successfulJobsHistoryLimit: + description: |- + The number of successful finished jobs to retain. + This is a pointer to distinguish between explicit zero and not specified. + format: int32 + minimum: 0 + type: integer + suspend: + description: |- + This flag tells the controller to suspend subsequent executions, it does + not apply to already started executions. Defaults to false. + type: boolean required: - - ingress - - service - statefulset type: object status: description: DevcontainerAppStatus defines the observed state of DevcontainerApp + properties: + active: + description: A list of pointers to currently running jobs. + items: + description: ObjectReference contains enough information to let + you inspect or modify the referred object. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + type: array + lastScheduleTime: + description: Information when was the last time the job was successfully + scheduled. + format: date-time + type: string type: object type: object served: true diff --git a/config/samples/devcontainer_v1_devcontainerapp.yaml b/config/samples/devcontainer_v1_devcontainerapp.yaml index 5718092..d5f5c67 100644 --- a/config/samples/devcontainer_v1_devcontainerapp.yaml +++ b/config/samples/devcontainer_v1_devcontainerapp.yaml @@ -8,8 +8,28 @@ metadata: app.kubernetes.io/managed-by: kustomize spec: statefulset: - image: nginx:latest + image: mcr.microsoft.com/devcontainers/base:dev-ubuntu-20.04 + command: + - /bin/bash + - -c + - echo 'root:root' | chpasswd && useradd -m -s /bin/bash username && echo 'username:password' | chpasswd && usermod -aG sudo username && apt-get update && apt-get install -y openssh-server && service ssh start && apt-get clean && tail -f /dev/null + containerPort: 22 service: - containerPort: 2222 - ingress: - port: 22 + servicePort: 22 + # nodePort: 30000 # 建议动态分配，不建议写入固定 NodePort 值 + + +###################################################################################################################################### +# 后记：SSH连接方式 + +# ```bash +# >>>>> minikube ip +# # 192.168.49.2 +# +# >>>>> minikube service list +# # |-------------------------|----------------------------------------------------------|--------------|---------------------------| +# # | NAMESPACE | NAME | TARGET PORT | URL | +# # |-------------------------|----------------------------------------------------------|--------------|---------------------------| +# # | devstar-devcontainer-ns | daimingchen-devstar-beef092a69c011ef9c00000c2952a362-svc | ssh-port/22 | http://192.168.49.2:32598 | +# +# >>>>> ssh -p 32598 username@192.168.49.2 diff --git a/internal/controller/devcontainerapp_controller.go b/internal/controller/devcontainerapp_controller.go index ccc7f09..686b89a 100644 --- a/internal/controller/devcontainerapp_controller.go +++ b/internal/controller/devcontainerapp_controller.go @@ -31,7 +31,6 @@ import ( devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils" app_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" - networking_v1 "k8s.io/api/networking/v1" k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -56,9 +55,6 @@ type DevcontainerAppReconciler struct { // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logger := log.FromContext(ctx) - - logger.Info("[Reconciler] ENTER controller.Reconcile(...)") - var err error // 1. 读取缓存中的 app @@ -117,26 +113,6 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ } } - // 2.3 Ingress 处理 - ingress := devcontainer_controller_utils.NewIngress(app) - err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, ingress, r.Scheme) - if err != nil { - return ctrl.Result{}, err - } - ingressInNamespace := &networking_v1.Ingress{} - err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, ingressInNamespace) - if err != nil { - if !errors.IsNotFound(err) { - return ctrl.Result{}, err - } - err := r.Create(ctx, ingress) - if err != nil && !errors.IsAlreadyExists(err) { - logger.Error(err, "failed to create ingress") - return ctrl.Result{}, err - } - } - - logger.Info("[Reconciler] LEAVE controller.Reconcile(...)") return ctrl.Result{}, nil } @@ -146,6 +122,5 @@ func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&devcontainer_v1.DevcontainerApp{}). Owns(&app_v1.StatefulSet{}). Owns(&core_v1.Service{}). - Owns(&networking_v1.Ingress{}). Complete(r) } diff --git a/internal/controller/templates/ingress.yaml b/internal/controller/templates/ingress.yaml deleted file mode 100644 index b2af8f1..0000000 --- a/internal/controller/templates/ingress.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: nginx-{{.ObjectMeta.Name}}-ingress - namespace: {{.ObjectMeta.Namespace}} -spec: - ingressClassName: nginx - # TODO: create port-forwarding rule for SSH - # - https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/ - rules: - - host: {{.ObjectMeta.Name}}.devcontainer.devstar.cn - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{.ObjectMeta.Name}}-svc - port: - number: {{.Spec.Ingress.Port}} diff --git a/internal/controller/templates/service.yaml b/internal/controller/templates/service.yaml index a3d1487..046dabe 100644 --- a/internal/controller/templates/service.yaml +++ b/internal/controller/templates/service.yaml @@ -6,8 +6,18 @@ metadata: spec: selector: app: {{.ObjectMeta.Name}} + sessionAffinity: None + type: NodePort + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilyPolicy: SingleStack + ipFamilies: + - IPv4 ports: - - name: ssh - port: {{.Spec.Ingress.Port}} - targetPort: ssh-port - protocol: TCP + - name: ssh-port + protocol: TCP + port: 22 + targetPort: {{.Spec.StatefulSet.ContainerPort}} + {{ if .Spec.Service.NodePort}} + nodePort: {{.Spec.Service.NodePort}} + {{ end }} diff --git a/internal/controller/templates/statefulset.yaml b/internal/controller/templates/statefulset.yaml index ce77098..a9e1554 100644 --- a/internal/controller/templates/statefulset.yaml +++ b/internal/controller/templates/statefulset.yaml @@ -19,11 +19,18 @@ spec: containers: - name: {{.ObjectMeta.Name}} image: {{.Spec.StatefulSet.Image}} + command: + {{range .Spec.StatefulSet.Command}} + - {{.}} + {{end}} imagePullPolicy: IfNotPresent ports: - name: ssh-port protocol: TCP - containerPort: {{.Spec.Service.ContainerPort}} + containerPort: {{.Spec.StatefulSet.ContainerPort}} + volumeMounts: + - name: pvc-devcontainer + mountPath: /data livenessProbe: exec: command: @@ -37,9 +44,9 @@ spec: timeoutSeconds: 5 resources: limits: - cpu: 150m - ephemeral-storage: 2Gi - memory: 192Mi + cpu: 300m + ephemeral-storage: 8Gi + memory: 512Mi requests: cpu: 100m ephemeral-storage: 50Mi @@ -48,12 +55,11 @@ spec: - apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: devstar-devcontainer-pvc + name: pvc-devcontainer spec: + storageClassName: openebs-hostpath accessModes: - ReadWriteOnce resources: requests: - storage: 8Gi - storageClassName: local - volumeMode: Filesystem + storage: 10Gi diff --git a/internal/controller/utils/template_utils.go b/internal/controller/utils/template_utils.go index 41f246b..70940ad 100644 --- a/internal/controller/utils/template_utils.go +++ b/internal/controller/utils/template_utils.go @@ -7,13 +7,19 @@ import ( devcontainer_apps_v1 "devstar.cn/DevcontainerApp/api/v1" app_v1 "k8s.io/api/apps/v1" core_v1 "k8s.io/api/core/v1" - networking_v1 "k8s.io/api/networking/v1" yaml_util "k8s.io/apimachinery/pkg/util/yaml" ) +const ( + TemplatePath = "internal/controller/templates/" +) + // parseTemplate 解析 Go Template 模板文件 func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerApp) []byte { - tmpl, err := template.ParseFiles("internal/controller/templates/" + templateName + ".yaml") + tmpl, err := template. + New(templateName + ".yaml"). + Funcs(template.FuncMap{"default": DefaultFunc}). + ParseFiles(TemplatePath + templateName + ".yaml") if err != nil { panic(err) } @@ -25,7 +31,7 @@ func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerAp return b.Bytes() } -// NewStatefulSet 根据创建 StatefulSet +// NewStatefulSet 创建 StatefulSet func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet { statefulSet := &app_v1.StatefulSet{} err := yaml_util.Unmarshal(parseTemplate("statefulset", app), statefulSet) @@ -45,12 +51,10 @@ func NewService(app *devcontainer_apps_v1.DevcontainerApp) *core_v1.Service { return service } -// NewIngress 创建新的 Ingress Controller 规则 -func NewIngress(app *devcontainer_apps_v1.DevcontainerApp) *networking_v1.Ingress { - ingress := &networking_v1.Ingress{} - err := yaml_util.Unmarshal(parseTemplate("ingress", app), ingress) - if err != nil { - panic(err) +// DefaultFunc 函数用于实现默认值 +func DefaultFunc(value interface{}, defaultValue interface{}) interface{} { + if value == nil || value == "" { + return defaultValue } - return ingress + return value }