Replaced Nginx Ingress Controller with NodePort Service

api/v1/devcontainerapp_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -29,29 +30,70 @@ type DevcontainerAppSpec struct {
 	// Important: Run "make" to regenerate code after modifying this file
 
 	StatefulSet StatefulSetSpec `json:"statefulset"`
-	Service     ServiceSpec     `json:"service"`
+	// +optional
-	Ingress     IngressSpec     `json:"ingress"`
+	Service ServiceSpec `json:"service"`
+
+	// +kubebuilder:validation:Minimum=0
+
+	// Optional deadline in seconds for starting the job if it misses scheduled
+	// time for any reason.  Missed jobs executions will be counted as failed ones.
+	// +optional
+	StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"`
+
+	// This flag tells the controller to suspend subsequent executions, it does
+	// not apply to already started executions.  Defaults to false.
+	// +optional
+	Suspend *bool `json:"suspend,omitempty"`
+
+	// +kubebuilder:validation:Minimum=0
+
+	// The number of successful finished jobs to retain.
+	// This is a pointer to distinguish between explicit zero and not specified.
+	// +optional
+	SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty"`
+
+	// +kubebuilder:validation:Minimum=0
+
+	// The number of failed finished jobs to retain.
+	// This is a pointer to distinguish between explicit zero and not specified.
+	// +optional
+	FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"`
 }
 
 // StatefulSetSpec specifies StatefulSet for DevContainer
 type StatefulSetSpec struct {
-	Image string `json:"image"`
+	Image   string   `json:"image"`
+	Command []string `json:"command"`
+
+	// +kubebuilder:validation:Minimum=0
+	// +optional
+	ContainerPort uint16 `json:"containerPort,omitempty"`
 }
 
 // ServiceSpec specifies Service for DevContainer
 type ServiceSpec struct {
-	ContainerPort uint16 `json:"containerPort"`
+	// +kubebuilder:validation:Minimum=30000
-}
+	// +kubebuilder:validation:Maximum=32767
+	// +optional
+	NodePort uint16 `json:"nodePort,omitempty"`
 
-// IngressSpec specifies Ingress Controller access point for DevContainer
+	// +kubebuilder:validation:Minimum=0
-type IngressSpec struct {
+	// +optional
-	Port uint16 `json:"port"`
+	ServicePort uint16 `json:"servicePort,omitempty"`
 }
 
 // DevcontainerAppStatus defines the observed state of DevcontainerApp
 type DevcontainerAppStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
+
+	// A list of pointers to currently running jobs.
+	// +optional
+	Active []corev1.ObjectReference `json:"active,omitempty"`
+
+	// Information when was the last time the job was successfully scheduled.
+	// +optional
+	LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1/zz_generated.deepcopy.go

@@ -21,6 +21,7 @@ limitations under the License.
 package v1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	runtime "k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -29,8 +30,8 @@ func (in *DevcontainerApp) DeepCopyInto(out *DevcontainerApp) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
-	out.Status = in.Status
+	in.Status.DeepCopyInto(&out.Status)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerApp.
@@ -86,9 +87,28 @@ func (in *DevcontainerAppList) DeepCopyObject() runtime.Object {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DevcontainerAppSpec) DeepCopyInto(out *DevcontainerAppSpec) {
 	*out = *in
-	out.StatefulSet = in.StatefulSet
+	in.StatefulSet.DeepCopyInto(&out.StatefulSet)
 	out.Service = in.Service
-	out.Ingress = in.Ingress
+	if in.StartingDeadlineSeconds != nil {
+		in, out := &in.StartingDeadlineSeconds, &out.StartingDeadlineSeconds
+		*out = new(int64)
+		**out = **in
+	}
+	if in.Suspend != nil {
+		in, out := &in.Suspend, &out.Suspend
+		*out = new(bool)
+		**out = **in
+	}
+	if in.SuccessfulJobsHistoryLimit != nil {
+		in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit
+		*out = new(int32)
+		**out = **in
+	}
+	if in.FailedJobsHistoryLimit != nil {
+		in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit
+		*out = new(int32)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppSpec.
@@ -104,6 +124,15 @@ func (in *DevcontainerAppSpec) DeepCopy() *DevcontainerAppSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DevcontainerAppStatus) DeepCopyInto(out *DevcontainerAppStatus) {
 	*out = *in
+	if in.Active != nil {
+		in, out := &in.Active, &out.Active
+		*out = make([]corev1.ObjectReference, len(*in))
+		copy(*out, *in)
+	}
+	if in.LastScheduleTime != nil {
+		in, out := &in.LastScheduleTime, &out.LastScheduleTime
+		*out = (*in).DeepCopy()
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppStatus.
@@ -116,21 +145,6 @@ func (in *DevcontainerAppStatus) DeepCopy() *DevcontainerAppStatus {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec.
-func (in *IngressSpec) DeepCopy() *IngressSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(IngressSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
 	*out = *in
@@ -149,6 +163,11 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
 	*out = *in
+	if in.Command != nil {
+		in, out := &in.Command, &out.Command
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec.

config/crd/bases/devcontainer.devstar.cn_devcontainerapps.yaml

@@ -39,38 +39,117 @@ spec:
           spec:
             description: DevcontainerAppSpec defines the desired state of DevcontainerApp
             properties:
-              ingress:
+              failedJobsHistoryLimit:
-                description: IngressSpec specifies Ingress Controller access point
+                description: |-
-                  for DevContainer
+                  The number of failed finished jobs to retain.
-                properties:
+                  This is a pointer to distinguish between explicit zero and not specified.
-                  port:
+                format: int32
-                    type: integer
+                minimum: 0
-                required:
+                type: integer
-                - port
-                type: object
               service:
                 description: ServiceSpec specifies Service for DevContainer
                 properties:
-                  containerPort:
+                  nodePort:
+                    maximum: 32767
+                    minimum: 30000
+                    type: integer
+                  servicePort:
+                    minimum: 0
                     type: integer
-                required:
-                - containerPort
                 type: object
+              startingDeadlineSeconds:
+                description: |-
+                  Optional deadline in seconds for starting the job if it misses scheduled
+                  time for any reason.  Missed jobs executions will be counted as failed ones.
+                format: int64
+                minimum: 0
+                type: integer
               statefulset:
                 description: StatefulSetSpec specifies StatefulSet for DevContainer
                 properties:
+                  command:
+                    items:
+                      type: string
+                    type: array
+                  containerPort:
+                    minimum: 0
+                    type: integer
                   image:
                     type: string
                 required:
+                - command
                 - image
                 type: object
+              successfulJobsHistoryLimit:
+                description: |-
+                  The number of successful finished jobs to retain.
+                  This is a pointer to distinguish between explicit zero and not specified.
+                format: int32
+                minimum: 0
+                type: integer
+              suspend:
+                description: |-
+                  This flag tells the controller to suspend subsequent executions, it does
+                  not apply to already started executions.  Defaults to false.
+                type: boolean
             required:
-            - ingress
-            - service
             - statefulset
             type: object
           status:
             description: DevcontainerAppStatus defines the observed state of DevcontainerApp
+            properties:
+              active:
+                description: A list of pointers to currently running jobs.
+                items:
+                  description: ObjectReference contains enough information to let
+                    you inspect or modify the referred object.
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: |-
+                        If referring to a piece of an object instead of an entire object, this string
+                        should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within a pod, this would take on a value like:
+                        "spec.containers{name}" (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]" (container with
+                        index 2 in this pod). This syntax is chosen only to have some well-defined way of
+                        referencing a part of an object.
+                      type: string
+                    kind:
+                      description: |-
+                        Kind of the referent.
+                        More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                      type: string
+                    name:
+                      description: |-
+                        Name of the referent.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      type: string
+                    namespace:
+                      description: |-
+                        Namespace of the referent.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
+                      type: string
+                    resourceVersion:
+                      description: |-
+                        Specific resourceVersion to which this reference is made, if any.
+                        More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
+                      type: string
+                    uid:
+                      description: |-
+                        UID of the referent.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
+                      type: string
+                  type: object
+                  x-kubernetes-map-type: atomic
+                type: array
+              lastScheduleTime:
+                description: Information when was the last time the job was successfully
+                  scheduled.
+                format: date-time
+                type: string
             type: object
         type: object
     served: true

config/samples/devcontainer_v1_devcontainerapp.yaml

@@ -8,8 +8,28 @@ metadata:
     app.kubernetes.io/managed-by: kustomize
 spec:
   statefulset:
-    image: nginx:latest
+    image: mcr.microsoft.com/devcontainers/base:dev-ubuntu-20.04
+    command:
+      - /bin/bash
+      - -c
+      - echo 'root:root' | chpasswd && useradd -m -s /bin/bash username && echo 'username:password' | chpasswd && usermod -aG sudo username && apt-get update && apt-get install -y openssh-server && service ssh start && apt-get clean && tail -f /dev/null
+    containerPort: 22
   service:
-    containerPort: 2222
+    servicePort: 22
-  ingress:
+    # nodePort: 30000 # 建议动态分配，不建议写入固定 NodePort 值
-    port: 22
+
+
+######################################################################################################################################
+# 后记：SSH连接方式
+
+# ```bash
+# >>>>> minikube ip
+# # 192.168.49.2
+#
+# >>>>> minikube service list
+# # |-------------------------|----------------------------------------------------------|--------------|---------------------------|
+# # |        NAMESPACE        |                           NAME                           | TARGET PORT  |            URL            |
+# # |-------------------------|----------------------------------------------------------|--------------|---------------------------|
+# # | devstar-devcontainer-ns | daimingchen-devstar-beef092a69c011ef9c00000c2952a362-svc | ssh-port/22  | http://192.168.49.2:32598 |
+#
+# >>>>> ssh -p 32598 username@192.168.49.2

internal/controller/devcontainerapp_controller.go

@@ -31,7 +31,6 @@ import (
 	devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils"
 	app_v1 "k8s.io/api/apps/v1"
 	core_v1 "k8s.io/api/core/v1"
-	networking_v1 "k8s.io/api/networking/v1"
 	k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
@@ -56,9 +55,6 @@ type DevcontainerAppReconciler struct {
 // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile
 func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
 	logger := log.FromContext(ctx)
-
-	logger.Info("[Reconciler] ENTER controller.Reconcile(...)")
-
 	var err error
 
 	// 1. 读取缓存中的 app
@@ -117,26 +113,6 @@ func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		}
 	}
 
-	// 2.3 Ingress 处理
-	ingress := devcontainer_controller_utils.NewIngress(app)
-	err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, ingress, r.Scheme)
-	if err != nil {
-		return ctrl.Result{}, err
-	}
-	ingressInNamespace := &networking_v1.Ingress{}
-	err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, ingressInNamespace)
-	if err != nil {
-		if !errors.IsNotFound(err) {
-			return ctrl.Result{}, err
-		}
-		err := r.Create(ctx, ingress)
-		if err != nil && !errors.IsAlreadyExists(err) {
-			logger.Error(err, "failed to create ingress")
-			return ctrl.Result{}, err
-		}
-	}
-
-	logger.Info("[Reconciler] LEAVE controller.Reconcile(...)")
 	return ctrl.Result{}, nil
 }
 
@@ -146,6 +122,5 @@ func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&devcontainer_v1.DevcontainerApp{}).
 		Owns(&app_v1.StatefulSet{}).
 		Owns(&core_v1.Service{}).
-		Owns(&networking_v1.Ingress{}).
 		Complete(r)
 }

internal/controller/templates/ingress.yaml

@@ -1,20 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nginx-{{.ObjectMeta.Name}}-ingress
-  namespace: {{.ObjectMeta.Namespace}}
-spec:
-  ingressClassName: nginx
-  # TODO: create port-forwarding rule for SSH
-  # - https://kubernetes.github.io/ingress-nginx/user-guide/exposing-tcp-udp-services/
-  rules:
-    - host: {{.ObjectMeta.Name}}.devcontainer.devstar.cn
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: {{.ObjectMeta.Name}}-svc
-                port:
-                  number: {{.Spec.Ingress.Port}}

internal/controller/templates/service.yaml

@@ -6,8 +6,18 @@ metadata:
 spec:
   selector:
     app: {{.ObjectMeta.Name}}
+  sessionAffinity: None
+  type: NodePort
+  externalTrafficPolicy: Cluster
+  internalTrafficPolicy: Cluster
+  ipFamilyPolicy: SingleStack
+  ipFamilies:
+  - IPv4
   ports:
-    - name: ssh
+  - name: ssh-port
-      port: {{.Spec.Ingress.Port}}
+    protocol: TCP
-      targetPort: ssh-port
+    port: 22
-      protocol: TCP
+    targetPort: {{.Spec.StatefulSet.ContainerPort}}
+    {{ if .Spec.Service.NodePort}}
+    nodePort: {{.Spec.Service.NodePort}}
+    {{ end }}

internal/controller/templates/statefulset.yaml

@@ -19,11 +19,18 @@ spec:
       containers:
         - name: {{.ObjectMeta.Name}}
           image: {{.Spec.StatefulSet.Image}}
+          command:
+          {{range .Spec.StatefulSet.Command}}
+          - {{.}}
+          {{end}}
           imagePullPolicy: IfNotPresent
           ports:
             - name: ssh-port
               protocol: TCP
-              containerPort: {{.Spec.Service.ContainerPort}}
+              containerPort: {{.Spec.StatefulSet.ContainerPort}}
+          volumeMounts:
+          - name: pvc-devcontainer
+            mountPath: /data
           livenessProbe:
             exec:
               command:
@@ -37,9 +44,9 @@ spec:
             timeoutSeconds: 5
           resources:
             limits:
-              cpu: 150m
+              cpu: 300m
-              ephemeral-storage: 2Gi
+              ephemeral-storage: 8Gi
-              memory: 192Mi
+              memory: 512Mi
             requests:
               cpu: 100m
               ephemeral-storage: 50Mi
@@ -48,12 +55,11 @@ spec:
     - apiVersion: v1
       kind: PersistentVolumeClaim
       metadata:
-        name: devstar-devcontainer-pvc
+        name: pvc-devcontainer
       spec:
+        storageClassName: openebs-hostpath
         accessModes:
         - ReadWriteOnce
         resources:
           requests:
-            storage: 8Gi
+            storage: 10Gi
-        storageClassName: local
-        volumeMode: Filesystem

internal/controller/utils/template_utils.go

@@ -7,13 +7,19 @@ import (
 	devcontainer_apps_v1 "devstar.cn/DevcontainerApp/api/v1"
 	app_v1 "k8s.io/api/apps/v1"
 	core_v1 "k8s.io/api/core/v1"
-	networking_v1 "k8s.io/api/networking/v1"
 	yaml_util "k8s.io/apimachinery/pkg/util/yaml"
 )
 
+const (
+	TemplatePath = "internal/controller/templates/"
+)
+
 // parseTemplate 解析 Go Template 模板文件
 func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerApp) []byte {
-	tmpl, err := template.ParseFiles("internal/controller/templates/" + templateName + ".yaml")
+	tmpl, err := template.
+		New(templateName + ".yaml").
+		Funcs(template.FuncMap{"default": DefaultFunc}).
+		ParseFiles(TemplatePath + templateName + ".yaml")
 	if err != nil {
 		panic(err)
 	}
@@ -25,7 +31,7 @@ func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerAp
 	return b.Bytes()
 }
 
-// NewStatefulSet 根据创建 StatefulSet
+// NewStatefulSet 创建 StatefulSet
 func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet {
 	statefulSet := &app_v1.StatefulSet{}
 	err := yaml_util.Unmarshal(parseTemplate("statefulset", app), statefulSet)
@@ -45,12 +51,10 @@ func NewService(app *devcontainer_apps_v1.DevcontainerApp) *core_v1.Service {
 	return service
 }
 
-// NewIngress 创建新的 Ingress Controller 规则
+// DefaultFunc 函数用于实现默认值
-func NewIngress(app *devcontainer_apps_v1.DevcontainerApp) *networking_v1.Ingress {
+func DefaultFunc(value interface{}, defaultValue interface{}) interface{} {
-	ingress := &networking_v1.Ingress{}
+	if value == nil || value == "" {
-	err := yaml_util.Unmarshal(parseTemplate("ingress", app), ingress)
+		return defaultValue
-	if err != nil {
-		panic(err)
 	}
-	return ingress
+	return value
 }