!1 可通过 YAML 创建 CRD DevcontainerApp (StatefulSet + NodePort_Service)

* 优化 DevcontainerApp Reconciler 逻辑
* Added Readiness Probing
* Updated port num validation
* Updated Quickstart Doc
* Replaced Nginx Ingress Controller with NodePort Service
* Updated resource creation:
* Added resource creation
* Added .editorconfig
This commit is contained in:
戴明辰 2024-09-25 01:51:11 +00:00
parent 5c990464ee
commit c10befe9f8
11 changed files with 494 additions and 61 deletions

13
.editorconfig Normal file

@ -0,0 +1,13 @@
root = true
[*]
indent_style = space
indent_size = 2
tab_width = 2
end_of_line = crlf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
[Makefile]
indent_style = tab

@ -15,6 +15,29 @@ DevStar DevContainer CRD 脚手架工程
- docker version 17.03+.
- kubectl version v1.11.3+.
- Access to a Kubernetes v1.11.3+ cluster.
- OpenEBS
### 使用 Helm Chart 安装 OpenEBS
DevStar DevContainer 需要 [OpenEBS](https://openebs.io/) 进行动态存储分配
```bash
helm repo add openebs https://openebs.github.io/openebs
helm repo update
helm install openebs --namespace openebs openebs/openebs --create-namespace
helm ls -n openebs
# NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
# openebs openebs 1 2024-09-13 02:18:28.88814725 +0000 UTC deployed openebs-4.1.0 4.1.0
kubectl get sc
# NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
# mayastor-etcd-localpv openebs.io/local Delete WaitForFirstConsumer false 3m48s
# mayastor-loki-localpv openebs.io/local Delete WaitForFirstConsumer false 3m48s
# openebs-hostpath openebs.io/local Delete WaitForFirstConsumer false 3m48s
# openebs-single-replica io.openebs.csi-mayastor Delete Immediate true 3m48s
# standard (default) k8s.io/minikube-hostpath Delete Immediate false 8d
```
### To Deploy on the cluster
**Build and push your image to the location specified by `IMG`:**

@ -17,6 +17,7 @@ limitations under the License.
package v1
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -28,32 +29,75 @@ type DevcontainerAppSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file
StatefulSet StatefulSetSpec `json:"stateful_set"`
Service ServiceSpec `json:"service"`
Ingress IngressSpec `json:"ingress"`
StatefulSet StatefulSetSpec `json:"statefulset"`
// +optional
Service ServiceSpec `json:"service"`
// +kubebuilder:validation:Minimum=0
// Optional deadline in seconds for starting the job if it misses scheduled
// time for any reason. Missed jobs executions will be counted as failed ones.
// +optional
StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty"`
// This flag tells the controller to suspend subsequent executions, it does
// not apply to already started executions. Defaults to false.
// +optional
Suspend *bool `json:"suspend,omitempty"`
// +kubebuilder:validation:Minimum=0
// The number of successful finished jobs to retain.
// This is a pointer to distinguish between explicit zero and not specified.
// +optional
SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty"`
// +kubebuilder:validation:Minimum=0
// The number of failed finished jobs to retain.
// This is a pointer to distinguish between explicit zero and not specified.
// +optional
FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty"`
}
// StatefulSetSpec specifies StatefulSet for DevContainer
type StatefulSetSpec struct {
Name string `json:"name"`
Image string `json:"image"`
PVC string `json:"pvc"`
Image string `json:"image"`
Command []string `json:"command"`
// +kubebuilder:validation:Minimum=1
// +optional
ContainerPort uint16 `json:"containerPort,omitempty"`
}
// ServiceSpec specifies Service for DevContainer
type ServiceSpec struct {
Name string `json:"name"`
}
// +kubebuilder:validation:Minimum=30000
// +kubebuilder:validation:Maximum=32767
// +optional
NodePort uint16 `json:"nodePort,omitempty"`
// IngressSpec specifies Ingress Controller access point for DevContainer
type IngressSpec struct {
Name string `json:"name"`
// +kubebuilder:validation:Minimum=1
// +optional
ServicePort uint16 `json:"servicePort,omitempty"`
}
// DevcontainerAppStatus defines the observed state of DevcontainerApp
type DevcontainerAppStatus struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
// Important: Run "make" to regenerate code after modifying this file
// A list of pointers to currently running jobs.
// +optional
Active []corev1.ObjectReference `json:"active,omitempty"`
// Information when was the last time the job was successfully scheduled.
// +optional
LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"`
// NodePortAssigned 存储 DevcontainerApp CRD调度后集群分配的 NodePort
// +optional
NodePortAssigned uint16 `json:"nodePortAssigned"`
}
// +kubebuilder:object:root=true

@ -21,6 +21,7 @@ limitations under the License.
package v1
import (
corev1 "k8s.io/api/core/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@ -29,8 +30,8 @@ func (in *DevcontainerApp) DeepCopyInto(out *DevcontainerApp) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
out.Status = in.Status
in.Spec.DeepCopyInto(&out.Spec)
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerApp.
@ -86,9 +87,28 @@ func (in *DevcontainerAppList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DevcontainerAppSpec) DeepCopyInto(out *DevcontainerAppSpec) {
*out = *in
out.StatefulSet = in.StatefulSet
in.StatefulSet.DeepCopyInto(&out.StatefulSet)
out.Service = in.Service
out.Ingress = in.Ingress
if in.StartingDeadlineSeconds != nil {
in, out := &in.StartingDeadlineSeconds, &out.StartingDeadlineSeconds
*out = new(int64)
**out = **in
}
if in.Suspend != nil {
in, out := &in.Suspend, &out.Suspend
*out = new(bool)
**out = **in
}
if in.SuccessfulJobsHistoryLimit != nil {
in, out := &in.SuccessfulJobsHistoryLimit, &out.SuccessfulJobsHistoryLimit
*out = new(int32)
**out = **in
}
if in.FailedJobsHistoryLimit != nil {
in, out := &in.FailedJobsHistoryLimit, &out.FailedJobsHistoryLimit
*out = new(int32)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppSpec.
@ -104,6 +124,15 @@ func (in *DevcontainerAppSpec) DeepCopy() *DevcontainerAppSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DevcontainerAppStatus) DeepCopyInto(out *DevcontainerAppStatus) {
*out = *in
if in.Active != nil {
in, out := &in.Active, &out.Active
*out = make([]corev1.ObjectReference, len(*in))
copy(*out, *in)
}
if in.LastScheduleTime != nil {
in, out := &in.LastScheduleTime, &out.LastScheduleTime
*out = (*in).DeepCopy()
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevcontainerAppStatus.
@ -116,21 +145,6 @@ func (in *DevcontainerAppStatus) DeepCopy() *DevcontainerAppStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *IngressSpec) DeepCopyInto(out *IngressSpec) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new IngressSpec.
func (in *IngressSpec) DeepCopy() *IngressSpec {
if in == nil {
return nil
}
out := new(IngressSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
*out = *in
@ -149,6 +163,11 @@ func (in *ServiceSpec) DeepCopy() *ServiceSpec {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
*out = *in
if in.Command != nil {
in, out := &in.Command, &out.Command
*out = make([]string, len(*in))
copy(*out, *in)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec.

@ -39,44 +39,120 @@ spec:
spec:
description: DevcontainerAppSpec defines the desired state of DevcontainerApp
properties:
ingress:
description: IngressSpec specifies Ingress Controller access point
for DevContainer
properties:
name:
type: string
required:
- name
type: object
failedJobsHistoryLimit:
description: |-
The number of failed finished jobs to retain.
This is a pointer to distinguish between explicit zero and not specified.
format: int32
minimum: 0
type: integer
service:
description: ServiceSpec specifies Service for DevContainer
properties:
name:
type: string
required:
- name
nodePort:
maximum: 32767
minimum: 30000
type: integer
servicePort:
minimum: 1
type: integer
type: object
stateful_set:
startingDeadlineSeconds:
description: |-
Optional deadline in seconds for starting the job if it misses scheduled
time for any reason. Missed jobs executions will be counted as failed ones.
format: int64
minimum: 0
type: integer
statefulset:
description: StatefulSetSpec specifies StatefulSet for DevContainer
properties:
command:
items:
type: string
type: array
containerPort:
minimum: 1
type: integer
image:
type: string
name:
type: string
pvc:
type: string
required:
- command
- image
- name
- pvc
type: object
successfulJobsHistoryLimit:
description: |-
The number of successful finished jobs to retain.
This is a pointer to distinguish between explicit zero and not specified.
format: int32
minimum: 0
type: integer
suspend:
description: |-
This flag tells the controller to suspend subsequent executions, it does
not apply to already started executions. Defaults to false.
type: boolean
required:
- ingress
- service
- stateful_set
- statefulset
type: object
status:
description: DevcontainerAppStatus defines the observed state of DevcontainerApp
properties:
active:
description: A list of pointers to currently running jobs.
items:
description: ObjectReference contains enough information to let
you inspect or modify the referred object.
properties:
apiVersion:
description: API version of the referent.
type: string
fieldPath:
description: |-
If referring to a piece of an object instead of an entire object, this string
should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
For example, if the object reference is to a container within a pod, this would take on a value like:
"spec.containers{name}" (where "name" refers to the name of the container that triggered
the event) or if no container name is specified "spec.containers[2]" (container with
index 2 in this pod). This syntax is chosen only to have some well-defined way of
referencing a part of an object.
type: string
kind:
description: |-
Kind of the referent.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
namespace:
description: |-
Namespace of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
type: string
resourceVersion:
description: |-
Specific resourceVersion to which this reference is made, if any.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
type: string
uid:
description: |-
UID of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
type: string
type: object
x-kubernetes-map-type: atomic
type: array
lastScheduleTime:
description: Information when was the last time the job was successfully
scheduled.
format: date-time
type: string
nodePortAssigned:
description: NodePortAssigned 存储 DevcontainerApp CRD调度后集群分配的 NodePort
type: integer
type: object
type: object
served: true

@ -1,9 +1,35 @@
apiVersion: devcontainer.devstar.cn/v1
kind: DevcontainerApp
metadata:
name: daimingchen-devstar-beef092a69c011ef9c00000c2952a362
namespace: devstar-studio-ns
labels:
app.kubernetes.io/name: devstar-devcontainer-kubebuilder-scaffold
app.kubernetes.io/managed-by: kustomize
name: devcontainerapp-sample
spec:
# TODO(user): Add fields here
statefulset:
image: mcr.microsoft.com/devcontainers/base:dev-ubuntu-20.04
command:
- /bin/bash
- -c
- echo 'root:root' | chpasswd && useradd -m -s /bin/bash username && echo 'username:password' | chpasswd && usermod -aG sudo username && apt-get update && apt-get install -y openssh-server && service ssh start && apt-get clean && tail -f /dev/null
containerPort: 22
service:
servicePort: 22
# nodePort: 30000 # 建议动态分配,不建议写入固定 NodePort 值
######################################################################################################################################
# 后记SSH连接方式
# ```bash
# >>>>> minikube ip
# # 192.168.49.2
#
# >>>>> minikube service list
# # |-------------------------|----------------------------------------------------------|--------------|---------------------------|
# # | NAMESPACE | NAME | TARGET PORT | URL |
# # |-------------------------|----------------------------------------------------------|--------------|---------------------------|
# # | devstar-devcontainer-ns | daimingchen-devstar-beef092a69c011ef9c00000c2952a362-svc | ssh-port/22 | http://192.168.49.2:32598 |
#
# >>>>> ssh -p 32598 username@192.168.49.2

2
go.mod

@ -84,7 +84,7 @@ require (
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.31.0 // indirect
k8s.io/api v0.31.0
k8s.io/apiextensions-apiserver v0.31.0 // indirect
k8s.io/apiserver v0.31.0 // indirect
k8s.io/component-base v0.31.0 // indirect

@ -19,12 +19,19 @@ package controller
import (
"context"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
devcontainerv1 "devstar.cn/DevcontainerApp/api/v1"
devcontainer_v1 "devstar.cn/DevcontainerApp/api/v1"
devcontainer_controller_utils "devstar.cn/DevcontainerApp/internal/controller/utils"
apps_v1 "k8s.io/api/apps/v1"
core_v1 "k8s.io/api/core/v1"
k8s_sigs_controller_runtime_utils "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)
// DevcontainerAppReconciler reconciles a DevcontainerApp object
@ -39,7 +46,7 @@ type DevcontainerAppReconciler struct {
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// Modify the Reconcile function to compare the state specified by
// the DevcontainerApp object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
@ -47,16 +54,83 @@ type DevcontainerAppReconciler struct {
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.0/pkg/reconcile
func (r *DevcontainerAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
logger := log.FromContext(ctx)
var err error
// TODO(user): your logic here
// 1. 读取缓存中的 DevcontainerApp
app := &devcontainer_v1.DevcontainerApp{}
err = r.Get(ctx, req.NamespacedName, app)
if err != nil {
// 当 CRD 资源 “DevcontainerApp” 被删除后,直接返回空结果,跳过剩下步骤
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// 2. 根据 DevcontainerApp 配置信息进行处理
// 2.1 StatefulSet 处理
statefulSet := devcontainer_controller_utils.NewStatefulSet(app)
err = k8s_sigs_controller_runtime_utils.SetControllerReference(app, statefulSet, r.Scheme)
if err != nil {
return ctrl.Result{}, err
}
// 2.2 查找 集群中同名称的 StatefulSet
statefulSetInNamespace := &apps_v1.StatefulSet{}
err = r.Get(ctx, req.NamespacedName, statefulSetInNamespace)
if err != nil {
if !errors.IsNotFound(err) {
return ctrl.Result{}, err
}
err = r.Create(ctx, statefulSet)
if err != nil && !errors.IsAlreadyExists(err) {
logger.Error(err, "Failed to create StatefulSet")
return ctrl.Result{}, err
}
} else {
// 这里会反复触发更新
// 原因:在 SetupWithManager方法中监听了 StatefulSet ,所以只要更新 StatefulSet 就会触发
// 此处更新和 controllerManager 更新 StatefulSet 都会触发更新事件,导致循环触发
//修复方法:加上判断条件,仅在 app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image 时才更新 StatefulSet
if app.Spec.StatefulSet.Image != statefulSet.Spec.Template.Spec.Containers[0].Image {
if err := r.Update(ctx, statefulSet); err != nil {
return ctrl.Result{}, err
}
}
}
// 2.2 Service 处理
service := devcontainer_controller_utils.NewService(app)
if err := k8s_sigs_controller_runtime_utils.SetControllerReference(app, service, r.Scheme); err != nil {
return ctrl.Result{}, err
}
serviceInCluster := &core_v1.Service{}
err = r.Get(ctx, types.NamespacedName{Name: app.Name, Namespace: app.Namespace}, serviceInCluster)
if err != nil {
if !errors.IsNotFound(err) {
return ctrl.Result{}, err
}
err = r.Create(ctx, service)
if err == nil {
// 创建 NodePort Service 成功只执行一次 ==> 将NodePort 端口分配信息更新到 app.Status
logger.Info("[DevStar][DevContainer] NodePort Assigned", "nodePortAssigned", service.Spec.Ports[0].NodePort)
app.Status.NodePortAssigned = uint16(service.Spec.Ports[0].NodePort)
if err := r.Status().Update(ctx, app); err != nil {
logger.Error(err, "Failed to update NodePort of DevcontainerApp", "nodePortAssigned", service.Spec.Ports[0].NodePort)
return ctrl.Result{}, err
}
} else if !errors.IsAlreadyExists(err) {
logger.Error(err, "Failed to create DevcontainerApp NodePort Service", "nodePortServiceName", service.Name)
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *DevcontainerAppReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&devcontainerv1.DevcontainerApp{}).
For(&devcontainer_v1.DevcontainerApp{}).
Owns(&apps_v1.StatefulSet{}).
Owns(&core_v1.Service{}).
Complete(r)
}

@ -0,0 +1,24 @@
apiVersion: v1
kind: Service
metadata:
name: {{.ObjectMeta.Name}}-svc
namespace: {{.ObjectMeta.Namespace}}
spec:
selector:
app: {{.ObjectMeta.Name}}
devstar-resource-type: devstar-devcontainer
sessionAffinity: None
type: NodePort
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilyPolicy: SingleStack
ipFamilies:
- IPv4
ports:
- name: ssh-port
protocol: TCP
port: 22
targetPort: {{.Spec.StatefulSet.ContainerPort}}
{{ if .Spec.Service.NodePort}}
nodePort: {{.Spec.Service.NodePort}}
{{ end }}

@ -0,0 +1,74 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{.ObjectMeta.Name}}
namespace: {{.ObjectMeta.Namespace}}
labels:
app: {{.ObjectMeta.Name}}
devstar-resource-type: devstar-devcontainer
spec:
podManagementPolicy: OrderedReady
replicas: 1
selector:
matchLabels:
app: {{.ObjectMeta.Name}}
devstar-resource-type: devstar-devcontainer
template:
metadata:
labels:
app: {{.ObjectMeta.Name}}
devstar-resource-type: devstar-devcontainer
spec:
containers:
- name: {{.ObjectMeta.Name}}
image: {{.Spec.StatefulSet.Image}}
command:
{{range .Spec.StatefulSet.Command}}
- {{.}}
{{end}}
imagePullPolicy: IfNotPresent
ports:
- name: ssh-port
protocol: TCP
containerPort: {{.Spec.StatefulSet.ContainerPort}}
volumeMounts:
- name: pvc-devcontainer
mountPath: /data
livenessProbe:
exec:
command:
- /bin/sh
- -c
- exec ls ~
failureThreshold: 6
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
readinessProbe:
exec:
command:
- /bin/sh
- -c
- exec cat /etc/ssh/ssh_host*.pub
resources:
limits:
cpu: 300m
ephemeral-storage: 8Gi
memory: 512Mi
requests:
cpu: 100m
ephemeral-storage: 50Mi
memory: 128Mi
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: pvc-devcontainer
spec:
storageClassName: openebs-hostpath
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi

@ -0,0 +1,60 @@
package utils
import (
"bytes"
"text/template"
devcontainer_apps_v1 "devstar.cn/DevcontainerApp/api/v1"
app_v1 "k8s.io/api/apps/v1"
core_v1 "k8s.io/api/core/v1"
yaml_util "k8s.io/apimachinery/pkg/util/yaml"
)
const (
TemplatePath = "internal/controller/templates/"
)
// parseTemplate 解析 Go Template 模板文件
func parseTemplate(templateName string, app *devcontainer_apps_v1.DevcontainerApp) []byte {
tmpl, err := template.
New(templateName + ".yaml").
Funcs(template.FuncMap{"default": DefaultFunc}).
ParseFiles(TemplatePath + templateName + ".yaml")
if err != nil {
panic(err)
}
b := new(bytes.Buffer)
err = tmpl.Execute(b, app)
if err != nil {
panic(err)
}
return b.Bytes()
}
// NewStatefulSet 创建 StatefulSet
func NewStatefulSet(app *devcontainer_apps_v1.DevcontainerApp) *app_v1.StatefulSet {
statefulSet := &app_v1.StatefulSet{}
err := yaml_util.Unmarshal(parseTemplate("statefulset", app), statefulSet)
if err != nil {
panic(err)
}
return statefulSet
}
// NewService 创建 Service
func NewService(app *devcontainer_apps_v1.DevcontainerApp) *core_v1.Service {
service := &core_v1.Service{}
err := yaml_util.Unmarshal(parseTemplate("service", app), service)
if err != nil {
panic(err)
}
return service
}
// DefaultFunc 函数用于实现默认值
func DefaultFunc(value interface{}, defaultValue interface{}) interface{} {
if value == nil || value == "" {
return defaultValue
}
return value
}