From 67f8bc32fa7be41de93d18a2945a4f60f1bcc2c5 Mon Sep 17 00:00:00 2001 From: Sabrina Demagny Date: Thu, 27 Feb 2025 02:52:13 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80(helm)=20deploy=20celery=20worker?= =?UTF-8?q?=20and=20celery=20beat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allow to start a celery worker and a celery beat --- src/helm/desk/Chart.yaml | 2 +- src/helm/desk/templates/_helpers.tpl | 19 +++ .../templates/celery_beat_deployment.yaml | 140 ++++++++++++++++++ .../templates/celery_worker_deployment.yaml | 140 ++++++++++++++++++ src/helm/desk/values.yaml | 138 +++++++++++++++++ .../dev-keycloak/values.desk.yaml.gotmpl | 1 + src/helm/env.d/dev/values.desk.yaml.gotmpl | 1 + 7 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 src/helm/desk/templates/celery_beat_deployment.yaml create mode 100644 src/helm/desk/templates/celery_worker_deployment.yaml diff --git a/src/helm/desk/Chart.yaml b/src/helm/desk/Chart.yaml index 4a9dba6..fdf6d1f 100644 --- a/src/helm/desk/Chart.yaml +++ b/src/helm/desk/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v2 type: application name: desk -version: 0.0.4 +version: 0.0.5 diff --git a/src/helm/desk/templates/_helpers.tpl b/src/helm/desk/templates/_helpers.tpl index 2c3b4da..2a6eb37 100644 --- a/src/helm/desk/templates/_helpers.tpl +++ b/src/helm/desk/templates/_helpers.tpl @@ -148,6 +148,25 @@ Requires top level scope {{ include "desk.fullname" . }}-frontend {{- end }} + +{{/* +Full name for the Celery Worker + +Requires top level scope +*/}} +{{- define "desk.celery.worker.fullname" -}} +{{ include "desk.fullname" . }}-celery-worker +{{- end }} + +{{/* +Full name for the Celery Beat + +Requires top level scope +*/}} +{{- define "desk.celery.beat.fullname" -}} +{{ include "desk.fullname" . }}-celery-beat +{{- end }} + {{/* Usage : {{ include "desk.secret.dockerconfigjson.name" (dict "fullname" (include "desk.fullname" .) "imageCredentials" .Values.path.to.the.image1) }} */}} diff --git a/src/helm/desk/templates/celery_beat_deployment.yaml b/src/helm/desk/templates/celery_beat_deployment.yaml new file mode 100644 index 0000000..4346a92 --- /dev/null +++ b/src/helm/desk/templates/celery_beat_deployment.yaml @@ -0,0 +1,140 @@ +{{- $backendEnvVars := include "desk.common.env" (list . .Values.backend) -}} +{{- $envVars := include "desk.common.env" (list . .Values.celeryBeat) -}} +{{- $fullName := include "desk.celery.beat.fullname" . -}} +{{- $component := "celeryBeat" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + annotations: + {{- with .Values.celeryBeat.dpAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "desk.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ .Values.celeryBeat.replicas }} + selector: + matchLabels: + {{- include "desk.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.celeryBeat.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "desk.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "desk.secret.dockerconfigjson.name" (dict "fullname" (include "desk.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + shareProcessNamespace: {{ .Values.celeryBeat.shareProcessNamespace }} + containers: + {{- with .Values.celeryBeat.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.backend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.backend.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.backend.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.celeryBeat.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.celeryBeat.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if $backendEnvVars}} + {{- $backendEnvVars | indent 12 }} + {{- end }} + {{- if $envVars}} + {{- $envVars | indent 12 }} + {{- end }} + {{- with .Values.celeryBeat.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.celeryBeat.probes.liveness }} + livenessProbe: + {{- include "desk.probes.abstract" (merge .Values.celeryBeat.probes.liveness (dict "targetPort" .Values.celeryBeat.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.celeryBeat.probes.readiness }} + readinessProbe: + {{- include "desk.probes.abstract" (merge .Values.celeryBeat.probes.readiness (dict "targetPort" .Values.celeryBeat.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.celeryBeat.probes.startup }} + startupProbe: + {{- include "desk.probes.abstract" (merge .Values.celeryBeat.probes.startup (dict "targetPort" .Values.celeryBeat.service.targetPort )) | nindent 12 }} + {{- end }} + {{- with .Values.celeryBeat.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.celeryBeat.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.celeryBeat.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.celeryBeat.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.celeryBeat.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.celeryBeat.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "desk.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.celeryBeat.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.celeryBeat.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/src/helm/desk/templates/celery_worker_deployment.yaml b/src/helm/desk/templates/celery_worker_deployment.yaml new file mode 100644 index 0000000..ed13e38 --- /dev/null +++ b/src/helm/desk/templates/celery_worker_deployment.yaml @@ -0,0 +1,140 @@ +{{- $backendEnvVars := include "desk.common.env" (list . .Values.backend) -}} +{{- $envVars := include "desk.common.env" (list . .Values.celeryWorker) -}} +{{- $fullName := include "desk.celery.worker.fullname" . -}} +{{- $component := "celery-worker" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + annotations: + {{- with .Values.celeryWorker.dpAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "desk.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ .Values.celeryWorker.replicas }} + selector: + matchLabels: + {{- include "desk.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.celeryWorker.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "desk.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "desk.secret.dockerconfigjson.name" (dict "fullname" (include "desk.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + shareProcessNamespace: {{ .Values.celeryWorker.shareProcessNamespace }} + containers: + {{- with .Values.celeryWorker.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.backend.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.backend.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.backend.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.celeryWorker.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.celeryWorker.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + env: + {{- if $backendEnvVars}} + {{- $backendEnvVars | indent 12 }} + {{- end }} + {{- if $envVars}} + {{- $envVars | indent 12 }} + {{- end }} + {{- with .Values.celeryWorker.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.celeryWorker.probes.liveness }} + livenessProbe: + {{- include "desk.probes.abstract" (merge .Values.celeryWorker.probes.liveness (dict "targetPort" .Values.celeryWorker.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.celeryWorker.probes.readiness }} + readinessProbe: + {{- include "desk.probes.abstract" (merge .Values.celeryWorker.probes.readiness (dict "targetPort" .Values.celeryWorker.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.celeryWorker.probes.startup }} + startupProbe: + {{- include "desk.probes.abstract" (merge .Values.celeryWorker.probes.startup (dict "targetPort" .Values.celeryWorker.service.targetPort )) | nindent 12 }} + {{- end }} + {{- with .Values.celeryWorker.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.celeryWorker.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.celeryWorker.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.celeryWorker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.celeryWorker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.celeryWorker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "desk.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.celeryWorker.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.celeryWorker.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} \ No newline at end of file diff --git a/src/helm/desk/values.yaml b/src/helm/desk/values.yaml index ddbbb14..aaa5daf 100644 --- a/src/helm/desk/values.yaml +++ b/src/helm/desk/values.yaml @@ -277,3 +277,141 @@ frontend: ## @param frontend.extraVolumes Additional volumes to mount on the frontend. extraVolumes: [] + + +## @section celeryWorker + +celeryWorker: + ## @param celeryWorker.dpAnnotations Annotations to add to the celeryWorker Deployment + dpAnnotations: {} + + ## @param celeryWorker.command Override the celeryWorker container command + command: ["celery", "-A", "people.celery_app", "worker"] + + ## @param celeryWorker.args Override the celeryWorker container args + args: [] + + ## @param celeryWorker.replicas Amount of celeryWorker replicas + replicas: 1 + + ## @param celeryWorker.shareProcessNamespace Enable share process namespace between containers + shareProcessNamespace: false + + ## @param celeryWorker.sidecars Add sidecars containers to celeryWorker deployment + sidecars: [] + + ## @param celeryWorker.migrateJobAnnotations Annotations for the migrate job + migrateJobAnnotations: {} + + ## @param celeryWorker.securityContext Configure celeryWorker Pod security context + securityContext: null + + ## @param celeryWorker.envVars Configure celeryWorker container environment variables + ## @extra celeryWorker.envVars.BY_VALUE Example environment variable by setting value directly + ## @extra celeryWorker.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap + ## @extra celeryWorker.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap + ## @extra celeryWorker.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret + ## @extra celeryWorker.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret + ## @skip celeryWorker.envVars + envVars: + <<: *commonEnvVars + + ## @param celeryWorker.podAnnotations Annotations to add to the celeryWorker Pod + podAnnotations: {} + + ## @param celeryWorker.probes Configure celeryWorker probes + ## @param celeryWorker.probes.liveness.path [nullable] Configure path for celeryWorker HTTP liveness probe + ## @param celeryWorker.probes.liveness.targetPort [nullable] Configure port for celeryWorker HTTP liveness probe + ## @param celeryWorker.probes.liveness.initialDelaySeconds [nullable] Configure initial delay for celeryWorker liveness probe + ## @param celeryWorker.probes.liveness.initialDelaySeconds [nullable] Configure timeout for celeryWorker liveness probe + ## @param celeryWorker.probes.startup.path [nullable] Configure path for celeryWorker HTTP startup probe + ## @param celeryWorker.probes.startup.targetPort [nullable] Configure port for celeryWorker HTTP startup probe + ## @param celeryWorker.probes.startup.initialDelaySeconds [nullable] Configure initial delay for celeryWorker startup probe + ## @param celeryWorker.probes.startup.initialDelaySeconds [nullable] Configure timeout for celeryWorker startup probe + ## @param celeryWorker.probes.readiness.path [nullable] Configure path for celeryWorker HTTP readiness probe + ## @param celeryWorker.probes.readiness.targetPort [nullable] Configure port for celeryWorker HTTP readiness probe + ## @param celeryWorker.probes.readiness.initialDelaySeconds [nullable] Configure initial delay for celeryWorker readiness probe + ## @param celeryWorker.probes.readiness.initialDelaySeconds [nullable] Configure timeout for celeryWorker readiness probe + probes: {} + + ## @param celeryWorker.resources Resource requirements for the celeryWorker container + resources: {} + + ## @param celeryWorker.nodeSelector Node selector for the celeryWorker Pod + nodeSelector: {} + + ## @param celeryWorker.tolerations Tolerations for the celeryWorker Pod + tolerations: [] + + ## @param celeryWorker.affinity Affinity for the celeryWorker Pod + affinity: {} + + ## @param celeryWorker.persistence Additional volumes to create and mount on the celeryWorker. Used for debugging purposes + ## @extra celeryWorker.persistence.volume-name.size Size of the additional volume + ## @extra celeryWorker.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir + ## @extra celeryWorker.persistence.volume-name.mountPath Path where the volume should be mounted to + persistence: {} + + ## @param celeryWorker.extraVolumeMounts Additional volumes to mount on the celeryWorker. + extraVolumeMounts: [] + + ## @param celeryWorker.extraVolumes Additional volumes to mount on the celeryWorker. + extraVolumes: [] + + +## @section celeryBeat + +celeryBeat: + ## @param celeryBeat.dpAnnotations Annotations to add to the celeryBeat Deployment + dpAnnotations: {} + + ## @param celeryBeat.command Override the celeryBeat container command + command: ["celery", "-A", "people.celery_app", "beat"] + + ## @param celeryBeat.args Override the celeryBeat container args + args: [] + + ## @param celeryBeat.replicas Amount of celeryBeat replicas + replicas: 1 + + ## @param celeryBeat.shareProcessNamespace Enable share process namespace between containers + shareProcessNamespace: false + + ## @param celeryBeat.sidecars Add sidecars containers to celeryBeat deployment + sidecars: [] + + ## @param celeryBeat.securityContext Configure celeryBeat Pod security context + securityContext: null + + ## @param celeryBeat.envVars Configure celeryBeat container environment variables + ## @extra celeryBeat.envVars.BY_VALUE Example environment variable by setting value directly + ## @extra celeryBeat.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap + ## @extra celeryBeat.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap + ## @extra celeryBeat.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret + ## @extra celeryBeat.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret + envVars: + <<: *commonEnvVars + + ## @param celeryBeat.probes Configure celeryBeat probes + probes: {} + + ## @param celeryBeat.resources Resource requirements for the celeryBeat container + resources: {} + + ## @param celeryBeat.nodeSelector Node selector for the celeryBeat Pod + nodeSelector: {} + + ## @param celeryBeat.tolerations Tolerations for the celeryBeat Pod + tolerations: [] + + ## @param celeryBeat.affinity Affinity for the celeryBeat Pod + affinity: {} + + ## @param celeryBeat.persistence Additional volumes to create and mount on the celeryBeat. Used for debugging purposes + persistence: {} + + ## @param celeryBeat.extraVolumeMounts Additional volumes to mount on the celeryBeat. + extraVolumeMounts: [] + + ## @param celeryBeat.extraVolumes Additional volumes to mount on the celeryBeat. + extraVolumes: [] diff --git a/src/helm/env.d/dev-keycloak/values.desk.yaml.gotmpl b/src/helm/env.d/dev-keycloak/values.desk.yaml.gotmpl index b14b717..8ecd58d 100644 --- a/src/helm/env.d/dev-keycloak/values.desk.yaml.gotmpl +++ b/src/helm/env.d/dev-keycloak/values.desk.yaml.gotmpl @@ -6,6 +6,7 @@ image: backend: replicas: 1 envVars: + CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1 DJANGO_CSRF_TRUSTED_ORIGINS: https://desk.127.0.0.1.nip.io,http://desk.127.0.0.1.nip.io DJANGO_CONFIGURATION: Local DJANGO_ALLOWED_HOSTS: "*" diff --git a/src/helm/env.d/dev/values.desk.yaml.gotmpl b/src/helm/env.d/dev/values.desk.yaml.gotmpl index 59a2eb9..0e29122 100644 --- a/src/helm/env.d/dev/values.desk.yaml.gotmpl +++ b/src/helm/env.d/dev/values.desk.yaml.gotmpl @@ -22,6 +22,7 @@ image: backend: replicas: 1 envVars: + CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1 DJANGO_CSRF_TRUSTED_ORIGINS: https://desk.127.0.0.1.nip.io,http://desk.127.0.0.1.nip.io DJANGO_CONFIGURATION: Local DJANGO_ALLOWED_HOSTS: "*"