💩(summary) kubernitize the micro service

Add the micro service to the helm chart.
This commit is contained in:
lebaudantoine
2024-11-25 16:01:09 +01:00
committed by aleb_the_flash
parent e92f084afb
commit dd0cb61ebc
9 changed files with 573 additions and 8 deletions

View File

@@ -27,6 +27,17 @@ docker_build(
] ]
) )
docker_build(
'localhost:5001/meet-summary:latest',
context='../src/summary',
dockerfile='../src/summary/Dockerfile',
only=['.', '../../docker', '../../.dockerignore'],
target = 'production',
live_update=[
sync('../src/summary', '/home/summary'),
]
)
k8s_yaml(local('cd ../src/helm && helmfile -n meet -e dev template .')) k8s_yaml(local('cd ../src/helm && helmfile -n meet -e dev template .'))
migration = ''' migration = '''

View File

@@ -113,3 +113,61 @@ posthog:
ingressAssets: ingressAssets:
enabled: false enabled: false
summary:
replicas: 1
envVars:
APP_NAME: summary-microservice
APP_API_TOKEN: password
AWS_STORAGE_BUCKET_NAME: meet-media-storage
AWS_S3_ENDPOINT_URL: minio.meet.svc.cluster.local:9000
AWS_S3_ACCESS_KEY_ID: meet
AWS_S3_SECRET_ACCESS_KEY: password
OPENAI_API_KEY: password
AWS_S3_SECURE_ACCESS: False
WEBHOOK_API_TOKEN: password
WEBHOOK_URL: https://www.mock-impress.com/webhook/
CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1
CELERY_RESULT_BACKEND: redis://default:pass@redis-master:6379/1
image:
repository: localhost:5001/meet-summary
pullPolicy: Always
tag: "latest"
command:
- "uvicorn"
- "summary.main:app"
- "--host"
- "0.0.0.0"
- "--port"
- "8000"
- "--reload"
celery:
replicas: 1
envVars:
APP_NAME: summary-microservice
APP_API_TOKEN: password
AWS_STORAGE_BUCKET_NAME: meet-media-storage
AWS_S3_ENDPOINT_URL: minio.meet.svc.cluster.local:9000
AWS_S3_ACCESS_KEY_ID: meet
AWS_S3_SECRET_ACCESS_KEY: password
OPENAI_API_KEY: password
AWS_S3_SECURE_ACCESS: False
WEBHOOK_API_TOKEN: password
WEBHOOK_URL: https://www.mock-impress.com/webhook/
CELERY_BROKER_URL: redis://default:pass@redis-master:6379/1
CELERY_RESULT_BACKEND: redis://default:pass@redis-master:6379/1
image:
repository: localhost:5001/meet-summary
pullPolicy: Always
tag: "latest"
command:
- "celery"
- "-A"
- "summary.celery_worker"
- "worker"
- "--pool=solo"
- "--loglevel=info"

View File

@@ -166,6 +166,24 @@ Requires top level scope
{{ include "meet.fullname" . }}-posthog {{ include "meet.fullname" . }}-posthog
{{- end }} {{- end }}
{{/*
Full name for the summary
Requires top level scope
*/}}
{{- define "meet.summary.fullname" -}}
{{ include "meet.fullname" . }}-summary
{{- end }}
{{/*
Full name for the Celery
Requires top level scope
*/}}
{{- define "meet.celery.fullname" -}}
{{ include "meet.fullname" . }}-celery
{{- end }}
{{/* {{/*
Usage : {{ include "meet.secret.dockerconfigjson.name" (dict "fullname" (include "meet.fullname" .) "imageCredentials" .Values.path.to.the.image1) }} Usage : {{ include "meet.secret.dockerconfigjson.name" (dict "fullname" (include "meet.fullname" .) "imageCredentials" .Values.path.to.the.image1) }}
*/}} */}}

View File

@@ -0,0 +1,137 @@
{{- $envVars := include "meet.common.env" (list . .Values.celery) -}}
{{- $fullName := include "meet.celery.fullname" . -}}
{{- $component := "celery" -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $fullName }}
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "meet.common.labels" (list . $component) | nindent 4 }}
spec:
replicas: {{ .Values.celery.replicas }}
selector:
matchLabels:
{{- include "meet.common.selectorLabels" (list . $component) | nindent 6 }}
template:
metadata:
annotations:
{{- with .Values.celery.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
checksum/config: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
labels:
{{- include "meet.common.selectorLabels" (list . $component) | nindent 8 }}
spec:
{{- if $.Values.image.credentials }}
imagePullSecrets:
- name: {{ include "meet.secret.dockerconfigjson.name" (dict "fullname" (include "meet.fullname" .) "imageCredentials" $.Values.image.credentials) }}
{{- end}}
shareProcessNamespace: {{ .Values.celery.shareProcessNamespace }}
containers:
{{- with .Values.celery.sidecars }}
{{- toYaml . | nindent 8 }}
{{- end }}
- name: {{ .Chart.Name }}
image: "{{ (.Values.celery.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.celery.image | default dict).tag | default .Values.image.tag }}"
imagePullPolicy: {{ (.Values.celery.image | default dict).pullPolicy | default .Values.image.pullPolicy }}
{{- with .Values.celery.command }}
command:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.celery.args }}
args:
{{- toYaml . | nindent 12 }}
{{- end }}
env:
{{- if $envVars}}
{{- $envVars | indent 12 }}
{{- end }}
{{- with .Values.celery.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.celery.service.targetPort }}
protocol: TCP
{{- if .Values.celery.probes.liveness }}
livenessProbe:
{{- include "meet.probes.abstract" (merge .Values.celery.probes.liveness (dict "targetPort" .Values.celery.service.targetPort )) | nindent 12 }}
{{- end }}
{{- if .Values.celery.probes.readiness }}
readinessProbe:
{{- include "meet.probes.abstract" (merge .Values.celery.probes.readiness (dict "targetPort" .Values.celery.service.targetPort )) | nindent 12 }}
{{- end }}
{{- if .Values.celery.probes.startup }}
startupProbe:
{{- include "meet.probes.abstract" (merge .Values.celery.probes.startup (dict "targetPort" .Values.celery.service.targetPort )) | nindent 12 }}
{{- end }}
{{- with .Values.celery.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
{{- range $index, $value := .Values.mountFiles }}
- name: "files-{{ $index }}"
mountPath: {{ $value.path }}
subPath: content
{{- end }}
{{- range $name, $volume := .Values.celery.persistence }}
- name: "{{ $name }}"
mountPath: "{{ $volume.mountPath }}"
{{- end }}
{{- range .Values.celery.extraVolumeMounts }}
- name: {{ .name }}
mountPath: {{ .mountPath }}
subPath: {{ .subPath | default "" }}
readOnly: {{ .readOnly }}
{{- end }}
{{- with .Values.celery.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.celery.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.celery.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
{{- range $index, $value := .Values.mountFiles }}
- name: "files-{{ $index }}"
configMap:
name: "{{ include "meet.fullname" $ }}-files-{{ $index }}"
{{- end }}
{{- range $name, $volume := .Values.celery.persistence }}
- name: "{{ $name }}"
{{- if eq $volume.type "emptyDir" }}
emptyDir: {}
{{- else }}
persistentVolumeClaim:
claimName: "{{ $fullName }}-{{ $name }}"
{{- end }}
{{- end }}
{{- range .Values.celery.extraVolumes }}
- name: {{ .name }}
{{- if .existingClaim }}
persistentVolumeClaim:
claimName: {{ .existingClaim }}
{{- else if .hostPath }}
hostPath:
{{ toYaml .hostPath | nindent 12 }}
{{- else if .csi }}
csi:
{{- toYaml .csi | nindent 12 }}
{{- else if .configMap }}
configMap:
{{- toYaml .configMap | nindent 12 }}
{{- else if .emptyDir }}
emptyDir:
{{- toYaml .emptyDir | nindent 12 }}
{{- else }}
emptyDir: {}
{{- end }}
{{- end }}

View File

@@ -0,0 +1,137 @@
{{- $envVars := include "meet.common.env" (list . .Values.summary) -}}
{{- $fullName := include "meet.summary.fullname" . -}}
{{- $component := "summary" -}}
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ $fullName }}
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "meet.common.labels" (list . $component) | nindent 4 }}
spec:
replicas: {{ .Values.summary.replicas }}
selector:
matchLabels:
{{- include "meet.common.selectorLabels" (list . $component) | nindent 6 }}
template:
metadata:
annotations:
{{- with .Values.summary.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
checksum/config: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
labels:
{{- include "meet.common.selectorLabels" (list . $component) | nindent 8 }}
spec:
{{- if $.Values.image.credentials }}
imagePullSecrets:
- name: {{ include "meet.secret.dockerconfigjson.name" (dict "fullname" (include "meet.fullname" .) "imageCredentials" $.Values.image.credentials) }}
{{- end}}
shareProcessNamespace: {{ .Values.summary.shareProcessNamespace }}
containers:
{{- with .Values.summary.sidecars }}
{{- toYaml . | nindent 8 }}
{{- end }}
- name: {{ .Chart.Name }}
image: "{{ (.Values.summary.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.summary.image | default dict).tag | default .Values.image.tag }}"
imagePullPolicy: {{ (.Values.summary.image | default dict).pullPolicy | default .Values.image.pullPolicy }}
{{- with .Values.summary.command }}
command:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.summary.args }}
args:
{{- toYaml . | nindent 12 }}
{{- end }}
env:
{{- if $envVars}}
{{- $envVars | indent 12 }}
{{- end }}
{{- with .Values.summary.securityContext }}
securityContext:
{{- toYaml . | nindent 12 }}
{{- end }}
ports:
- name: http
containerPort: {{ .Values.summary.service.targetPort }}
protocol: TCP
{{- if .Values.summary.probes.liveness }}
livenessProbe:
{{- include "meet.probes.abstract" (merge .Values.summary.probes.liveness (dict "targetPort" .Values.summary.service.targetPort )) | nindent 12 }}
{{- end }}
{{- if .Values.summary.probes.readiness }}
readinessProbe:
{{- include "meet.probes.abstract" (merge .Values.summary.probes.readiness (dict "targetPort" .Values.summary.service.targetPort )) | nindent 12 }}
{{- end }}
{{- if .Values.summary.probes.startup }}
startupProbe:
{{- include "meet.probes.abstract" (merge .Values.summary.probes.startup (dict "targetPort" .Values.summary.service.targetPort )) | nindent 12 }}
{{- end }}
{{- with .Values.summary.resources }}
resources:
{{- toYaml . | nindent 12 }}
{{- end }}
volumeMounts:
{{- range $index, $value := .Values.mountFiles }}
- name: "files-{{ $index }}"
mountPath: {{ $value.path }}
subPath: content
{{- end }}
{{- range $name, $volume := .Values.summary.persistence }}
- name: "{{ $name }}"
mountPath: "{{ $volume.mountPath }}"
{{- end }}
{{- range .Values.summary.extraVolumeMounts }}
- name: {{ .name }}
mountPath: {{ .mountPath }}
subPath: {{ .subPath | default "" }}
readOnly: {{ .readOnly }}
{{- end }}
{{- with .Values.summary.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.summary.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.summary.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
volumes:
{{- range $index, $value := .Values.mountFiles }}
- name: "files-{{ $index }}"
configMap:
name: "{{ include "meet.fullname" $ }}-files-{{ $index }}"
{{- end }}
{{- range $name, $volume := .Values.summary.persistence }}
- name: "{{ $name }}"
{{- if eq $volume.type "emptyDir" }}
emptyDir: {}
{{- else }}
persistentVolumeClaim:
claimName: "{{ $fullName }}-{{ $name }}"
{{- end }}
{{- end }}
{{- range .Values.summary.extraVolumes }}
- name: {{ .name }}
{{- if .existingClaim }}
persistentVolumeClaim:
claimName: {{ .existingClaim }}
{{- else if .hostPath }}
hostPath:
{{ toYaml .hostPath | nindent 12 }}
{{- else if .csi }}
csi:
{{- toYaml .csi | nindent 12 }}
{{- else if .configMap }}
configMap:
{{- toYaml .configMap | nindent 12 }}
{{- else if .emptyDir }}
emptyDir:
{{- toYaml .emptyDir | nindent 12 }}
{{- else }}
emptyDir: {}
{{- end }}
{{- end }}

View File

@@ -0,0 +1,21 @@
{{- $envVars := include "meet.common.env" (list . .Values.summary) -}}
{{- $fullName := include "meet.summary.fullname" . -}}
{{- $component := "summary" -}}
apiVersion: v1
kind: Service
metadata:
name: {{ $fullName }}
namespace: {{ .Release.Namespace | quote }}
labels:
{{- include "meet.common.labels" (list . $component) | nindent 4 }}
annotations:
{{- toYaml $.Values.summary.service.annotations | nindent 4 }}
spec:
type: {{ .Values.summary.service.type }}
ports:
- port: {{ .Values.summary.service.port }}
targetPort: {{ .Values.summary.service.targetPort }}
protocol: TCP
name: http
selector:
{{- include "meet.common.selectorLabels" (list . $component) | nindent 4 }}

View File

@@ -305,3 +305,184 @@ posthog:
externalName: eu-assets.i.posthog.com externalName: eu-assets.i.posthog.com
port: 443 port: 443
annotations: {} annotations: {}
## @section summary
summary:
## @param summary.command Override the summary container command
command: []
## @param summary.args Override the summary container args
args: []
## @param summary.replicas Amount of summary replicas
replicas: 1
## @param summary.shareProcessNamespace Enable share process namespace between containers
shareProcessNamespace: false
## @param summary.sidecars Add sidecars containers to summary deployment
sidecars: []
## @param summary.migrateJobAnnotations Annotations for the migrate job
migrateJobAnnotations: {}
## @param summary.securityContext Configure summary Pod security context
securityContext: null
## @param summary.envVars Configure summary container environment variables
## @extra summary.envVars.BY_VALUE Example environment variable by setting value directly
## @extra summary.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap
## @extra summary.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap
## @extra summary.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret
## @extra summary.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret
## @skip summary.envVars
envVars:
<<: *commonEnvVars
## @param summary.podAnnotations Annotations to add to the summary Pod
podAnnotations: {}
## @param summary.service.type summary Service type
## @param summary.service.port summary Service listening port
## @param summary.service.targetPort summary container listening port
## @param summary.service.annotations Annotations to add to the summary Service
service:
type: ClusterIP
port: 80
targetPort: 8000
annotations: {}
## @param summary.probes.liveness.path [nullable] Configure path for summary HTTP liveness probe
## @param summary.probes.liveness.targetPort [nullable] Configure port for summary HTTP liveness probe
## @param summary.probes.liveness.initialDelaySeconds [nullable] Configure initial delay for summary liveness probe
## @param summary.probes.liveness.initialDelaySeconds [nullable] Configure timeout for summary liveness probe
## @param summary.probes.startup.path [nullable] Configure path for summary HTTP startup probe
## @param summary.probes.startup.targetPort [nullable] Configure port for summary HTTP startup probe
## @param summary.probes.startup.initialDelaySeconds [nullable] Configure initial delay for summary startup probe
## @param summary.probes.startup.initialDelaySeconds [nullable] Configure timeout for summary startup probe
## @param summary.probes.readiness.path [nullable] Configure path for summary HTTP readiness probe
## @param summary.probes.readiness.targetPort [nullable] Configure port for summary HTTP readiness probe
## @param summary.probes.readiness.initialDelaySeconds [nullable] Configure initial delay for summary readiness probe
## @param summary.probes.readiness.initialDelaySeconds [nullable] Configure timeout for summary readiness probe
probes:
liveness:
path: /__heartbeat__
initialDelaySeconds: 30
readiness:
path: /__lbheartbeat__
initialDelaySeconds: 30
## @param summary.resources Resource requirements for the summary container
resources: {}
## @param summary.nodeSelector Node selector for the summary Pod
nodeSelector: {}
## @param summary.tolerations Tolerations for the summary Pod
tolerations: []
## @param summary.affinity Affinity for the summary Pod
affinity: {}
## @param summary.persistence Additional volumes to create and mount on the summary. Used for debugging purposes
## @extra summary.persistence.volume-name.size Size of the additional volume
## @extra summary.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir
## @extra summary.persistence.volume-name.mountPath Path where the volume should be mounted to
persistence: {}
## @param summary.extraVolumeMounts Additional volumes to mount on the summary.
extraVolumeMounts: []
## @param summary.extraVolumes Additional volumes to mount on the summary.
extraVolumes: []
## @section celery
celery:
## @param celery.command Override the celery container command
command: []
## @param celery.args Override the celery container args
args: []
## @param celery.replicas Amount of celery replicas
replicas: 1
## @param celery.shareProcessNamespace Enable share process namespace between containers
shareProcessNamespace: false
## @param celery.sidecars Add sidecars containers to celery deployment
sidecars: []
## @param celery.migrateJobAnnotations Annotations for the migrate job
migrateJobAnnotations: {}
## @param celery.securityContext Configure celery Pod security context
securityContext: null
## @param celery.envVars Configure celery container environment variables
## @extra celery.envVars.BY_VALUE Example environment variable by setting value directly
## @extra celery.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap
## @extra celery.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap
## @extra celery.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret
## @extra celery.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret
## @skip celery.envVars
envVars:
<<: *commonEnvVars
## @param celery.podAnnotations Annotations to add to the celery Pod
podAnnotations: {}
## @param celery.service.type celery Service type
## @param celery.service.port celery Service listening port
## @param celery.service.targetPort celery container listening port
## @param celery.service.annotations Annotations to add to the celery Service
service:
type: ClusterIP
port: 80
targetPort: 8000
annotations: {}
## @param celery.probes.liveness.path [nullable] Configure path for celery HTTP liveness probe
## @param celery.probes.liveness.targetPort [nullable] Configure port for celery HTTP liveness probe
## @param celery.probes.liveness.initialDelaySeconds [nullable] Configure initial delay for celery liveness probe
## @param celery.probes.liveness.initialDelaySeconds [nullable] Configure timeout for celery liveness probe
## @param celery.probes.startup.path [nullable] Configure path for celery HTTP startup probe
## @param celery.probes.startup.targetPort [nullable] Configure port for celery HTTP startup probe
## @param celery.probes.startup.initialDelaySeconds [nullable] Configure initial delay for celery startup probe
## @param celery.probes.startup.initialDelaySeconds [nullable] Configure timeout for celery startup probe
## @param celery.probes.readiness.path [nullable] Configure path for celery HTTP readiness probe
## @param celery.probes.readiness.targetPort [nullable] Configure port for celery HTTP readiness probe
## @param celery.probes.readiness.initialDelaySeconds [nullable] Configure initial delay for celery readiness probe
## @param celery.probes.readiness.initialDelaySeconds [nullable] Configure timeout for celery readiness probe
probes: {}
## @param celery.resources Resource requirements for the celery container
resources: {}
## @param celery.nodeSelector Node selector for the celery Pod
nodeSelector: {}
## @param celery.tolerations Tolerations for the celery Pod
tolerations: []
## @param celery.affinity Affinity for the celery Pod
affinity: {}
## @param celery.persistence Additional volumes to create and mount on the celery. Used for debugging purposes
## @extra celery.persistence.volume-name.size Size of the additional volume
## @extra celery.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir
## @extra celery.persistence.volume-name.mountPath Path where the volume should be mounted to
persistence: {}
## @param celery.extraVolumeMounts Additional volumes to mount on the celery.
extraVolumeMounts: []
## @param celery.extraVolumes Additional volumes to mount on the celery.
extraVolumes: []

View File

@@ -74,15 +74,16 @@ def process_audio_transcribe_summarize(filename: str, email: str, sub: str):
logger.debug("filename: %s", filename) logger.debug("filename: %s", filename)
minio_client = Minio( minio_client = Minio(
settings.minio_url, settings.aws_s3_endpoint_url,
access_key=settings.minio_access_key, access_key=settings.aws_s3_access_key_id,
secret_key=settings.minio_secret_key, secret_key=settings.aws_s3_secret_access_key,
secure=settings.aws_s3_secure_access,
) )
logger.debug("Connection to the Minio bucket successful") logger.debug("Connection to the Minio bucket successful")
audio_file_stream = minio_client.get_object( audio_file_stream = minio_client.get_object(
settings.minio_bucket, object_name=filename settings.aws_storage_bucket_name, object_name=filename
) )
temp_file_path = save_audio_stream(audio_file_stream) temp_file_path = save_audio_stream(audio_file_stream)

View File

@@ -19,10 +19,11 @@ class Settings(BaseSettings):
celery_result_backend: str = "redis://redis/0" celery_result_backend: str = "redis://redis/0"
# Minio settings # Minio settings
minio_bucket: str aws_storage_bucket_name: str
minio_url: str aws_s3_endpoint_url: str
minio_access_key: str aws_s3_access_key_id: str
minio_secret_key: str aws_s3_secret_access_key: str
aws_s3_secure_access: bool = True
# AI-related settings # AI-related settings
openai_api_key: str openai_api_key: str