From f7ed70dc9c36a8ede3cb1c0ea9d5fd6dadd8b720 Mon Sep 17 00:00:00 2001
From: lebaudantoine <lebaud.antoine131@gmail.com>
Date: Fri, 25 Oct 2024 13:58:15 +0200
Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(livekit)=20add=20Livekit=20Egress?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Egress is already deployed in staging. But, while
working locally on feature relying on Egress, it's not
suitable to test your development or iterate.

Especially I'll need to test the connection between the Egress
and the minio bucket in my next PR.

We faced quite a few issue while starting the whole stack.
Egress didn't want to start. Its connection with the livekit server
while the egress participant was joining the room was not successful.

The Turn part of the livekit server helm chart was activated. We needed
to update few values to in the helm configuration to enabled this turn.

Updated CoreDNS to expose Egress pod. Egress tries connecting to MinIO at
127.0.0.1, where no instance exists. Using minio.127.0.0.1.nip.io resolves
to 127.0.0.1, causing Egress to connect to itself for uploads. The CoreDNS
rewrite directs this to the Ingress IP, correctly routing to MinIO.
---
 bin/start-kind.sh                             | 35 +++++++++++++++
 src/helm/env.d/dev/values.egress.yaml.gotmpl  | 43 +++++++++++++++++++
 src/helm/env.d/dev/values.livekit.yaml.gotmpl | 11 ++++-
 src/helm/helmfile.yaml                        |  9 ++++
 4 files changed, 96 insertions(+), 2 deletions(-)
 create mode 100644 src/helm/env.d/dev/values.egress.yaml.gotmpl

diff --git a/bin/start-kind.sh b/bin/start-kind.sh
index a77f12f6..99595407 100755
--- a/bin/start-kind.sh
+++ b/bin/start-kind.sh
@@ -97,6 +97,41 @@ data:
     help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+          lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+          max_concurrent 1000
+        }
+        rewrite stop {
+          name regex (.*).127.0.0.1.nip.io ingress-nginx-controller.ingress-nginx.svc.cluster.local answer auto
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+EOF
+
+kubectl -n kube-system rollout restart deployments/coredns
+
 echo "6. Install ingress-nginx"
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
 kubectl -n ingress-nginx create secret tls mkcert --key /tmp/127.0.0.1.nip.io+1-key.pem --cert /tmp/127.0.0.1.nip.io+1.pem
diff --git a/src/helm/env.d/dev/values.egress.yaml.gotmpl b/src/helm/env.d/dev/values.egress.yaml.gotmpl
new file mode 100644
index 00000000..5d09dfca
--- /dev/null
+++ b/src/helm/env.d/dev/values.egress.yaml.gotmpl
@@ -0,0 +1,43 @@
+replicaCount: 1
+terminationGracePeriodSeconds: 18000
+
+egress:
+  log_level: debug
+  ws_url: ws://livekit-livekit-server:80
+  insecure: true
+  enable_chrome_sandbox: true
+  {{- with .Values.livekit.keys }}
+  {{- range $key, $value := . }}
+  api_key: {{ $key }}
+  api_secret: {{ $value }}
+  {{- end }}
+  {{- end }}
+  redis:
+    address: redis-master:6379
+    password: pass
+  s3:
+    access_key: meet
+    secret: password
+    region: local
+    bucket: meet-media-storage
+    endpoint: http://minio:9000
+    force_path_style: true
+
+loadBalancer:
+  type: nginx
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+  tls:
+    - hosts:
+        - livekit-egress.127.0.0.1.nip.io
+      secretName: livekit-egress-dinum-cert
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 5
+
+nodeSelector: {}
+resources: {}
diff --git a/src/helm/env.d/dev/values.livekit.yaml.gotmpl b/src/helm/env.d/dev/values.livekit.yaml.gotmpl
index e223774d..1671e2b0 100644
--- a/src/helm/env.d/dev/values.livekit.yaml.gotmpl
+++ b/src/helm/env.d/dev/values.livekit.yaml.gotmpl
@@ -4,13 +4,20 @@ terminationGracePeriodSeconds: 18000
 livekit:
   log_level: debug
   rtc:
-    use_external_ip: true
+    use_external_ip: false
     port_range_start: 50000
     port_range_end: 60000
     tcp_port: 7881
   redis:
-    address:
+    address: redis-master:6379
+    password: pass
   keys:
+  turn:
+    enabled: true
+    udp_port: 443
+    domain: livekit.127.0.0.1.nip.io
+    loadBalancerAnnotations: {}
+
 
 loadBalancer:
   type: nginx
diff --git a/src/helm/helmfile.yaml b/src/helm/helmfile.yaml
index a24e1a54..e17bc48b 100644
--- a/src/helm/helmfile.yaml
+++ b/src/helm/helmfile.yaml
@@ -100,3 +100,12 @@ releases:
       - env.d/{{ .Environment.Name }}/values.livekit.yaml.gotmpl
     secrets:
       - env.d/{{ .Environment.Name }}/secrets.enc.yaml
+
+  - name: livekit-egress
+    installed: {{ eq .Environment.Name "dev" | toYaml }}
+    namespace: {{ .Namespace }}
+    chart: livekit/egress
+    values:
+      - env.d/{{ .Environment.Name }}/values.egress.yaml.gotmpl
+    secrets:
+      - env.d/{{ .Environment.Name }}/secrets.enc.yaml