✨(livekit) add Livekit Egress

Egress is already deployed in staging. But, while
working locally on feature relying on Egress, it's not
suitable to test your development or iterate.

Especially I'll need to test the connection between the Egress
and the minio bucket in my next PR.

We faced quite a few issue while starting the whole stack.
Egress didn't want to start. Its connection with the livekit server
while the egress participant was joining the room was not successful.

The Turn part of the livekit server helm chart was activated. We needed
to update few values to in the helm configuration to enabled this turn.

Updated CoreDNS to expose Egress pod. Egress tries connecting to MinIO at
127.0.0.1, where no instance exists. Using minio.127.0.0.1.nip.io resolves
to 127.0.0.1, causing Egress to connect to itself for uploads. The CoreDNS
rewrite directs this to the Ingress IP, correctly routing to MinIO.

bin/start-kind.sh

@@ -97,6 +97,41 @@ data:
     help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
 EOF
 
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+          lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+          pods insecure
+          fallthrough in-addr.arpa ip6.arpa
+          ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+          max_concurrent 1000
+        }
+        rewrite stop {
+          name regex (.*).127.0.0.1.nip.io ingress-nginx-controller.ingress-nginx.svc.cluster.local answer auto
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+EOF
+
+kubectl -n kube-system rollout restart deployments/coredns
+
 echo "6. Install ingress-nginx"
 kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
 kubectl -n ingress-nginx create secret tls mkcert --key /tmp/127.0.0.1.nip.io+1-key.pem --cert /tmp/127.0.0.1.nip.io+1.pem

src/helm/env.d/dev/values.egress.yaml.gotmpl

@@ -0,0 +1,43 @@
+replicaCount: 1
+terminationGracePeriodSeconds: 18000
+
+egress:
+  log_level: debug
+  ws_url: ws://livekit-livekit-server:80
+  insecure: true
+  enable_chrome_sandbox: true
+  {{- with .Values.livekit.keys }}
+  {{- range $key, $value := . }}
+  api_key: {{ $key }}
+  api_secret: {{ $value }}
+  {{- end }}
+  {{- end }}
+  redis:
+    address: redis-master:6379
+    password: pass
+  s3:
+    access_key: meet
+    secret: password
+    region: local
+    bucket: meet-media-storage
+    endpoint: http://minio:9000
+    force_path_style: true
+
+loadBalancer:
+  type: nginx
+  annotations:
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+  tls:
+    - hosts:
+        - livekit-egress.127.0.0.1.nip.io
+      secretName: livekit-egress-dinum-cert
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 5
+
+nodeSelector: {}
+resources: {}

src/helm/env.d/dev/values.livekit.yaml.gotmpl

@@ -4,13 +4,20 @@ terminationGracePeriodSeconds: 18000
 livekit:
   log_level: debug
   rtc:
-    use_external_ip: true
+    use_external_ip: false
     port_range_start: 50000
     port_range_end: 60000
     tcp_port: 7881
   redis:
-    address:
+    address: redis-master:6379
+    password: pass
   keys:
+  turn:
+    enabled: true
+    udp_port: 443
+    domain: livekit.127.0.0.1.nip.io
+    loadBalancerAnnotations: {}
+
 
 loadBalancer:
   type: nginx

src/helm/helmfile.yaml

@@ -100,3 +100,12 @@ releases:
       - env.d/{{ .Environment.Name }}/values.livekit.yaml.gotmpl
     secrets:
       - env.d/{{ .Environment.Name }}/secrets.enc.yaml
+
+  - name: livekit-egress
+    installed: {{ eq .Environment.Name "dev" | toYaml }}
+    namespace: {{ .Namespace }}
+    chart: livekit/egress
+    values:
+      - env.d/{{ .Environment.Name }}/values.egress.yaml.gotmpl
+    secrets:
+      - env.d/{{ .Environment.Name }}/secrets.enc.yaml