feat: replace nginx placeholder with custom Pingora proxy; add Postfix MTA

Ingress: - Deploy custom sunbeam-proxy (Pingora/Rust) replacing nginx placeholder - HTTPS termination with mkcert (local) / rustls-acme (production) - Host-prefix routing with path-based sub-routing for auth virtual host: /oauth2 + /.well-known + /userinfo → Hydra, /kratos → Kratos (prefix stripped), default → login-ui - HTTP→HTTPS redirect, WebSocket passthrough, JSON audit logging, OTEL stub - cert-manager HTTP-01 ACME challenge routing via Ingress watcher - RBAC for Ingress watcher (pingora-watcher ClusterRole) - local overlay: hostPorts 80/443, LiveKit TURN demoted to ClusterIP to avoid klipper conflict Infrastructure: - socket_vmnet shared network for host↔VM reachability (192.168.105.2) - local-up.sh: cert-manager installation, eth1-based LIMA_IP detection, correct DOMAIN_SUFFIX sed substitution - Postfix MTA in lasuite namespace: outbound relay via Scaleway TEM, accepts SMTP from cluster pods - Kratos SMTP courier pointed at postfix.lasuite.svc.cluster.local:25 - Production overlay: cert-manager ClusterIssuer, ACME-enabled Pingora values
2026-03-01 16:25:11 +00:00
parent a589e6280d
commit cdddc334ff
15 changed files with 391 additions and 64 deletions
--- a/base/ingress/kustomization.yaml
+++ b/base/ingress/kustomization.yaml
@@ -5,6 +5,7 @@ namespace: ingress

 resources:
  - namespace.yaml
+  - pingora-rbac.yaml
  - pingora-deployment.yaml
  - pingora-service.yaml
  - pingora-config.yaml
--- a/base/ingress/pingora-config.yaml
+++ b/base/ingress/pingora-config.yaml
@@ -5,39 +5,39 @@ metadata:
  namespace: ingress
 data:
  config.toml: |
-    # Pingora hostname routing table
-    # The domain suffix (sunbeam.pt / <LIMA_IP>.sslip.io) is patched per overlay.
-    # TLS cert source (rustls-acme / mkcert) is patched per overlay.
-
-    [tls]
-    cert_path = "/etc/tls/tls.crt"
-    key_path  = "/etc/tls/tls.key"
-    # acme = true  # Uncommented in production overlay (rustls-acme + Let's Encrypt)
-    acme = false
+    # Sunbeam proxy config.
+    #
+    # Substitution placeholders (replaced by sed at deploy time):
+    #   DOMAIN_SUFFIX — e.g. <LIMA_IP>.sslip.io (local) or yourdomain.com (production)

    [listen]
    http  = "0.0.0.0:80"
    https = "0.0.0.0:443"

-    [turn]
-    backend             = "livekit.media.svc.cluster.local:7880"
-    udp_listen          = "0.0.0.0:3478"
-    relay_port_start    = 49152
-    relay_port_end      = 49252
+    [tls]
+    # Cert files are written here by the proxy on startup and on cert renewal
+    # via the K8s API.  The /etc/tls directory is an emptyDir volume.
+    cert_path = "/etc/tls/tls.crt"
+    key_path  = "/etc/tls/tls.key"

-    # Host-prefix → backend mapping.
-    # Pingora matches on the subdomain prefix regardless of domain suffix,
-    # so these routes work identically for sunbeam.pt and *.sslip.io.
+    [telemetry]
+    # Empty = OTEL disabled. Set to http://otel-collector.data.svc:4318 when ready.
+    otlp_endpoint = ""
+
+    # Host-prefix → backend routing table.
+    # The prefix is the subdomain before the first dot, so these routes work
+    # identically for yourdomain.com and *.sslip.io.
+    # Edit to match your own service names and namespaces.

    [[routes]]
    host_prefix = "docs"
    backend     = "http://docs.lasuite.svc.cluster.local:8000"
-    websocket   = true   # Y.js CRDT sync
+    websocket   = true

    [[routes]]
    host_prefix = "meet"
    backend     = "http://meet.lasuite.svc.cluster.local:8000"
-    websocket   = true   # LiveKit signaling
+    websocket   = true

    [[routes]]
    host_prefix = "drive"
@@ -50,7 +50,7 @@ data:
    [[routes]]
    host_prefix = "chat"
    backend     = "http://conversations.lasuite.svc.cluster.local:8000"
-    websocket   = true   # Vercel AI SDK streaming
+    websocket   = true

    [[routes]]
    host_prefix = "people"
@@ -58,12 +58,31 @@ data:

    [[routes]]
    host_prefix = "src"
-    backend     = "http://gitea.devtools.svc.cluster.local:3000"
-    websocket   = true   # Gitea Actions runner
+    backend     = "http://gitea-http.devtools.svc.cluster.local:3000"
+    websocket   = true

+    # auth: login-ui handles browser UI; Hydra handles OAuth2/OIDC; Kratos handles self-service flows.
    [[routes]]
    host_prefix = "auth"
-    backend     = "http://hydra.ory.svc.cluster.local:4444"
+    backend     = "http://login-ui.ory.svc.cluster.local:3000"
+
+      [[routes.paths]]
+      prefix   = "/oauth2"
+      backend  = "http://hydra-public.ory.svc.cluster.local:4444"
+
+      [[routes.paths]]
+      prefix   = "/.well-known"
+      backend  = "http://hydra-public.ory.svc.cluster.local:4444"
+
+      [[routes.paths]]
+      prefix   = "/userinfo"
+      backend  = "http://hydra-public.ory.svc.cluster.local:4444"
+
+      # /kratos prefix is stripped before forwarding so Kratos sees its native paths.
+      [[routes.paths]]
+      prefix        = "/kratos"
+      backend       = "http://kratos-public.ory.svc.cluster.local:4433"
+      strip_prefix  = true

    [[routes]]
    host_prefix = "s3"
--- a/base/ingress/pingora-deployment.yaml
+++ b/base/ingress/pingora-deployment.yaml
@@ -5,6 +5,9 @@ metadata:
  namespace: ingress
 spec:
  replicas: 1
+  # Recreate avoids rolling-update conflicts (single-node; hostPorts in local overlay)
+  strategy:
+    type: Recreate
  selector:
    matchLabels:
      app: pingora
@@ -16,9 +19,10 @@ spec:
        # Pingora terminates TLS at the mesh boundary; sidecar injection is disabled here
        linkerd.io/inject: disabled
    spec:
+      serviceAccountName: pingora
      containers:
        - name: pingora
-          image: nginx:alpine   # placeholder until custom Pingora image is built
+          image: sunbeam-proxy:latest   # overridden per overlay via kustomize images:
          ports:
            - name: http
              containerPort: 80
@@ -34,19 +38,20 @@ spec:
            - name: config
              mountPath: /etc/pingora
              readOnly: true
+            # /etc/tls is an emptyDir written by the proxy via the K8s API on
+            # startup and on cert renewal, so Pingora always reads a fresh cert
+            # without depending on kubelet volume-sync timing.
            - name: tls
              mountPath: /etc/tls
-              readOnly: true
          resources:
            limits:
-              memory: 64Mi
+              memory: 256Mi
            requests:
-              memory: 32Mi
-              cpu: 50m
+              memory: 128Mi
+              cpu: 100m
      volumes:
        - name: config
          configMap:
            name: pingora-config
        - name: tls
-          secret:
-            secretName: pingora-tls
+          emptyDir: {}
--- a/base/ingress/pingora-rbac.yaml
+++ b/base/ingress/pingora-rbac.yaml
@@ -0,0 +1,44 @@
+---
+# ServiceAccount used by the Pingora pod.
+# The watcher in sunbeam-proxy uses in-cluster credentials (this SA's token) to
+# watch the pingora-tls Secret and pingora-config ConfigMap for changes.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: pingora
+  namespace: ingress
+---
+# Minimal read-only role: list+watch on the two objects that drive cert reloads.
+# Scoped to the ingress namespace by the Role kind (not ClusterRole).
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pingora-watcher
+  namespace: ingress
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch"]
+  # Ingresses are watched to route cert-manager HTTP-01 challenges to the
+  # correct per-domain solver pod (one Ingress per challenge, created by
+  # cert-manager with the exact token path and solver Service name).
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pingora-watcher
+  namespace: ingress
+subjects:
+  - kind: ServiceAccount
+    name: pingora
+    namespace: ingress
+roleRef:
+  kind: Role
+  name: pingora-watcher
+  apiGroup: rbac.authorization.k8s.io
--- a/base/lasuite/kustomization.yaml
+++ b/base/lasuite/kustomization.yaml
@@ -5,6 +5,7 @@ namespace: lasuite

 resources:
  - namespace.yaml
+  - postfix-deployment.yaml
  - hive-config.yaml
  - hive-deployment.yaml
  - hive-service.yaml
--- a/base/lasuite/postfix-deployment.yaml
+++ b/base/lasuite/postfix-deployment.yaml
@@ -0,0 +1,81 @@
+# Postfix MTA for the Messages email platform.
+#
+# MTA-out: accepts SMTP from cluster-internal services (Kratos, Messages Django),
+#          signs with DKIM, and relays outbound via Scaleway TEM.
+#
+# MTA-in: receives inbound email from the internet (routed via Pingora on port 25).
+#         In local dev, no MX record points here so inbound never arrives.
+#
+# Credentials: Secret "postfix-tem-credentials" with keys:
+#   smtp_user     — Scaleway TEM SMTP username (project ID)
+#   smtp_password — Scaleway TEM SMTP password (API key)
+#
+# DKIM keys: Secret "postfix-dkim" with key:
+#   private.key   — DKIM private key for sunbeam.pt (generated once; add DNS TXT record)
+#   selector      — DKIM selector (e.g. "mail")
+#
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postfix
+  namespace: lasuite
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: postfix
+  template:
+    metadata:
+      labels:
+        app: postfix
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: postfix
+          image: boky/postfix:latest
+          ports:
+            - name: smtp
+              containerPort: 25
+              protocol: TCP
+          env:
+            # Accept mail from all cluster-internal pods.
+            - name: MYNETWORKS
+              value: "10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 127.0.0.0/8"
+            # Sending domain — replaced by sed at deploy time.
+            - name: ALLOWED_SENDER_DOMAINS
+              value: "DOMAIN_SUFFIX"
+            # Scaleway TEM outbound relay.
+            - name: RELAYHOST
+              value: "[smtp.tem.scw.cloud]:587"
+            - name: SASL_USER
+              valueFrom:
+                secretKeyRef:
+                  name: postfix-tem-credentials
+                  key: smtp_user
+                  optional: true  # allows pod to start before secret exists
+            - name: SASL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: postfix-tem-credentials
+                  key: smtp_password
+                  optional: true
+          resources:
+            limits:
+              memory: 64Mi
+            requests:
+              memory: 32Mi
+              cpu: 10m
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postfix
+  namespace: lasuite
+spec:
+  selector:
+    app: postfix
+  ports:
+    - name: smtp
+      port: 25
+      targetPort: 25
+      protocol: TCP
--- a/base/ory/kratos-values.yaml
+++ b/base/ory/kratos-values.yaml
@@ -39,7 +39,7 @@ kratos:

    courier:
      smtp:
-        connection_uri: "smtp://local:local@localhost:25/"
+        connection_uri: "smtp://postfix.lasuite.svc.cluster.local:25/?skip_ssl_verify=true"
        from_address: no-reply@DOMAIN_SUFFIX
        from_name: Sunbeam