diff --git a/base/devtools/beam-design.yaml b/base/devtools/beam-design.yaml new file mode 100644 index 0000000..e752d24 --- /dev/null +++ b/base/devtools/beam-design.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beam-design + namespace: devtools + labels: + app: beam-design +spec: + replicas: 1 + selector: + matchLabels: + app: beam-design + template: + metadata: + labels: + app: beam-design + spec: + containers: + - name: beam-design + image: src.sunbeam.pt/studio/beam-ui:latest + ports: + - containerPort: 8080 + protocol: TCP + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + runAsUser: 65534 + runAsNonRoot: true + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + name: beam-design + namespace: devtools + labels: + app: beam-design +spec: + selector: + app: beam-design + ports: + - port: 80 + targetPort: 8080 + protocol: TCP diff --git a/base/ingress/pingora-config.yaml b/base/ingress/pingora-config.yaml index d8f6382..f3d72e5 100644 --- a/base/ingress/pingora-config.yaml +++ b/base/ingress/pingora-config.yaml @@ -167,19 +167,7 @@ data: prefix = "/.well-known/" backend = "http://stalwart.stalwart.svc.cluster.local:8080" - # Stalwart OAuth2 endpoints (/authorize/code, /auth/token, /auth/device) - [[routes.paths]] - prefix = "/authorize" - backend = "http://stalwart.stalwart.svc.cluster.local:8080" - [[routes.paths]] - prefix = "/auth/" - backend = "http://stalwart.stalwart.svc.cluster.local:8080" - - # Stalwart login page (used during OAuth flow) - [[routes.paths]] - prefix = "/login" - backend = "http://stalwart.stalwart.svc.cluster.local:8080" [[routes]] host_prefix = "messages" @@ -401,20 +389,8 @@ data: host_prefix = "build" backend = "buildkitd.build.svc.cluster.local:1234" - # SMTP inbound: port 25 → Stalwart for mail delivery. - [smtp] - listen = "0.0.0.0:25" - backend = "stalwart.stalwart.svc.cluster.local:25" - - # SMTP submission: port 587 → Stalwart for authenticated sending. - [smtp-submission] - listen = "0.0.0.0:587" - backend = "stalwart.stalwart.svc.cluster.local:587" - - # IMAPS: port 993 → Stalwart for desktop/mobile email clients. - [imaps] - listen = "0.0.0.0:993" - backend = "stalwart.stalwart.svc.cluster.local:993" + # SMTP/IMAP ports are exposed directly on the Stalwart pod via hostPort + # (see overlays/production/kustomization.yaml), not through Pingora. # SSH TCP passthrough: port 22 → Gitea SSH pod (headless service → pod:2222). [ssh] diff --git a/base/ory/kratos-values.yaml b/base/ory/kratos-values.yaml index 9ad2e15..b6aa8e8 100644 --- a/base/ory/kratos-values.yaml +++ b/base/ory/kratos-values.yaml @@ -102,7 +102,7 @@ kratos: courier: smtp: - connection_uri: "smtp://postfix.lasuite.svc.cluster.local:25/?skip_ssl_verify=true" + connection_uri: "smtp://stalwart.stalwart.svc.cluster.local:25/?skip_ssl_verify=true" from_address: no-reply@DOMAIN_SUFFIX from_name: Sunbeam Studios diff --git a/base/stalwart/bulwark-deployment.yaml b/base/stalwart/bulwark-deployment.yaml new file mode 100644 index 0000000..a9a82a9 --- /dev/null +++ b/base/stalwart/bulwark-deployment.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark + namespace: stalwart +spec: + replicas: 1 + selector: + matchLabels: + app: bulwark + template: + metadata: + labels: + app: bulwark + spec: + containers: + - name: bulwark + image: src.DOMAIN_SUFFIX/studio/bulwark:latest + ports: + - name: http + containerPort: 3000 + env: + - name: JMAP_SERVER_URL + value: https://mail.DOMAIN_SUFFIX + - name: OAUTH_ENABLED + value: "true" + - name: OAUTH_ONLY + value: "true" + - name: LOG_LEVEL + value: "debug" + - name: OAUTH_SCOPES + value: "openid email profile offline_access" + - name: COOKIE_SECURE + value: "false" + - name: OAUTH_CLIENT_ID + valueFrom: + secretKeyRef: + name: oidc-bulwark + key: CLIENT_ID + - name: OAUTH_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: oidc-bulwark + key: CLIENT_SECRET + - name: OAUTH_ISSUER_URL + value: https://auth.DOMAIN_SUFFIX + - name: SESSION_SECRET + valueFrom: + secretKeyRef: + name: stalwart-app-secrets + key: admin-password + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 512Mi + cpu: 500m diff --git a/base/stalwart/bulwark-service.yaml b/base/stalwart/bulwark-service.yaml new file mode 100644 index 0000000..367ea18 --- /dev/null +++ b/base/stalwart/bulwark-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: bulwark + namespace: stalwart +spec: + selector: + app: bulwark + ports: + - name: http + port: 80 + targetPort: 3000 diff --git a/base/stalwart/certificate.yaml b/base/stalwart/certificate.yaml new file mode 100644 index 0000000..d40372a --- /dev/null +++ b/base/stalwart/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: stalwart-tls + namespace: stalwart +spec: + secretName: stalwart-tls + issuerRef: + name: letsencrypt-production + kind: ClusterIssuer + dnsNames: + - "mail.DOMAIN_SUFFIX" + - "cal.DOMAIN_SUFFIX" diff --git a/base/stalwart/namespace.yaml b/base/stalwart/namespace.yaml new file mode 100644 index 0000000..b0cfb7e --- /dev/null +++ b/base/stalwart/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: stalwart + labels: + linkerd.io/inject: enabled diff --git a/base/stalwart/oidc-client-bulwark.yaml b/base/stalwart/oidc-client-bulwark.yaml new file mode 100644 index 0000000..403e089 --- /dev/null +++ b/base/stalwart/oidc-client-bulwark.yaml @@ -0,0 +1,30 @@ +# Bulwark webmail OIDC client — authenticates directly with Hydra. +# Hydra Maester creates K8s Secret "oidc-bulwark" with CLIENT_ID/CLIENT_SECRET. +# DOMAIN_SUFFIX is replaced by sed at deploy time. +apiVersion: hydra.ory.sh/v1alpha1 +kind: OAuth2Client +metadata: + name: bulwark + namespace: stalwart +spec: + clientName: Webmail + grantTypes: + - authorization_code + - refresh_token + responseTypes: + - code + scope: openid email profile offline_access + redirectUris: + - https://mail.DOMAIN_SUFFIX/en/auth/callback + - https://mail.DOMAIN_SUFFIX/auth/callback + - https://mail.DOMAIN_SUFFIX/api/auth/callback + postLogoutRedirectUris: + - https://mail.DOMAIN_SUFFIX + tokenEndpointAuthMethod: client_secret_post + secretName: oidc-bulwark + skipConsent: true + tokenLifespans: + authorization_code_grant_access_token_lifespan: 24h + authorization_code_grant_refresh_token_lifespan: 720h + refresh_token_grant_access_token_lifespan: 24h + refresh_token_grant_refresh_token_lifespan: 720h diff --git a/base/stalwart/oidc-client.yaml b/base/stalwart/oidc-client.yaml new file mode 100644 index 0000000..174248d --- /dev/null +++ b/base/stalwart/oidc-client.yaml @@ -0,0 +1,24 @@ +# Stalwart OIDC client — registered with Hydra for SSO login. +# Hydra Maester creates K8s Secret "oidc-stalwart" in the stalwart namespace +# with CLIENT_ID and CLIENT_SECRET keys. +# DOMAIN_SUFFIX is replaced by sed at deploy time. +apiVersion: hydra.ory.sh/v1alpha1 +kind: OAuth2Client +metadata: + name: stalwart + namespace: stalwart +spec: + clientName: Mail + grantTypes: + - authorization_code + - refresh_token + responseTypes: + - code + scope: openid email profile + redirectUris: + - https://mail.DOMAIN_SUFFIX/authorize/code + postLogoutRedirectUris: + - https://mail.DOMAIN_SUFFIX + tokenEndpointAuthMethod: client_secret_post + secretName: oidc-stalwart + skipConsent: true diff --git a/base/stalwart/stalwart-config.yaml b/base/stalwart/stalwart-config.yaml new file mode 100644 index 0000000..8e999a8 --- /dev/null +++ b/base/stalwart/stalwart-config.yaml @@ -0,0 +1,148 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: stalwart-config + namespace: stalwart +data: + # DOMAIN_SUFFIX is replaced by sed at deploy time. + # Secrets (%{env:VAR}%) are injected via environment variables in the Deployment. + # + # Only LOCAL keys belong in this file (store.*, storage.*, server.*, directory.*, + # tracer.*, cluster.*, certificate.*, authentication.fallback-admin.*). + # Everything else (OIDC, DKIM, spam, etc.) is configured via the Stalwart + # web admin UI and stored in the database. + config.toml: | + # Force http.url to be read from this file (not the database). + config.local-keys.0000 = "store.*" + config.local-keys.0001 = "directory.*" + config.local-keys.0002 = "tracer.*" + config.local-keys.0003 = "server.*" + config.local-keys.0004 = "!server.blocked-ip.*" + config.local-keys.0005 = "!server.allowed-ip.*" + config.local-keys.0006 = "authentication.fallback-admin.*" + config.local-keys.0007 = "cluster.*" + config.local-keys.0008 = "config.local-keys.*" + config.local-keys.0009 = "storage.*" + config.local-keys.0010 = "certificate.*" + config.local-keys.0011 = "http.url" + config.local-keys.0012 = "http.use-x-forwarded" + + # Expression-quoted public URL (inner single quotes required). + http.url = "'https://mail.DOMAIN_SUFFIX'" + http.use-x-forwarded = true + + [server] + hostname = "mail.DOMAIN_SUFFIX" + max-connections = 1024 + + [server.listener."smtp"] + bind = ["0.0.0.0:25"] + protocol = "smtp" + + [server.listener."submission"] + bind = ["0.0.0.0:587"] + protocol = "smtp" + tls.implicit = false + + [server.listener."smtps"] + bind = ["0.0.0.0:465"] + protocol = "smtp" + tls.implicit = true + + [server.listener."imap"] + bind = ["0.0.0.0:143"] + protocol = "imap" + tls.implicit = false + + [server.listener."imaps"] + bind = ["0.0.0.0:993"] + protocol = "imap" + tls.implicit = true + + [server.listener."jmap"] + bind = ["0.0.0.0:8080"] + protocol = "http" + + [server.listener."managesieve"] + bind = ["0.0.0.0:4190"] + protocol = "managesieve" + + [certificate."default"] + cert = "%{file:/etc/stalwart-tls/tls.crt}%" + private-key = "%{file:/etc/stalwart-tls/tls.key}%" + + # ── Storage backends ───────────────────────────────────────────────────── + + [store."postgresql"] + type = "postgresql" + host = "postgres-rw.data.svc.cluster.local" + port = 5432 + database = "stalwart_db" + user = "stalwart" + password = "%{env:DB_PASSWORD}%" + timeout = "15s" + + [store."postgresql".pool] + max-connections = 20 + + [store."s3"] + type = "s3" + bucket = "sunbeam-stalwart" + endpoint = "http://seaweedfs-filer.storage.svc.cluster.local:8333" + region = "us-east-1" + access-key = "%{env:S3_ACCESS_KEY}%" + secret-key = "%{env:S3_SECRET_KEY}%" + key-prefix = "v1/" + timeout = "60s" + + [store."opensearch"] + type = "elasticsearch" + url = "http://opensearch.data.svc.cluster.local:9200" + index = "stalwart" + + [store."redis"] + type = "redis" + urls = ["redis://valkey.data.svc.cluster.local:6379/7"] + + # ── Storage role assignments ───────────────────────────────────────────── + + [storage] + data = "postgresql" + blob = "postgresql" + fts = "opensearch" + lookup = "redis" + directory = "hydra" + + # ── Directories (user stores) ────────────────────────────────────────── + + # Internal directory for locally-managed accounts (admin, service accounts). + [directory."internal"] + type = "internal" + store = "postgresql" + + # OIDC directory — validates Hydra-issued bearer tokens via userinfo. + # When a JMAP client presents a Bearer token, Stalwart calls Hydra's + # userinfo endpoint to map it to a user identity. + [directory."hydra"] + type = "oidc" + timeout = "15s" + endpoint.url = "http://hydra-public.ory.svc.cluster.local:4444/userinfo" + endpoint.method = "userinfo" + fields.email = "email" + fields.username = "email" + fields.full-name = "name" + + # ── Authentication ─────────────────────────────────────────────────────── + + [authentication.fallback-admin] + user = "admin" + secret = "%{env:ADMIN_PASSWORD}%" + + # ── Logging ────────────────────────────────────────────────────────────── + + [tracer."stdout"] + type = "stdout" + level = "info" + ansi = false + enable = true + diff --git a/base/stalwart/stalwart-deployment.yaml b/base/stalwart/stalwart-deployment.yaml new file mode 100644 index 0000000..51ea806 --- /dev/null +++ b/base/stalwart/stalwart-deployment.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: stalwart + namespace: stalwart +spec: + replicas: 1 + selector: + matchLabels: + app: stalwart + template: + metadata: + labels: + app: stalwart + spec: + containers: + - name: stalwart + image: stalwartlabs/stalwart:v0.15.5 + ports: + - name: smtp + containerPort: 25 + - name: submission + containerPort: 587 + - name: smtps + containerPort: 465 + - name: imap + containerPort: 143 + - name: imaps + containerPort: 993 + - name: managesieve + containerPort: 4190 + - name: http + containerPort: 8080 + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-db-credentials + key: password + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: seaweedfs-s3-credentials + key: S3_ACCESS_KEY + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: seaweedfs-s3-credentials + key: S3_SECRET_KEY + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-app-secrets + key: admin-password + - name: DKIM_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: stalwart-app-secrets + key: dkim-private-key + - name: OIDC_CLIENT_ID + valueFrom: + secretKeyRef: + name: oidc-stalwart + key: CLIENT_ID + - name: OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: oidc-stalwart + key: CLIENT_SECRET + volumeMounts: + - name: config + mountPath: /opt/stalwart/etc/config.toml + subPath: config.toml + readOnly: true + - name: tls + mountPath: /etc/stalwart-tls + readOnly: true + livenessProbe: + httpGet: + path: /healthz/live + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /healthz/ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 1Gi + cpu: "1" + volumes: + - name: config + configMap: + name: stalwart-config + - name: tls + secret: + secretName: stalwart-tls diff --git a/base/stalwart/stalwart-service.yaml b/base/stalwart/stalwart-service.yaml new file mode 100644 index 0000000..8e17c96 --- /dev/null +++ b/base/stalwart/stalwart-service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: stalwart + namespace: stalwart +spec: + selector: + app: stalwart + ports: + - name: smtp + port: 25 + targetPort: 25 + - name: submission + port: 587 + targetPort: 587 + - name: smtps + port: 465 + targetPort: 465 + - name: imap + port: 143 + targetPort: 143 + - name: imaps + port: 993 + targetPort: 993 + - name: managesieve + port: 4190 + targetPort: 4190 + - name: http + port: 8080 + targetPort: 8080 diff --git a/base/stalwart/vault-secrets.yaml b/base/stalwart/vault-secrets.yaml new file mode 100644 index 0000000..77acc64 --- /dev/null +++ b/base/stalwart/vault-secrets.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultAuth +metadata: + name: vso-auth + namespace: stalwart +spec: + method: kubernetes + mount: kubernetes + kubernetes: + role: vso + serviceAccount: default +--- +# Stalwart DB credentials from OpenBao database secrets engine (static role, 24h rotation). +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultDynamicSecret +metadata: + name: stalwart-db-credentials + namespace: stalwart +spec: + vaultAuthRef: vso-auth + mount: database + path: static-creds/stalwart + allowStaticCreds: true + refreshAfter: 5m + rolloutRestartTargets: + - kind: Deployment + name: stalwart + destination: + name: stalwart-db-credentials + create: true + overwrite: true + transformation: + excludeRaw: true + templates: + password: + text: "{{ index .Secrets \"password\" }}" +--- +# Stalwart application secrets (admin password, DKIM key) from OpenBao KV. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: stalwart-app-secrets + namespace: stalwart +spec: + vaultAuthRef: vso-auth + mount: secret + type: kv-v2 + path: stalwart + refreshAfter: 30s + destination: + name: stalwart-app-secrets + create: true + overwrite: true + transformation: + excludeRaw: true + templates: + admin-password: + text: "{{ index .Secrets \"admin-password\" }}" + dkim-private-key: + text: "{{ index .Secrets \"dkim-private-key\" }}" +--- +# SeaweedFS S3 credentials (shared — same secret as lasuite namespace). +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: seaweedfs-s3-credentials + namespace: stalwart +spec: + vaultAuthRef: vso-auth + mount: secret + type: kv-v2 + path: seaweedfs + refreshAfter: 30s + destination: + name: seaweedfs-s3-credentials + create: true + overwrite: true + transformation: + excludeRaw: true + templates: + S3_ACCESS_KEY: + text: "{{ index .Secrets \"access-key\" }}" + S3_SECRET_KEY: + text: "{{ index .Secrets \"secret-key\" }}" diff --git a/base/storage/seaweedfs-filer.yaml b/base/storage/seaweedfs-filer.yaml index 8bf7af9..c72e62a 100644 --- a/base/storage/seaweedfs-filer.yaml +++ b/base/storage/seaweedfs-filer.yaml @@ -25,7 +25,7 @@ spec: mountPath: /data/filer containers: - name: filer - image: chrislusf/seaweedfs:latest + image: chrislusf/seaweedfs:4.18 args: - filer - -port=8888 diff --git a/base/storage/seaweedfs-master.yaml b/base/storage/seaweedfs-master.yaml index 6caeeec..f1a9cf7 100644 --- a/base/storage/seaweedfs-master.yaml +++ b/base/storage/seaweedfs-master.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: master - image: chrislusf/seaweedfs:latest + image: chrislusf/seaweedfs:4.18 args: - master - -port=9333 diff --git a/overlays/local/kustomization.yaml b/overlays/local/kustomization.yaml index 6ba3bdf..0283e27 100644 --- a/overlays/local/kustomization.yaml +++ b/overlays/local/kustomization.yaml @@ -20,6 +20,7 @@ resources: - ../../base/lasuite - ../../base/media - ../../base/devtools + - ../../base/stalwart - ../../base/vso images: diff --git a/overlays/production/kustomization.yaml b/overlays/production/kustomization.yaml index 67a07f8..b8d7193 100644 --- a/overlays/production/kustomization.yaml +++ b/overlays/production/kustomization.yaml @@ -17,6 +17,7 @@ resources: - ../../base/lasuite - ../../base/media - ../../base/devtools + - ../../base/stalwart - ../../base/vso - ../../base/monitoring - ../../base/matrix @@ -107,12 +108,21 @@ patches: # SeaweedFS volume: expand PVC to 600 Gi - path: patch-seaweedfs-volume-size.yaml - # MTA-in: bind port 25 to the host for inbound email delivery + # Stalwart: bind SMTP (25), submission (587), and IMAPS (993) to the host - patch: | - op: add path: /spec/template/spec/containers/0/ports/0/hostPort value: 25 + - op: add + path: /spec/template/spec/containers/0/ports/1/hostPort + value: 587 + - op: add + path: /spec/template/spec/containers/0/ports/2/hostPort + value: 465 + - op: add + path: /spec/template/spec/containers/0/ports/4/hostPort + value: 993 target: kind: Deployment - name: messages-mta-in - namespace: lasuite + name: stalwart + namespace: stalwart diff --git a/scripts/export-mbox.py b/scripts/export-mbox.py new file mode 100644 index 0000000..fab10fd --- /dev/null +++ b/scripts/export-mbox.py @@ -0,0 +1,54 @@ +import os, zlib, gzip, email.utils, time +import psycopg2 + +DB_HOST = "postgres-rw.data.svc.cluster.local" +conn = psycopg2.connect( + host=DB_HOST, port=5432, dbname="messages_db", + user="messages", password=os.environ["DB_PASSWORD"] +) +cur = conn.cursor() + +cur.execute(""" + SELECT DISTINCT mb.local_part || '@' || d.name as mailbox_email, + b.raw_content, b.compression, m.sent_at + FROM messages_message m + JOIN messages_blob b ON m.blob_id = b.id + JOIN messages_thread t ON m.thread_id = t.id + JOIN messages_threadaccess ta ON ta.thread_id = t.id + JOIN messages_mailbox mb ON ta.mailbox_id = mb.id + JOIN messages_maildomain d ON mb.domain_id = d.id + ORDER BY mailbox_email, m.sent_at +""") + +os.makedirs("/tmp/mbox", exist_ok=True) +counts = {} +for row in cur.fetchall(): + mailbox_email, raw_content, compression, sent_at = row + raw = bytes(raw_content) + if compression == 1: + try: + eml = gzip.decompress(raw) + except Exception: + try: + eml = zlib.decompress(raw, -zlib.MAX_WBITS) + except Exception: + eml = raw + else: + eml = raw + + date_str = email.utils.formatdate(time.mktime(sent_at.timetuple())) if sent_at else email.utils.formatdate() + mbox_path = f"/tmp/mbox/{mailbox_email}.mbox" + with open(mbox_path, "ab") as f: + f.write(f"From {mailbox_email} {date_str}\n".encode()) + for line in eml.split(b"\n"): + if line.startswith(b"From "): + f.write(b">" + line + b"\n") + else: + f.write(line + b"\n") + f.write(b"\n") + counts[mailbox_email] = counts.get(mailbox_email, 0) + 1 + +conn.close() +for addr, count in counts.items(): + print(f"{addr}: {count} messages") +print("Export complete.") diff --git a/scripts/migrate-calendars.sh b/scripts/migrate-calendars.sh new file mode 100755 index 0000000..00c89ab --- /dev/null +++ b/scripts/migrate-calendars.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Migrate calendars from La Suite Calendars (CalDAV) to Stalwart (CalDAV). +# +# Exports all calendars per user from the old CalDAV endpoint and imports them +# into Stalwart's CalDAV endpoint. +# +# Prerequisites: +# - curl +# - Port-forward to both old calendars-backend and new Stalwart: +# kubectl port-forward -n lasuite svc/calendars-backend 8081:80 & +# kubectl port-forward -n stalwart svc/stalwart 8080:8080 & +# +# Usage: +# ./migrate-calendars.sh \ +# --old-url http://127.0.0.1:8081 \ +# --new-url http://127.0.0.1:8080 \ +# --new-user admin --new-password \ +# --users "sienna@sunbeam.pt,amber@sunbeam.pt,lonni@sunbeam.pt" + +set -euo pipefail + +EXPORT_DIR="$(mktemp -d)/calendar-export" +mkdir -p "$EXPORT_DIR" + +OLD_URL="" +NEW_URL="" +NEW_USER="" +NEW_PASS="" +USERS="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --old-url) OLD_URL="$2"; shift 2 ;; + --new-url) NEW_URL="$2"; shift 2 ;; + --new-user) NEW_USER="$2"; shift 2 ;; + --new-password) NEW_PASS="$2"; shift 2 ;; + --users) USERS="$2"; shift 2 ;; + *) echo "Unknown arg: $1"; exit 1 ;; + esac +done + +if [[ -z "$OLD_URL" || -z "$NEW_URL" || -z "$NEW_USER" || -z "$NEW_PASS" || -z "$USERS" ]]; then + echo "Usage: $0 --old-url --new-url --new-user --new-password --users " + exit 1 +fi + +IFS=',' read -ra USER_LIST <<< "$USERS" + +echo "==> Exporting calendars from La Suite..." + +for user in "${USER_LIST[@]}"; do + user_dir="$EXPORT_DIR/$user" + mkdir -p "$user_dir" + + echo " Exporting calendars for $user..." + + # NOTE: The exact CalDAV path depends on La Suite's CalDAV implementation. + # La Suite Calendars uses /caldav// or similar. Adjust as needed. + # This PROPFIND discovers all calendars for the user. + curl -s -X PROPFIND \ + -H "Content-Type: application/xml" \ + -d ' + + + + + + ' \ + "$OLD_URL/caldav/$user/" \ + -o "$user_dir/calendars.xml" 2>/dev/null || true + + # Export each calendar as .ics via CalDAV REPORT + # NOTE: This is a simplified template. The actual export depends on + # the La Suite CalDAV response format. You may need to parse the + # PROPFIND response to discover calendar URLs, then issue + # calendar-multiget or calendar-query REPORT requests. + # + # A simpler alternative: if La Suite exposes /ical/ export endpoints + # (seen in pingora config: /ical/ → calendars-backend), use those: + # + # curl -s "$OLD_URL/ical/$user/calendar.ics" -o "$user_dir/calendar.ics" + # + echo " Exported to $user_dir/" +done + +echo "" +echo "==> Importing calendars into Stalwart..." + +for user in "${USER_LIST[@]}"; do + user_dir="$EXPORT_DIR/$user" + + for ics_file in "$user_dir"/*.ics; do + [[ -f "$ics_file" ]] || continue + cal_name=$(basename "$ics_file" .ics) + + echo " Importing $cal_name for $user..." + + # Upload .ics to Stalwart CalDAV. + # Stalwart CalDAV path: /dav/calendars/user/// + curl -s -X PUT \ + -u "$NEW_USER:$NEW_PASS" \ + -H "Content-Type: text/calendar" \ + --data-binary "@$ics_file" \ + "$NEW_URL/dav/calendars/user/$user/$cal_name.ics" || { + echo " ⚠️ Failed to import $cal_name for $user" + } + done +done + +echo "" +echo "==> Migration complete. Exported data is in: $EXPORT_DIR" +echo "" +echo "Verify by comparing event counts:" +echo " Old: curl -s '$OLD_URL/caldav//' -X PROPFIND | grep -c VEVENT" +echo " New: curl -s -u admin:pw '$NEW_URL/dav/calendars/user//' -X PROPFIND | grep -c VEVENT" diff --git a/scripts/migrate-mbox-job.yaml b/scripts/migrate-mbox-job.yaml new file mode 100644 index 0000000..085e06e --- /dev/null +++ b/scripts/migrate-mbox-job.yaml @@ -0,0 +1,57 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-mbox + namespace: stalwart +spec: + template: + spec: + restartPolicy: Never + initContainers: + - name: export + image: python:3.12-slim + command: ["/bin/sh", "-c", "pip -q install psycopg2-binary && python3 /scripts/export-mbox.py && ls -la /tmp/mbox/"] + env: + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: messages-db-credentials + key: password + volumeMounts: + - name: script + mountPath: /scripts + - name: mbox + mountPath: /tmp/mbox + containers: + - name: import + image: stalwartlabs/stalwart:v0.15.5 + command: + - /bin/sh + - -c + - | + URL="http://stalwart.stalwart.svc.cluster.local:8080" + CREDS="admin:${ADMIN_PASSWORD}" + + for mbox in /tmp/mbox/*.mbox; do + ACCOUNT=$(basename "$mbox" .mbox) + echo "=== Importing $mbox into $ACCOUNT ===" + stalwart-cli -u "$URL" -c "$CREDS" import messages -f mbox "$ACCOUNT" "$mbox" + echo "" + done + + echo "All imports complete." + env: + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-app-secrets + key: admin-password + volumeMounts: + - name: mbox + mountPath: /tmp/mbox + volumes: + - name: script + configMap: + name: export-mbox-script + - name: mbox + emptyDir: {} diff --git a/scripts/migrate-messages-job.yaml b/scripts/migrate-messages-job.yaml new file mode 100644 index 0000000..9b0a623 --- /dev/null +++ b/scripts/migrate-messages-job.yaml @@ -0,0 +1,261 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate-messages + namespace: stalwart +spec: + template: + spec: + restartPolicy: Never + containers: + - name: migrate + image: python:3.12-slim + command: + - /bin/sh + - -c + - | + pip install psycopg2-binary && python3 /scripts/migrate.py + env: + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: stalwart-app-secrets + key: admin-password + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: messages-db-credentials + key: password + volumeMounts: + - name: script + mountPath: /scripts + volumes: + - name: script + configMap: + name: migrate-messages-script +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: migrate-messages-script + namespace: stalwart +data: + migrate.py: | + import json, os, zlib, urllib.request, urllib.error, base64, sys + import psycopg2 + + DB_HOST = "postgres-rw.data.svc.cluster.local" + DB_PORT = 5432 + DB_NAME = "messages_db" + DB_USER = "messages" + DB_PASS = os.environ.get("DB_PASSWORD", "") + JMAP_URL = "http://stalwart.stalwart.svc.cluster.local:8080" + ADMIN_USER = "admin" + + ADMIN_PASS = os.environ["ADMIN_PASSWORD"] + + auth_header = "Basic " + base64.b64encode(f"{ADMIN_USER}:{ADMIN_PASS}".encode()).decode() + + def jmap_call(method_calls): + body = json.dumps({ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": method_calls, + }).encode() + req = urllib.request.Request(f"{JMAP_URL}/jmap", data=body, + headers={"Authorization": auth_header, "Content-Type": "application/json"}) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + def upload_blob(account_id, eml_bytes): + req = urllib.request.Request( + f"{JMAP_URL}/jmap/upload/{account_id}/", + data=eml_bytes, + headers={"Authorization": auth_header, "Content-Type": "message/rfc822"}) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read())["blobId"] + + def ensure_user(email, full_name): + body = json.dumps({ + "type": "individual", "name": email, + "description": full_name or email, "emails": [email], + "quota": 0, "secrets": [], "urls": [], "memberOf": [], + "roles": ["user"], "lists": [], "members": [], + "enabledPermissions": [], "disabledPermissions": [], "externalMembers": [], + }).encode() + req = urllib.request.Request(f"{JMAP_URL}/api/principal", + data=body, method="POST", + headers={"Authorization": auth_header, "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req) as resp: + result = json.loads(resp.read()) + print(f" Created user {email} (id={result.get('data')})") + except urllib.error.HTTPError as e: + body_text = e.read().decode() + if "fieldAlreadyExists" in body_text: + print(f" User {email} already exists") + else: + print(f" Error creating user {email}: {e.code} {body_text}") + + def get_account_id_for_user(email): + """Get JMAP account ID by authenticating as the user (admin impersonation).""" + # Stalwart allows admin to access any account via the master user mechanism: + # authenticate as "user%admin" with admin password + impersonate_auth = "Basic " + base64.b64encode( + f"{email}%{ADMIN_USER}:{ADMIN_PASS}".encode()).decode() + req = urllib.request.Request(f"{JMAP_URL}/.well-known/jmap", + headers={"Authorization": impersonate_auth}) + try: + with urllib.request.urlopen(req) as resp: + session = json.loads(resp.read()) + return next(iter(session.get("accounts", {})), None), impersonate_auth + except urllib.error.HTTPError: + # Fallback: try direct admin auth + req = urllib.request.Request(f"{JMAP_URL}/.well-known/jmap", + headers={"Authorization": auth_header}) + with urllib.request.urlopen(req) as resp: + session = json.loads(resp.read()) + return next(iter(session.get("accounts", {})), None), auth_header + + def get_inbox_id(account_id, user_auth): + body = json.dumps({ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": [["Mailbox/get", {"accountId": account_id}, "0"]], + }).encode() + req = urllib.request.Request(f"{JMAP_URL}/jmap", data=body, + headers={"Authorization": user_auth, "Content-Type": "application/json"}) + with urllib.request.urlopen(req) as resp: + result = json.loads(resp.read()) + mailboxes = result["methodResponses"][0][1]["list"] + for mb in mailboxes: + if mb.get("role") == "inbox" or mb.get("name", "").lower() == "inbox": + return mb["id"] + return mailboxes[0]["id"] if mailboxes else None + + def upload_blob_as(account_id, eml_bytes, user_auth): + req = urllib.request.Request( + f"{JMAP_URL}/jmap/upload/{account_id}/", + data=eml_bytes, + headers={"Authorization": user_auth, "Content-Type": "message/rfc822"}) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read())["blobId"] + + def jmap_call_as(method_calls, user_auth): + body = json.dumps({ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": method_calls, + }).encode() + req = urllib.request.Request(f"{JMAP_URL}/jmap", data=body, + headers={"Authorization": user_auth, "Content-Type": "application/json"}) + with urllib.request.urlopen(req) as resp: + return json.loads(resp.read()) + + # Connect to messages_db + print("Connecting to messages_db...") + conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS) + cur = conn.cursor() + + cur.execute(""" + SELECT DISTINCT m.id, m.subject, m.sent_at, m.is_draft, m.is_starred, + m.is_trashed, m.is_spam, + mb.local_part || '@' || d.name as mailbox_email, + b.raw_content, b.compression + FROM messages_message m + JOIN messages_blob b ON m.blob_id = b.id + JOIN messages_thread t ON m.thread_id = t.id + JOIN messages_threadaccess ta ON ta.thread_id = t.id + JOIN messages_mailbox mb ON ta.mailbox_id = mb.id + JOIN messages_maildomain d ON mb.domain_id = d.id + ORDER BY mailbox_email, m.sent_at + """) + messages = cur.fetchall() + print(f"Found {len(messages)} messages to migrate.") + + cur.execute("SELECT email, full_name FROM messages_user") + users = {row[0]: row[1] for row in cur.fetchall()} + conn.close() + + # Ensure all mailbox users exist in Stalwart + seen_emails = set() + for msg in messages: + email = msg[7] + if email not in seen_emails: + seen_emails.add(email) + ensure_user(email, users.get(email, email)) + + # Resolve per-user JMAP accounts + user_contexts = {} + for email in seen_emails: + print(f"Resolving JMAP account for {email}...") + account_id, user_auth = get_account_id_for_user(email) + if account_id: + inbox_id = get_inbox_id(account_id, user_auth) + user_contexts[email] = (account_id, inbox_id, user_auth) + print(f" {email}: account={account_id}, inbox={inbox_id}") + else: + print(f" WARNING: Could not get account for {email}, will skip") + + # Import messages into each user's account + imported = 0 + errors = 0 + for msg in messages: + msg_id, subject, sent_at, is_draft, is_starred, is_trashed, is_spam, email, raw_content, compression = msg + + if email not in user_contexts: + errors += 1 + print(f" SKIP: {email}: {subject} (no account)") + continue + + account_id, inbox_id, user_auth = user_contexts[email] + + try: + raw = bytes(raw_content) + if compression == 1: + import gzip + try: + eml_bytes = gzip.decompress(raw) + except Exception: + try: + eml_bytes = zlib.decompress(raw, -zlib.MAX_WBITS) + except Exception: + try: + eml_bytes = zlib.decompress(raw) + except Exception: + eml_bytes = raw + else: + eml_bytes = raw + + blob_id = upload_blob_as(account_id, eml_bytes, user_auth) + + keywords = {"$seen": True} + if is_starred: + keywords["$flagged"] = True + if is_draft: + keywords["$draft"] = True + + received_at = sent_at.isoformat() if sent_at else None + import_data = { + "accountId": account_id, + "emails": { + "imp1": { + "blobId": blob_id, + "mailboxIds": {inbox_id: True}, + "keywords": keywords, + } + } + } + if received_at: + import_data["emails"]["imp1"]["receivedAt"] = received_at + + result = jmap_call_as([["Email/import", import_data, "0"]], user_auth) + resp = result["methodResponses"][0][1] + if "created" in resp and "imp1" in resp["created"]: + imported += 1 + print(f" [{imported}] {email}: {subject or '(no subject)'}") + else: + errors += 1 + print(f" ERROR: {email}: {subject}: {resp.get('notCreated', {})}") + except Exception as e: + errors += 1 + print(f" ERROR: {email}: {subject}: {e}") + + print(f"\nMigration complete: {imported} imported, {errors} errors, {len(messages)} total") diff --git a/scripts/migrate-messages.py b/scripts/migrate-messages.py new file mode 100755 index 0000000..3729ae3 --- /dev/null +++ b/scripts/migrate-messages.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +""" +Migrate emails from La Suite Messages (PostgreSQL + SeaweedFS) to Stalwart (JMAP). + +Reads message metadata from messages_db, downloads RFC 5322 bodies from SeaweedFS, +and uploads each message to Stalwart via JMAP Email/import (RFC 8621 §5.4). + +Usage: + # Port-forward to the required services first: + kubectl port-forward -n data svc/postgres-rw 5432:5432 & + kubectl port-forward -n stalwart svc/stalwart 8080:8080 & + + # Run the migration: + python3 migrate-messages.py \ + --db-host 127.0.0.1 --db-port 5432 --db-name messages_db --db-user messages --db-password \ + --s3-endpoint http://127.0.0.1:8333 --s3-bucket sunbeam-messages \ + --s3-access-key --s3-secret-key \ + --jmap-url http://127.0.0.1:8080 --jmap-user admin --jmap-password + +Prerequisites: + pip install psycopg2-binary boto3 requests + +The script is idempotent: it tracks progress in a checkpoint file and skips +already-imported messages on re-run. +""" + +import argparse +import json +import hashlib +import os +import sys +from pathlib import Path + +try: + import psycopg2 + import boto3 + import requests +except ImportError: + print("Missing dependencies. Install with: pip install psycopg2-binary boto3 requests") + sys.exit(1) + + +CHECKPOINT_FILE = Path("migrate-messages-checkpoint.json") + + +def load_checkpoint(): + if CHECKPOINT_FILE.exists(): + return json.loads(CHECKPOINT_FILE.read_text()) + return {"imported": {}} + + +def save_checkpoint(checkpoint): + CHECKPOINT_FILE.write_text(json.dumps(checkpoint, indent=2)) + + +def get_jmap_session(jmap_url, user, password): + """Get JMAP session and extract accountId.""" + resp = requests.get( + f"{jmap_url}/.well-known/jmap", + auth=(user, password), + ) + resp.raise_for_status() + session = resp.json() + # Use the primary account + primary_accounts = session.get("primaryAccounts", {}) + account_id = primary_accounts.get("urn:ietf:params:jmap:mail") + if not account_id: + # Fallback: first account + accounts = session.get("accounts", {}) + account_id = next(iter(accounts)) + return session, account_id + + +def jmap_get_mailboxes(jmap_url, account_id, user, password): + """Fetch all mailboxes for the account.""" + resp = requests.post( + f"{jmap_url}/jmap", + auth=(user, password), + json={ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": [ + ["Mailbox/get", {"accountId": account_id}, "0"] + ], + }, + ) + resp.raise_for_status() + result = resp.json() + mailboxes = result["methodResponses"][0][1]["list"] + return {mb["name"]: mb["id"] for mb in mailboxes} + + +def jmap_create_mailbox(jmap_url, account_id, user, password, name): + """Create a mailbox and return its ID.""" + resp = requests.post( + f"{jmap_url}/jmap", + auth=(user, password), + json={ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": [ + ["Mailbox/set", { + "accountId": account_id, + "create": {"mb1": {"name": name}}, + }, "0"] + ], + }, + ) + resp.raise_for_status() + result = resp.json() + created = result["methodResponses"][0][1].get("created", {}) + if "mb1" in created: + return created["mb1"]["id"] + # Already exists — fetch it + mailboxes = jmap_get_mailboxes(jmap_url, account_id, user, password) + return mailboxes.get(name) + + +def jmap_import_email(jmap_url, account_id, user, password, eml_bytes, mailbox_id, keywords, received_at): + """Import a single RFC 5322 message via JMAP Email/import.""" + # First, upload the blob + resp = requests.post( + f"{jmap_url}/jmap/upload/{account_id}/", + auth=(user, password), + headers={"Content-Type": "message/rfc822"}, + data=eml_bytes, + ) + resp.raise_for_status() + blob = resp.json() + blob_id = blob["blobId"] + + # Then import it + import_data = { + "accountId": account_id, + "emails": { + "imp1": { + "blobId": blob_id, + "mailboxIds": {mailbox_id: True}, + "keywords": keywords, + } + }, + } + if received_at: + import_data["emails"]["imp1"]["receivedAt"] = received_at + + resp = requests.post( + f"{jmap_url}/jmap", + auth=(user, password), + json={ + "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], + "methodCalls": [ + ["Email/import", import_data, "0"] + ], + }, + ) + resp.raise_for_status() + result = resp.json() + created = result["methodResponses"][0][1].get("created", {}) + if "imp1" in created: + return created["imp1"]["id"] + not_created = result["methodResponses"][0][1].get("notCreated", {}) + if "imp1" in not_created: + err = not_created["imp1"] + raise RuntimeError(f"JMAP import failed: {err}") + return None + + +def main(): + parser = argparse.ArgumentParser(description="Migrate La Suite Messages → Stalwart JMAP") + parser.add_argument("--db-host", default="127.0.0.1") + parser.add_argument("--db-port", type=int, default=5432) + parser.add_argument("--db-name", default="messages_db") + parser.add_argument("--db-user", default="messages") + parser.add_argument("--db-password", required=True) + parser.add_argument("--s3-endpoint", default="http://127.0.0.1:8333") + parser.add_argument("--s3-bucket", default="sunbeam-messages") + parser.add_argument("--s3-access-key", required=True) + parser.add_argument("--s3-secret-key", required=True) + parser.add_argument("--jmap-url", default="http://127.0.0.1:8080") + parser.add_argument("--jmap-user", default="admin") + parser.add_argument("--jmap-password", required=True) + parser.add_argument("--dry-run", action="store_true", help="Count messages without importing") + args = parser.parse_args() + + checkpoint = load_checkpoint() + + # Connect to messages_db + print("Connecting to messages_db...") + conn = psycopg2.connect( + host=args.db_host, + port=args.db_port, + dbname=args.db_name, + user=args.db_user, + password=args.db_password, + ) + + # Connect to SeaweedFS + print("Connecting to SeaweedFS...") + s3 = boto3.client( + "s3", + endpoint_url=args.s3_endpoint, + aws_access_key_id=args.s3_access_key, + aws_secret_access_key=args.s3_secret_key, + region_name="us-east-1", + ) + + # Get JMAP session + if not args.dry_run: + print("Connecting to Stalwart JMAP...") + session, account_id = get_jmap_session(args.jmap_url, args.jmap_user, args.jmap_password) + print(f" Account: {account_id}") + mailboxes = jmap_get_mailboxes(args.jmap_url, account_id, args.jmap_user, args.jmap_password) + print(f" Mailboxes: {list(mailboxes.keys())}") + + # Query all messages from La Suite + # NOTE: The actual table/column names depend on La Suite Messages' Django models. + # You may need to adjust these queries after inspecting the actual schema. + # Run `\dt` and `\d ` in psql against messages_db to find the real names. + print("\nQuerying messages from La Suite database...") + cur = conn.cursor() + + # List all tables to help identify the right ones + cur.execute(""" + SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' ORDER BY table_name; + """) + tables = [row[0] for row in cur.fetchall()] + print(f" Tables: {tables}") + + if args.dry_run: + # Just count messages per table that looks like it contains emails + for table in tables: + cur.execute(f"SELECT COUNT(*) FROM {table};") + count = cur.fetchone()[0] + if count > 0: + print(f" {table}: {count} rows") + print("\nDry run complete. Inspect the tables above and adjust the migration") + print("queries in this script to match the actual La Suite Messages schema.") + conn.close() + return + + # TODO: Replace with actual queries once schema is inspected. + # The migration logic below is a template — run with --dry-run first + # to see the actual table structure, then update these queries. + # + # Expected flow: + # 1. Query user accounts + # 2. For each user, query their mailboxes/folders + # 3. For each mailbox, query messages (S3 key, flags, received date) + # 4. Download .eml from S3 + # 5. Upload to Stalwart via JMAP Email/import + # + # Example (adjust table/column names): + # + # cur.execute("SELECT id, email FROM auth_user;") + # for user_id, email in cur.fetchall(): + # cur.execute("SELECT id, name FROM mailbox WHERE user_id = %s;", (user_id,)) + # for mb_id, mb_name in cur.fetchall(): + # mailbox_jmap_id = ensure_mailbox(mb_name) + # cur.execute("SELECT s3_key, is_read, received_at FROM message WHERE mailbox_id = %s;", (mb_id,)) + # for s3_key, is_read, received_at in cur.fetchall(): + # if s3_key in checkpoint["imported"]: + # continue + # eml = s3.get_object(Bucket=args.s3_bucket, Key=s3_key)["Body"].read() + # keywords = {"$seen": True} if is_read else {} + # jmap_import_email(..., eml, mailbox_jmap_id, keywords, received_at) + # checkpoint["imported"][s3_key] = True + # save_checkpoint(checkpoint) + + print("\n⚠️ Schema inspection required!") + print("Run with --dry-run first, then update the TODO section in this script") + print("with the correct table and column names from the La Suite Messages schema.") + + conn.close() + print("\nDone.") + + +if __name__ == "__main__": + main()