checkpoint: stalwart deploy, beam-design, migration scripts, config tweaks
Stalwart + Bulwark mail server deployment with OIDC, TLS cert, vault secrets. Beam design service. Pingora config cleanup. SeaweedFS replication fix. Kratos values tweak. Migration scripts for mbox/messages /calendars from La Suite to Stalwart.
This commit is contained in:
49
base/devtools/beam-design.yaml
Normal file
49
base/devtools/beam-design.yaml
Normal file
@@ -0,0 +1,49 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: beam-design
|
||||
namespace: devtools
|
||||
labels:
|
||||
app: beam-design
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: beam-design
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: beam-design
|
||||
spec:
|
||||
containers:
|
||||
- name: beam-design
|
||||
image: src.sunbeam.pt/studio/beam-ui:latest
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
protocol: TCP
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
securityContext:
|
||||
runAsUser: 65534
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: beam-design
|
||||
namespace: devtools
|
||||
labels:
|
||||
app: beam-design
|
||||
spec:
|
||||
selector:
|
||||
app: beam-design
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
@@ -167,19 +167,7 @@ data:
|
||||
prefix = "/.well-known/"
|
||||
backend = "http://stalwart.stalwart.svc.cluster.local:8080"
|
||||
|
||||
# Stalwart OAuth2 endpoints (/authorize/code, /auth/token, /auth/device)
|
||||
[[routes.paths]]
|
||||
prefix = "/authorize"
|
||||
backend = "http://stalwart.stalwart.svc.cluster.local:8080"
|
||||
|
||||
[[routes.paths]]
|
||||
prefix = "/auth/"
|
||||
backend = "http://stalwart.stalwart.svc.cluster.local:8080"
|
||||
|
||||
# Stalwart login page (used during OAuth flow)
|
||||
[[routes.paths]]
|
||||
prefix = "/login"
|
||||
backend = "http://stalwart.stalwart.svc.cluster.local:8080"
|
||||
|
||||
[[routes]]
|
||||
host_prefix = "messages"
|
||||
@@ -401,20 +389,8 @@ data:
|
||||
host_prefix = "build"
|
||||
backend = "buildkitd.build.svc.cluster.local:1234"
|
||||
|
||||
# SMTP inbound: port 25 → Stalwart for mail delivery.
|
||||
[smtp]
|
||||
listen = "0.0.0.0:25"
|
||||
backend = "stalwart.stalwart.svc.cluster.local:25"
|
||||
|
||||
# SMTP submission: port 587 → Stalwart for authenticated sending.
|
||||
[smtp-submission]
|
||||
listen = "0.0.0.0:587"
|
||||
backend = "stalwart.stalwart.svc.cluster.local:587"
|
||||
|
||||
# IMAPS: port 993 → Stalwart for desktop/mobile email clients.
|
||||
[imaps]
|
||||
listen = "0.0.0.0:993"
|
||||
backend = "stalwart.stalwart.svc.cluster.local:993"
|
||||
# SMTP/IMAP ports are exposed directly on the Stalwart pod via hostPort
|
||||
# (see overlays/production/kustomization.yaml), not through Pingora.
|
||||
|
||||
# SSH TCP passthrough: port 22 → Gitea SSH pod (headless service → pod:2222).
|
||||
[ssh]
|
||||
|
||||
@@ -102,7 +102,7 @@ kratos:
|
||||
|
||||
courier:
|
||||
smtp:
|
||||
connection_uri: "smtp://postfix.lasuite.svc.cluster.local:25/?skip_ssl_verify=true"
|
||||
connection_uri: "smtp://stalwart.stalwart.svc.cluster.local:25/?skip_ssl_verify=true"
|
||||
from_address: no-reply@DOMAIN_SUFFIX
|
||||
from_name: Sunbeam Studios
|
||||
|
||||
|
||||
70
base/stalwart/bulwark-deployment.yaml
Normal file
70
base/stalwart/bulwark-deployment.yaml
Normal file
@@ -0,0 +1,70 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: bulwark
|
||||
namespace: stalwart
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: bulwark
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: bulwark
|
||||
spec:
|
||||
containers:
|
||||
- name: bulwark
|
||||
image: src.DOMAIN_SUFFIX/studio/bulwark:latest
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 3000
|
||||
env:
|
||||
- name: JMAP_SERVER_URL
|
||||
value: https://mail.DOMAIN_SUFFIX
|
||||
- name: OAUTH_ENABLED
|
||||
value: "true"
|
||||
- name: OAUTH_ONLY
|
||||
value: "true"
|
||||
- name: LOG_LEVEL
|
||||
value: "debug"
|
||||
- name: OAUTH_SCOPES
|
||||
value: "openid email profile offline_access"
|
||||
- name: COOKIE_SECURE
|
||||
value: "false"
|
||||
- name: OAUTH_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: oidc-bulwark
|
||||
key: CLIENT_ID
|
||||
- name: OAUTH_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: oidc-bulwark
|
||||
key: CLIENT_SECRET
|
||||
- name: OAUTH_ISSUER_URL
|
||||
value: https://auth.DOMAIN_SUFFIX
|
||||
- name: SESSION_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-app-secrets
|
||||
key: admin-password
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 3000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 3000
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
memory: 128Mi
|
||||
cpu: 50m
|
||||
limits:
|
||||
memory: 512Mi
|
||||
cpu: 500m
|
||||
12
base/stalwart/bulwark-service.yaml
Normal file
12
base/stalwart/bulwark-service.yaml
Normal file
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: bulwark
|
||||
namespace: stalwart
|
||||
spec:
|
||||
selector:
|
||||
app: bulwark
|
||||
ports:
|
||||
- name: http
|
||||
port: 80
|
||||
targetPort: 3000
|
||||
13
base/stalwart/certificate.yaml
Normal file
13
base/stalwart/certificate.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: stalwart-tls
|
||||
namespace: stalwart
|
||||
spec:
|
||||
secretName: stalwart-tls
|
||||
issuerRef:
|
||||
name: letsencrypt-production
|
||||
kind: ClusterIssuer
|
||||
dnsNames:
|
||||
- "mail.DOMAIN_SUFFIX"
|
||||
- "cal.DOMAIN_SUFFIX"
|
||||
6
base/stalwart/namespace.yaml
Normal file
6
base/stalwart/namespace.yaml
Normal file
@@ -0,0 +1,6 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: stalwart
|
||||
labels:
|
||||
linkerd.io/inject: enabled
|
||||
30
base/stalwart/oidc-client-bulwark.yaml
Normal file
30
base/stalwart/oidc-client-bulwark.yaml
Normal file
@@ -0,0 +1,30 @@
|
||||
# Bulwark webmail OIDC client — authenticates directly with Hydra.
|
||||
# Hydra Maester creates K8s Secret "oidc-bulwark" with CLIENT_ID/CLIENT_SECRET.
|
||||
# DOMAIN_SUFFIX is replaced by sed at deploy time.
|
||||
apiVersion: hydra.ory.sh/v1alpha1
|
||||
kind: OAuth2Client
|
||||
metadata:
|
||||
name: bulwark
|
||||
namespace: stalwart
|
||||
spec:
|
||||
clientName: Webmail
|
||||
grantTypes:
|
||||
- authorization_code
|
||||
- refresh_token
|
||||
responseTypes:
|
||||
- code
|
||||
scope: openid email profile offline_access
|
||||
redirectUris:
|
||||
- https://mail.DOMAIN_SUFFIX/en/auth/callback
|
||||
- https://mail.DOMAIN_SUFFIX/auth/callback
|
||||
- https://mail.DOMAIN_SUFFIX/api/auth/callback
|
||||
postLogoutRedirectUris:
|
||||
- https://mail.DOMAIN_SUFFIX
|
||||
tokenEndpointAuthMethod: client_secret_post
|
||||
secretName: oidc-bulwark
|
||||
skipConsent: true
|
||||
tokenLifespans:
|
||||
authorization_code_grant_access_token_lifespan: 24h
|
||||
authorization_code_grant_refresh_token_lifespan: 720h
|
||||
refresh_token_grant_access_token_lifespan: 24h
|
||||
refresh_token_grant_refresh_token_lifespan: 720h
|
||||
24
base/stalwart/oidc-client.yaml
Normal file
24
base/stalwart/oidc-client.yaml
Normal file
@@ -0,0 +1,24 @@
|
||||
# Stalwart OIDC client — registered with Hydra for SSO login.
|
||||
# Hydra Maester creates K8s Secret "oidc-stalwart" in the stalwart namespace
|
||||
# with CLIENT_ID and CLIENT_SECRET keys.
|
||||
# DOMAIN_SUFFIX is replaced by sed at deploy time.
|
||||
apiVersion: hydra.ory.sh/v1alpha1
|
||||
kind: OAuth2Client
|
||||
metadata:
|
||||
name: stalwart
|
||||
namespace: stalwart
|
||||
spec:
|
||||
clientName: Mail
|
||||
grantTypes:
|
||||
- authorization_code
|
||||
- refresh_token
|
||||
responseTypes:
|
||||
- code
|
||||
scope: openid email profile
|
||||
redirectUris:
|
||||
- https://mail.DOMAIN_SUFFIX/authorize/code
|
||||
postLogoutRedirectUris:
|
||||
- https://mail.DOMAIN_SUFFIX
|
||||
tokenEndpointAuthMethod: client_secret_post
|
||||
secretName: oidc-stalwart
|
||||
skipConsent: true
|
||||
148
base/stalwart/stalwart-config.yaml
Normal file
148
base/stalwart/stalwart-config.yaml
Normal file
@@ -0,0 +1,148 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: stalwart-config
|
||||
namespace: stalwart
|
||||
data:
|
||||
# DOMAIN_SUFFIX is replaced by sed at deploy time.
|
||||
# Secrets (%{env:VAR}%) are injected via environment variables in the Deployment.
|
||||
#
|
||||
# Only LOCAL keys belong in this file (store.*, storage.*, server.*, directory.*,
|
||||
# tracer.*, cluster.*, certificate.*, authentication.fallback-admin.*).
|
||||
# Everything else (OIDC, DKIM, spam, etc.) is configured via the Stalwart
|
||||
# web admin UI and stored in the database.
|
||||
config.toml: |
|
||||
# Force http.url to be read from this file (not the database).
|
||||
config.local-keys.0000 = "store.*"
|
||||
config.local-keys.0001 = "directory.*"
|
||||
config.local-keys.0002 = "tracer.*"
|
||||
config.local-keys.0003 = "server.*"
|
||||
config.local-keys.0004 = "!server.blocked-ip.*"
|
||||
config.local-keys.0005 = "!server.allowed-ip.*"
|
||||
config.local-keys.0006 = "authentication.fallback-admin.*"
|
||||
config.local-keys.0007 = "cluster.*"
|
||||
config.local-keys.0008 = "config.local-keys.*"
|
||||
config.local-keys.0009 = "storage.*"
|
||||
config.local-keys.0010 = "certificate.*"
|
||||
config.local-keys.0011 = "http.url"
|
||||
config.local-keys.0012 = "http.use-x-forwarded"
|
||||
|
||||
# Expression-quoted public URL (inner single quotes required).
|
||||
http.url = "'https://mail.DOMAIN_SUFFIX'"
|
||||
http.use-x-forwarded = true
|
||||
|
||||
[server]
|
||||
hostname = "mail.DOMAIN_SUFFIX"
|
||||
max-connections = 1024
|
||||
|
||||
[server.listener."smtp"]
|
||||
bind = ["0.0.0.0:25"]
|
||||
protocol = "smtp"
|
||||
|
||||
[server.listener."submission"]
|
||||
bind = ["0.0.0.0:587"]
|
||||
protocol = "smtp"
|
||||
tls.implicit = false
|
||||
|
||||
[server.listener."smtps"]
|
||||
bind = ["0.0.0.0:465"]
|
||||
protocol = "smtp"
|
||||
tls.implicit = true
|
||||
|
||||
[server.listener."imap"]
|
||||
bind = ["0.0.0.0:143"]
|
||||
protocol = "imap"
|
||||
tls.implicit = false
|
||||
|
||||
[server.listener."imaps"]
|
||||
bind = ["0.0.0.0:993"]
|
||||
protocol = "imap"
|
||||
tls.implicit = true
|
||||
|
||||
[server.listener."jmap"]
|
||||
bind = ["0.0.0.0:8080"]
|
||||
protocol = "http"
|
||||
|
||||
[server.listener."managesieve"]
|
||||
bind = ["0.0.0.0:4190"]
|
||||
protocol = "managesieve"
|
||||
|
||||
[certificate."default"]
|
||||
cert = "%{file:/etc/stalwart-tls/tls.crt}%"
|
||||
private-key = "%{file:/etc/stalwart-tls/tls.key}%"
|
||||
|
||||
# ── Storage backends ─────────────────────────────────────────────────────
|
||||
|
||||
[store."postgresql"]
|
||||
type = "postgresql"
|
||||
host = "postgres-rw.data.svc.cluster.local"
|
||||
port = 5432
|
||||
database = "stalwart_db"
|
||||
user = "stalwart"
|
||||
password = "%{env:DB_PASSWORD}%"
|
||||
timeout = "15s"
|
||||
|
||||
[store."postgresql".pool]
|
||||
max-connections = 20
|
||||
|
||||
[store."s3"]
|
||||
type = "s3"
|
||||
bucket = "sunbeam-stalwart"
|
||||
endpoint = "http://seaweedfs-filer.storage.svc.cluster.local:8333"
|
||||
region = "us-east-1"
|
||||
access-key = "%{env:S3_ACCESS_KEY}%"
|
||||
secret-key = "%{env:S3_SECRET_KEY}%"
|
||||
key-prefix = "v1/"
|
||||
timeout = "60s"
|
||||
|
||||
[store."opensearch"]
|
||||
type = "elasticsearch"
|
||||
url = "http://opensearch.data.svc.cluster.local:9200"
|
||||
index = "stalwart"
|
||||
|
||||
[store."redis"]
|
||||
type = "redis"
|
||||
urls = ["redis://valkey.data.svc.cluster.local:6379/7"]
|
||||
|
||||
# ── Storage role assignments ─────────────────────────────────────────────
|
||||
|
||||
[storage]
|
||||
data = "postgresql"
|
||||
blob = "postgresql"
|
||||
fts = "opensearch"
|
||||
lookup = "redis"
|
||||
directory = "hydra"
|
||||
|
||||
# ── Directories (user stores) ──────────────────────────────────────────
|
||||
|
||||
# Internal directory for locally-managed accounts (admin, service accounts).
|
||||
[directory."internal"]
|
||||
type = "internal"
|
||||
store = "postgresql"
|
||||
|
||||
# OIDC directory — validates Hydra-issued bearer tokens via userinfo.
|
||||
# When a JMAP client presents a Bearer token, Stalwart calls Hydra's
|
||||
# userinfo endpoint to map it to a user identity.
|
||||
[directory."hydra"]
|
||||
type = "oidc"
|
||||
timeout = "15s"
|
||||
endpoint.url = "http://hydra-public.ory.svc.cluster.local:4444/userinfo"
|
||||
endpoint.method = "userinfo"
|
||||
fields.email = "email"
|
||||
fields.username = "email"
|
||||
fields.full-name = "name"
|
||||
|
||||
# ── Authentication ───────────────────────────────────────────────────────
|
||||
|
||||
[authentication.fallback-admin]
|
||||
user = "admin"
|
||||
secret = "%{env:ADMIN_PASSWORD}%"
|
||||
|
||||
# ── Logging ──────────────────────────────────────────────────────────────
|
||||
|
||||
[tracer."stdout"]
|
||||
type = "stdout"
|
||||
level = "info"
|
||||
ansi = false
|
||||
enable = true
|
||||
|
||||
103
base/stalwart/stalwart-deployment.yaml
Normal file
103
base/stalwart/stalwart-deployment.yaml
Normal file
@@ -0,0 +1,103 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: stalwart
|
||||
namespace: stalwart
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: stalwart
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: stalwart
|
||||
spec:
|
||||
containers:
|
||||
- name: stalwart
|
||||
image: stalwartlabs/stalwart:v0.15.5
|
||||
ports:
|
||||
- name: smtp
|
||||
containerPort: 25
|
||||
- name: submission
|
||||
containerPort: 587
|
||||
- name: smtps
|
||||
containerPort: 465
|
||||
- name: imap
|
||||
containerPort: 143
|
||||
- name: imaps
|
||||
containerPort: 993
|
||||
- name: managesieve
|
||||
containerPort: 4190
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
- name: DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-db-credentials
|
||||
key: password
|
||||
- name: S3_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: seaweedfs-s3-credentials
|
||||
key: S3_ACCESS_KEY
|
||||
- name: S3_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: seaweedfs-s3-credentials
|
||||
key: S3_SECRET_KEY
|
||||
- name: ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-app-secrets
|
||||
key: admin-password
|
||||
- name: DKIM_PRIVATE_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-app-secrets
|
||||
key: dkim-private-key
|
||||
- name: OIDC_CLIENT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: oidc-stalwart
|
||||
key: CLIENT_ID
|
||||
- name: OIDC_CLIENT_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: oidc-stalwart
|
||||
key: CLIENT_SECRET
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /opt/stalwart/etc/config.toml
|
||||
subPath: config.toml
|
||||
readOnly: true
|
||||
- name: tls
|
||||
mountPath: /etc/stalwart-tls
|
||||
readOnly: true
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz/live
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz/ready
|
||||
port: 8080
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
resources:
|
||||
requests:
|
||||
memory: 256Mi
|
||||
cpu: 100m
|
||||
limits:
|
||||
memory: 1Gi
|
||||
cpu: "1"
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: stalwart-config
|
||||
- name: tls
|
||||
secret:
|
||||
secretName: stalwart-tls
|
||||
30
base/stalwart/stalwart-service.yaml
Normal file
30
base/stalwart/stalwart-service.yaml
Normal file
@@ -0,0 +1,30 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: stalwart
|
||||
namespace: stalwart
|
||||
spec:
|
||||
selector:
|
||||
app: stalwart
|
||||
ports:
|
||||
- name: smtp
|
||||
port: 25
|
||||
targetPort: 25
|
||||
- name: submission
|
||||
port: 587
|
||||
targetPort: 587
|
||||
- name: smtps
|
||||
port: 465
|
||||
targetPort: 465
|
||||
- name: imap
|
||||
port: 143
|
||||
targetPort: 143
|
||||
- name: imaps
|
||||
port: 993
|
||||
targetPort: 993
|
||||
- name: managesieve
|
||||
port: 4190
|
||||
targetPort: 4190
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
85
base/stalwart/vault-secrets.yaml
Normal file
85
base/stalwart/vault-secrets.yaml
Normal file
@@ -0,0 +1,85 @@
|
||||
---
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultAuth
|
||||
metadata:
|
||||
name: vso-auth
|
||||
namespace: stalwart
|
||||
spec:
|
||||
method: kubernetes
|
||||
mount: kubernetes
|
||||
kubernetes:
|
||||
role: vso
|
||||
serviceAccount: default
|
||||
---
|
||||
# Stalwart DB credentials from OpenBao database secrets engine (static role, 24h rotation).
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultDynamicSecret
|
||||
metadata:
|
||||
name: stalwart-db-credentials
|
||||
namespace: stalwart
|
||||
spec:
|
||||
vaultAuthRef: vso-auth
|
||||
mount: database
|
||||
path: static-creds/stalwart
|
||||
allowStaticCreds: true
|
||||
refreshAfter: 5m
|
||||
rolloutRestartTargets:
|
||||
- kind: Deployment
|
||||
name: stalwart
|
||||
destination:
|
||||
name: stalwart-db-credentials
|
||||
create: true
|
||||
overwrite: true
|
||||
transformation:
|
||||
excludeRaw: true
|
||||
templates:
|
||||
password:
|
||||
text: "{{ index .Secrets \"password\" }}"
|
||||
---
|
||||
# Stalwart application secrets (admin password, DKIM key) from OpenBao KV.
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: stalwart-app-secrets
|
||||
namespace: stalwart
|
||||
spec:
|
||||
vaultAuthRef: vso-auth
|
||||
mount: secret
|
||||
type: kv-v2
|
||||
path: stalwart
|
||||
refreshAfter: 30s
|
||||
destination:
|
||||
name: stalwart-app-secrets
|
||||
create: true
|
||||
overwrite: true
|
||||
transformation:
|
||||
excludeRaw: true
|
||||
templates:
|
||||
admin-password:
|
||||
text: "{{ index .Secrets \"admin-password\" }}"
|
||||
dkim-private-key:
|
||||
text: "{{ index .Secrets \"dkim-private-key\" }}"
|
||||
---
|
||||
# SeaweedFS S3 credentials (shared — same secret as lasuite namespace).
|
||||
apiVersion: secrets.hashicorp.com/v1beta1
|
||||
kind: VaultStaticSecret
|
||||
metadata:
|
||||
name: seaweedfs-s3-credentials
|
||||
namespace: stalwart
|
||||
spec:
|
||||
vaultAuthRef: vso-auth
|
||||
mount: secret
|
||||
type: kv-v2
|
||||
path: seaweedfs
|
||||
refreshAfter: 30s
|
||||
destination:
|
||||
name: seaweedfs-s3-credentials
|
||||
create: true
|
||||
overwrite: true
|
||||
transformation:
|
||||
excludeRaw: true
|
||||
templates:
|
||||
S3_ACCESS_KEY:
|
||||
text: "{{ index .Secrets \"access-key\" }}"
|
||||
S3_SECRET_KEY:
|
||||
text: "{{ index .Secrets \"secret-key\" }}"
|
||||
@@ -25,7 +25,7 @@ spec:
|
||||
mountPath: /data/filer
|
||||
containers:
|
||||
- name: filer
|
||||
image: chrislusf/seaweedfs:latest
|
||||
image: chrislusf/seaweedfs:4.18
|
||||
args:
|
||||
- filer
|
||||
- -port=8888
|
||||
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
spec:
|
||||
containers:
|
||||
- name: master
|
||||
image: chrislusf/seaweedfs:latest
|
||||
image: chrislusf/seaweedfs:4.18
|
||||
args:
|
||||
- master
|
||||
- -port=9333
|
||||
|
||||
@@ -20,6 +20,7 @@ resources:
|
||||
- ../../base/lasuite
|
||||
- ../../base/media
|
||||
- ../../base/devtools
|
||||
- ../../base/stalwart
|
||||
- ../../base/vso
|
||||
|
||||
images:
|
||||
|
||||
@@ -17,6 +17,7 @@ resources:
|
||||
- ../../base/lasuite
|
||||
- ../../base/media
|
||||
- ../../base/devtools
|
||||
- ../../base/stalwart
|
||||
- ../../base/vso
|
||||
- ../../base/monitoring
|
||||
- ../../base/matrix
|
||||
@@ -107,12 +108,21 @@ patches:
|
||||
# SeaweedFS volume: expand PVC to 600 Gi
|
||||
- path: patch-seaweedfs-volume-size.yaml
|
||||
|
||||
# MTA-in: bind port 25 to the host for inbound email delivery
|
||||
# Stalwart: bind SMTP (25), submission (587), and IMAPS (993) to the host
|
||||
- patch: |
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/ports/0/hostPort
|
||||
value: 25
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/ports/1/hostPort
|
||||
value: 587
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/ports/2/hostPort
|
||||
value: 465
|
||||
- op: add
|
||||
path: /spec/template/spec/containers/0/ports/4/hostPort
|
||||
value: 993
|
||||
target:
|
||||
kind: Deployment
|
||||
name: messages-mta-in
|
||||
namespace: lasuite
|
||||
name: stalwart
|
||||
namespace: stalwart
|
||||
|
||||
54
scripts/export-mbox.py
Normal file
54
scripts/export-mbox.py
Normal file
@@ -0,0 +1,54 @@
|
||||
import os, zlib, gzip, email.utils, time
|
||||
import psycopg2
|
||||
|
||||
DB_HOST = "postgres-rw.data.svc.cluster.local"
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=5432, dbname="messages_db",
|
||||
user="messages", password=os.environ["DB_PASSWORD"]
|
||||
)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("""
|
||||
SELECT DISTINCT mb.local_part || '@' || d.name as mailbox_email,
|
||||
b.raw_content, b.compression, m.sent_at
|
||||
FROM messages_message m
|
||||
JOIN messages_blob b ON m.blob_id = b.id
|
||||
JOIN messages_thread t ON m.thread_id = t.id
|
||||
JOIN messages_threadaccess ta ON ta.thread_id = t.id
|
||||
JOIN messages_mailbox mb ON ta.mailbox_id = mb.id
|
||||
JOIN messages_maildomain d ON mb.domain_id = d.id
|
||||
ORDER BY mailbox_email, m.sent_at
|
||||
""")
|
||||
|
||||
os.makedirs("/tmp/mbox", exist_ok=True)
|
||||
counts = {}
|
||||
for row in cur.fetchall():
|
||||
mailbox_email, raw_content, compression, sent_at = row
|
||||
raw = bytes(raw_content)
|
||||
if compression == 1:
|
||||
try:
|
||||
eml = gzip.decompress(raw)
|
||||
except Exception:
|
||||
try:
|
||||
eml = zlib.decompress(raw, -zlib.MAX_WBITS)
|
||||
except Exception:
|
||||
eml = raw
|
||||
else:
|
||||
eml = raw
|
||||
|
||||
date_str = email.utils.formatdate(time.mktime(sent_at.timetuple())) if sent_at else email.utils.formatdate()
|
||||
mbox_path = f"/tmp/mbox/{mailbox_email}.mbox"
|
||||
with open(mbox_path, "ab") as f:
|
||||
f.write(f"From {mailbox_email} {date_str}\n".encode())
|
||||
for line in eml.split(b"\n"):
|
||||
if line.startswith(b"From "):
|
||||
f.write(b">" + line + b"\n")
|
||||
else:
|
||||
f.write(line + b"\n")
|
||||
f.write(b"\n")
|
||||
counts[mailbox_email] = counts.get(mailbox_email, 0) + 1
|
||||
|
||||
conn.close()
|
||||
for addr, count in counts.items():
|
||||
print(f"{addr}: {count} messages")
|
||||
print("Export complete.")
|
||||
115
scripts/migrate-calendars.sh
Executable file
115
scripts/migrate-calendars.sh
Executable file
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# Migrate calendars from La Suite Calendars (CalDAV) to Stalwart (CalDAV).
|
||||
#
|
||||
# Exports all calendars per user from the old CalDAV endpoint and imports them
|
||||
# into Stalwart's CalDAV endpoint.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - curl
|
||||
# - Port-forward to both old calendars-backend and new Stalwart:
|
||||
# kubectl port-forward -n lasuite svc/calendars-backend 8081:80 &
|
||||
# kubectl port-forward -n stalwart svc/stalwart 8080:8080 &
|
||||
#
|
||||
# Usage:
|
||||
# ./migrate-calendars.sh \
|
||||
# --old-url http://127.0.0.1:8081 \
|
||||
# --new-url http://127.0.0.1:8080 \
|
||||
# --new-user admin --new-password <pw> \
|
||||
# --users "sienna@sunbeam.pt,amber@sunbeam.pt,lonni@sunbeam.pt"
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
EXPORT_DIR="$(mktemp -d)/calendar-export"
|
||||
mkdir -p "$EXPORT_DIR"
|
||||
|
||||
OLD_URL=""
|
||||
NEW_URL=""
|
||||
NEW_USER=""
|
||||
NEW_PASS=""
|
||||
USERS=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--old-url) OLD_URL="$2"; shift 2 ;;
|
||||
--new-url) NEW_URL="$2"; shift 2 ;;
|
||||
--new-user) NEW_USER="$2"; shift 2 ;;
|
||||
--new-password) NEW_PASS="$2"; shift 2 ;;
|
||||
--users) USERS="$2"; shift 2 ;;
|
||||
*) echo "Unknown arg: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$OLD_URL" || -z "$NEW_URL" || -z "$NEW_USER" || -z "$NEW_PASS" || -z "$USERS" ]]; then
|
||||
echo "Usage: $0 --old-url <url> --new-url <url> --new-user <user> --new-password <pw> --users <comma-separated>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IFS=',' read -ra USER_LIST <<< "$USERS"
|
||||
|
||||
echo "==> Exporting calendars from La Suite..."
|
||||
|
||||
for user in "${USER_LIST[@]}"; do
|
||||
user_dir="$EXPORT_DIR/$user"
|
||||
mkdir -p "$user_dir"
|
||||
|
||||
echo " Exporting calendars for $user..."
|
||||
|
||||
# NOTE: The exact CalDAV path depends on La Suite's CalDAV implementation.
|
||||
# La Suite Calendars uses /caldav/<user>/ or similar. Adjust as needed.
|
||||
# This PROPFIND discovers all calendars for the user.
|
||||
curl -s -X PROPFIND \
|
||||
-H "Content-Type: application/xml" \
|
||||
-d '<?xml version="1.0" encoding="utf-8"?>
|
||||
<d:propfind xmlns:d="DAV:" xmlns:cs="urn:ietf:params:xml:ns:caldav">
|
||||
<d:prop>
|
||||
<d:displayname/>
|
||||
<d:resourcetype/>
|
||||
</d:prop>
|
||||
</d:propfind>' \
|
||||
"$OLD_URL/caldav/$user/" \
|
||||
-o "$user_dir/calendars.xml" 2>/dev/null || true
|
||||
|
||||
# Export each calendar as .ics via CalDAV REPORT
|
||||
# NOTE: This is a simplified template. The actual export depends on
|
||||
# the La Suite CalDAV response format. You may need to parse the
|
||||
# PROPFIND response to discover calendar URLs, then issue
|
||||
# calendar-multiget or calendar-query REPORT requests.
|
||||
#
|
||||
# A simpler alternative: if La Suite exposes /ical/ export endpoints
|
||||
# (seen in pingora config: /ical/ → calendars-backend), use those:
|
||||
#
|
||||
# curl -s "$OLD_URL/ical/$user/calendar.ics" -o "$user_dir/calendar.ics"
|
||||
#
|
||||
echo " Exported to $user_dir/"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "==> Importing calendars into Stalwart..."
|
||||
|
||||
for user in "${USER_LIST[@]}"; do
|
||||
user_dir="$EXPORT_DIR/$user"
|
||||
|
||||
for ics_file in "$user_dir"/*.ics; do
|
||||
[[ -f "$ics_file" ]] || continue
|
||||
cal_name=$(basename "$ics_file" .ics)
|
||||
|
||||
echo " Importing $cal_name for $user..."
|
||||
|
||||
# Upload .ics to Stalwart CalDAV.
|
||||
# Stalwart CalDAV path: /dav/calendars/user/<user>/<calendar-name>/
|
||||
curl -s -X PUT \
|
||||
-u "$NEW_USER:$NEW_PASS" \
|
||||
-H "Content-Type: text/calendar" \
|
||||
--data-binary "@$ics_file" \
|
||||
"$NEW_URL/dav/calendars/user/$user/$cal_name.ics" || {
|
||||
echo " ⚠️ Failed to import $cal_name for $user"
|
||||
}
|
||||
done
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "==> Migration complete. Exported data is in: $EXPORT_DIR"
|
||||
echo ""
|
||||
echo "Verify by comparing event counts:"
|
||||
echo " Old: curl -s '$OLD_URL/caldav/<user>/' -X PROPFIND | grep -c VEVENT"
|
||||
echo " New: curl -s -u admin:pw '$NEW_URL/dav/calendars/user/<user>/' -X PROPFIND | grep -c VEVENT"
|
||||
57
scripts/migrate-mbox-job.yaml
Normal file
57
scripts/migrate-mbox-job.yaml
Normal file
@@ -0,0 +1,57 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: migrate-mbox
|
||||
namespace: stalwart
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
initContainers:
|
||||
- name: export
|
||||
image: python:3.12-slim
|
||||
command: ["/bin/sh", "-c", "pip -q install psycopg2-binary && python3 /scripts/export-mbox.py && ls -la /tmp/mbox/"]
|
||||
env:
|
||||
- name: DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: messages-db-credentials
|
||||
key: password
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: mbox
|
||||
mountPath: /tmp/mbox
|
||||
containers:
|
||||
- name: import
|
||||
image: stalwartlabs/stalwart:v0.15.5
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
URL="http://stalwart.stalwart.svc.cluster.local:8080"
|
||||
CREDS="admin:${ADMIN_PASSWORD}"
|
||||
|
||||
for mbox in /tmp/mbox/*.mbox; do
|
||||
ACCOUNT=$(basename "$mbox" .mbox)
|
||||
echo "=== Importing $mbox into $ACCOUNT ==="
|
||||
stalwart-cli -u "$URL" -c "$CREDS" import messages -f mbox "$ACCOUNT" "$mbox"
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "All imports complete."
|
||||
env:
|
||||
- name: ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-app-secrets
|
||||
key: admin-password
|
||||
volumeMounts:
|
||||
- name: mbox
|
||||
mountPath: /tmp/mbox
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: export-mbox-script
|
||||
- name: mbox
|
||||
emptyDir: {}
|
||||
261
scripts/migrate-messages-job.yaml
Normal file
261
scripts/migrate-messages-job.yaml
Normal file
@@ -0,0 +1,261 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: migrate-messages
|
||||
namespace: stalwart
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: migrate
|
||||
image: python:3.12-slim
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
pip install psycopg2-binary && python3 /scripts/migrate.py
|
||||
env:
|
||||
- name: ADMIN_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: stalwart-app-secrets
|
||||
key: admin-password
|
||||
- name: DB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: messages-db-credentials
|
||||
key: password
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: migrate-messages-script
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: migrate-messages-script
|
||||
namespace: stalwart
|
||||
data:
|
||||
migrate.py: |
|
||||
import json, os, zlib, urllib.request, urllib.error, base64, sys
|
||||
import psycopg2
|
||||
|
||||
DB_HOST = "postgres-rw.data.svc.cluster.local"
|
||||
DB_PORT = 5432
|
||||
DB_NAME = "messages_db"
|
||||
DB_USER = "messages"
|
||||
DB_PASS = os.environ.get("DB_PASSWORD", "")
|
||||
JMAP_URL = "http://stalwart.stalwart.svc.cluster.local:8080"
|
||||
ADMIN_USER = "admin"
|
||||
|
||||
ADMIN_PASS = os.environ["ADMIN_PASSWORD"]
|
||||
|
||||
auth_header = "Basic " + base64.b64encode(f"{ADMIN_USER}:{ADMIN_PASS}".encode()).decode()
|
||||
|
||||
def jmap_call(method_calls):
|
||||
body = json.dumps({
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": method_calls,
|
||||
}).encode()
|
||||
req = urllib.request.Request(f"{JMAP_URL}/jmap", data=body,
|
||||
headers={"Authorization": auth_header, "Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
def upload_blob(account_id, eml_bytes):
|
||||
req = urllib.request.Request(
|
||||
f"{JMAP_URL}/jmap/upload/{account_id}/",
|
||||
data=eml_bytes,
|
||||
headers={"Authorization": auth_header, "Content-Type": "message/rfc822"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())["blobId"]
|
||||
|
||||
def ensure_user(email, full_name):
|
||||
body = json.dumps({
|
||||
"type": "individual", "name": email,
|
||||
"description": full_name or email, "emails": [email],
|
||||
"quota": 0, "secrets": [], "urls": [], "memberOf": [],
|
||||
"roles": ["user"], "lists": [], "members": [],
|
||||
"enabledPermissions": [], "disabledPermissions": [], "externalMembers": [],
|
||||
}).encode()
|
||||
req = urllib.request.Request(f"{JMAP_URL}/api/principal",
|
||||
data=body, method="POST",
|
||||
headers={"Authorization": auth_header, "Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
result = json.loads(resp.read())
|
||||
print(f" Created user {email} (id={result.get('data')})")
|
||||
except urllib.error.HTTPError as e:
|
||||
body_text = e.read().decode()
|
||||
if "fieldAlreadyExists" in body_text:
|
||||
print(f" User {email} already exists")
|
||||
else:
|
||||
print(f" Error creating user {email}: {e.code} {body_text}")
|
||||
|
||||
def get_account_id_for_user(email):
|
||||
"""Get JMAP account ID by authenticating as the user (admin impersonation)."""
|
||||
# Stalwart allows admin to access any account via the master user mechanism:
|
||||
# authenticate as "user%admin" with admin password
|
||||
impersonate_auth = "Basic " + base64.b64encode(
|
||||
f"{email}%{ADMIN_USER}:{ADMIN_PASS}".encode()).decode()
|
||||
req = urllib.request.Request(f"{JMAP_URL}/.well-known/jmap",
|
||||
headers={"Authorization": impersonate_auth})
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
session = json.loads(resp.read())
|
||||
return next(iter(session.get("accounts", {})), None), impersonate_auth
|
||||
except urllib.error.HTTPError:
|
||||
# Fallback: try direct admin auth
|
||||
req = urllib.request.Request(f"{JMAP_URL}/.well-known/jmap",
|
||||
headers={"Authorization": auth_header})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
session = json.loads(resp.read())
|
||||
return next(iter(session.get("accounts", {})), None), auth_header
|
||||
|
||||
def get_inbox_id(account_id, user_auth):
|
||||
body = json.dumps({
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": [["Mailbox/get", {"accountId": account_id}, "0"]],
|
||||
}).encode()
|
||||
req = urllib.request.Request(f"{JMAP_URL}/jmap", data=body,
|
||||
headers={"Authorization": user_auth, "Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
result = json.loads(resp.read())
|
||||
mailboxes = result["methodResponses"][0][1]["list"]
|
||||
for mb in mailboxes:
|
||||
if mb.get("role") == "inbox" or mb.get("name", "").lower() == "inbox":
|
||||
return mb["id"]
|
||||
return mailboxes[0]["id"] if mailboxes else None
|
||||
|
||||
def upload_blob_as(account_id, eml_bytes, user_auth):
|
||||
req = urllib.request.Request(
|
||||
f"{JMAP_URL}/jmap/upload/{account_id}/",
|
||||
data=eml_bytes,
|
||||
headers={"Authorization": user_auth, "Content-Type": "message/rfc822"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())["blobId"]
|
||||
|
||||
def jmap_call_as(method_calls, user_auth):
|
||||
body = json.dumps({
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": method_calls,
|
||||
}).encode()
|
||||
req = urllib.request.Request(f"{JMAP_URL}/jmap", data=body,
|
||||
headers={"Authorization": user_auth, "Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
# Connect to messages_db
|
||||
print("Connecting to messages_db...")
|
||||
conn = psycopg2.connect(host=DB_HOST, port=DB_PORT, dbname=DB_NAME, user=DB_USER, password=DB_PASS)
|
||||
cur = conn.cursor()
|
||||
|
||||
cur.execute("""
|
||||
SELECT DISTINCT m.id, m.subject, m.sent_at, m.is_draft, m.is_starred,
|
||||
m.is_trashed, m.is_spam,
|
||||
mb.local_part || '@' || d.name as mailbox_email,
|
||||
b.raw_content, b.compression
|
||||
FROM messages_message m
|
||||
JOIN messages_blob b ON m.blob_id = b.id
|
||||
JOIN messages_thread t ON m.thread_id = t.id
|
||||
JOIN messages_threadaccess ta ON ta.thread_id = t.id
|
||||
JOIN messages_mailbox mb ON ta.mailbox_id = mb.id
|
||||
JOIN messages_maildomain d ON mb.domain_id = d.id
|
||||
ORDER BY mailbox_email, m.sent_at
|
||||
""")
|
||||
messages = cur.fetchall()
|
||||
print(f"Found {len(messages)} messages to migrate.")
|
||||
|
||||
cur.execute("SELECT email, full_name FROM messages_user")
|
||||
users = {row[0]: row[1] for row in cur.fetchall()}
|
||||
conn.close()
|
||||
|
||||
# Ensure all mailbox users exist in Stalwart
|
||||
seen_emails = set()
|
||||
for msg in messages:
|
||||
email = msg[7]
|
||||
if email not in seen_emails:
|
||||
seen_emails.add(email)
|
||||
ensure_user(email, users.get(email, email))
|
||||
|
||||
# Resolve per-user JMAP accounts
|
||||
user_contexts = {}
|
||||
for email in seen_emails:
|
||||
print(f"Resolving JMAP account for {email}...")
|
||||
account_id, user_auth = get_account_id_for_user(email)
|
||||
if account_id:
|
||||
inbox_id = get_inbox_id(account_id, user_auth)
|
||||
user_contexts[email] = (account_id, inbox_id, user_auth)
|
||||
print(f" {email}: account={account_id}, inbox={inbox_id}")
|
||||
else:
|
||||
print(f" WARNING: Could not get account for {email}, will skip")
|
||||
|
||||
# Import messages into each user's account
|
||||
imported = 0
|
||||
errors = 0
|
||||
for msg in messages:
|
||||
msg_id, subject, sent_at, is_draft, is_starred, is_trashed, is_spam, email, raw_content, compression = msg
|
||||
|
||||
if email not in user_contexts:
|
||||
errors += 1
|
||||
print(f" SKIP: {email}: {subject} (no account)")
|
||||
continue
|
||||
|
||||
account_id, inbox_id, user_auth = user_contexts[email]
|
||||
|
||||
try:
|
||||
raw = bytes(raw_content)
|
||||
if compression == 1:
|
||||
import gzip
|
||||
try:
|
||||
eml_bytes = gzip.decompress(raw)
|
||||
except Exception:
|
||||
try:
|
||||
eml_bytes = zlib.decompress(raw, -zlib.MAX_WBITS)
|
||||
except Exception:
|
||||
try:
|
||||
eml_bytes = zlib.decompress(raw)
|
||||
except Exception:
|
||||
eml_bytes = raw
|
||||
else:
|
||||
eml_bytes = raw
|
||||
|
||||
blob_id = upload_blob_as(account_id, eml_bytes, user_auth)
|
||||
|
||||
keywords = {"$seen": True}
|
||||
if is_starred:
|
||||
keywords["$flagged"] = True
|
||||
if is_draft:
|
||||
keywords["$draft"] = True
|
||||
|
||||
received_at = sent_at.isoformat() if sent_at else None
|
||||
import_data = {
|
||||
"accountId": account_id,
|
||||
"emails": {
|
||||
"imp1": {
|
||||
"blobId": blob_id,
|
||||
"mailboxIds": {inbox_id: True},
|
||||
"keywords": keywords,
|
||||
}
|
||||
}
|
||||
}
|
||||
if received_at:
|
||||
import_data["emails"]["imp1"]["receivedAt"] = received_at
|
||||
|
||||
result = jmap_call_as([["Email/import", import_data, "0"]], user_auth)
|
||||
resp = result["methodResponses"][0][1]
|
||||
if "created" in resp and "imp1" in resp["created"]:
|
||||
imported += 1
|
||||
print(f" [{imported}] {email}: {subject or '(no subject)'}")
|
||||
else:
|
||||
errors += 1
|
||||
print(f" ERROR: {email}: {subject}: {resp.get('notCreated', {})}")
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
print(f" ERROR: {email}: {subject}: {e}")
|
||||
|
||||
print(f"\nMigration complete: {imported} imported, {errors} errors, {len(messages)} total")
|
||||
277
scripts/migrate-messages.py
Executable file
277
scripts/migrate-messages.py
Executable file
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate emails from La Suite Messages (PostgreSQL + SeaweedFS) to Stalwart (JMAP).
|
||||
|
||||
Reads message metadata from messages_db, downloads RFC 5322 bodies from SeaweedFS,
|
||||
and uploads each message to Stalwart via JMAP Email/import (RFC 8621 §5.4).
|
||||
|
||||
Usage:
|
||||
# Port-forward to the required services first:
|
||||
kubectl port-forward -n data svc/postgres-rw 5432:5432 &
|
||||
kubectl port-forward -n stalwart svc/stalwart 8080:8080 &
|
||||
|
||||
# Run the migration:
|
||||
python3 migrate-messages.py \
|
||||
--db-host 127.0.0.1 --db-port 5432 --db-name messages_db --db-user messages --db-password <pw> \
|
||||
--s3-endpoint http://127.0.0.1:8333 --s3-bucket sunbeam-messages \
|
||||
--s3-access-key <key> --s3-secret-key <secret> \
|
||||
--jmap-url http://127.0.0.1:8080 --jmap-user admin --jmap-password <pw>
|
||||
|
||||
Prerequisites:
|
||||
pip install psycopg2-binary boto3 requests
|
||||
|
||||
The script is idempotent: it tracks progress in a checkpoint file and skips
|
||||
already-imported messages on re-run.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import psycopg2
|
||||
import boto3
|
||||
import requests
|
||||
except ImportError:
|
||||
print("Missing dependencies. Install with: pip install psycopg2-binary boto3 requests")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
CHECKPOINT_FILE = Path("migrate-messages-checkpoint.json")
|
||||
|
||||
|
||||
def load_checkpoint():
|
||||
if CHECKPOINT_FILE.exists():
|
||||
return json.loads(CHECKPOINT_FILE.read_text())
|
||||
return {"imported": {}}
|
||||
|
||||
|
||||
def save_checkpoint(checkpoint):
|
||||
CHECKPOINT_FILE.write_text(json.dumps(checkpoint, indent=2))
|
||||
|
||||
|
||||
def get_jmap_session(jmap_url, user, password):
|
||||
"""Get JMAP session and extract accountId."""
|
||||
resp = requests.get(
|
||||
f"{jmap_url}/.well-known/jmap",
|
||||
auth=(user, password),
|
||||
)
|
||||
resp.raise_for_status()
|
||||
session = resp.json()
|
||||
# Use the primary account
|
||||
primary_accounts = session.get("primaryAccounts", {})
|
||||
account_id = primary_accounts.get("urn:ietf:params:jmap:mail")
|
||||
if not account_id:
|
||||
# Fallback: first account
|
||||
accounts = session.get("accounts", {})
|
||||
account_id = next(iter(accounts))
|
||||
return session, account_id
|
||||
|
||||
|
||||
def jmap_get_mailboxes(jmap_url, account_id, user, password):
|
||||
"""Fetch all mailboxes for the account."""
|
||||
resp = requests.post(
|
||||
f"{jmap_url}/jmap",
|
||||
auth=(user, password),
|
||||
json={
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": [
|
||||
["Mailbox/get", {"accountId": account_id}, "0"]
|
||||
],
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
mailboxes = result["methodResponses"][0][1]["list"]
|
||||
return {mb["name"]: mb["id"] for mb in mailboxes}
|
||||
|
||||
|
||||
def jmap_create_mailbox(jmap_url, account_id, user, password, name):
|
||||
"""Create a mailbox and return its ID."""
|
||||
resp = requests.post(
|
||||
f"{jmap_url}/jmap",
|
||||
auth=(user, password),
|
||||
json={
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": [
|
||||
["Mailbox/set", {
|
||||
"accountId": account_id,
|
||||
"create": {"mb1": {"name": name}},
|
||||
}, "0"]
|
||||
],
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
created = result["methodResponses"][0][1].get("created", {})
|
||||
if "mb1" in created:
|
||||
return created["mb1"]["id"]
|
||||
# Already exists — fetch it
|
||||
mailboxes = jmap_get_mailboxes(jmap_url, account_id, user, password)
|
||||
return mailboxes.get(name)
|
||||
|
||||
|
||||
def jmap_import_email(jmap_url, account_id, user, password, eml_bytes, mailbox_id, keywords, received_at):
|
||||
"""Import a single RFC 5322 message via JMAP Email/import."""
|
||||
# First, upload the blob
|
||||
resp = requests.post(
|
||||
f"{jmap_url}/jmap/upload/{account_id}/",
|
||||
auth=(user, password),
|
||||
headers={"Content-Type": "message/rfc822"},
|
||||
data=eml_bytes,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
blob = resp.json()
|
||||
blob_id = blob["blobId"]
|
||||
|
||||
# Then import it
|
||||
import_data = {
|
||||
"accountId": account_id,
|
||||
"emails": {
|
||||
"imp1": {
|
||||
"blobId": blob_id,
|
||||
"mailboxIds": {mailbox_id: True},
|
||||
"keywords": keywords,
|
||||
}
|
||||
},
|
||||
}
|
||||
if received_at:
|
||||
import_data["emails"]["imp1"]["receivedAt"] = received_at
|
||||
|
||||
resp = requests.post(
|
||||
f"{jmap_url}/jmap",
|
||||
auth=(user, password),
|
||||
json={
|
||||
"using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"],
|
||||
"methodCalls": [
|
||||
["Email/import", import_data, "0"]
|
||||
],
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
result = resp.json()
|
||||
created = result["methodResponses"][0][1].get("created", {})
|
||||
if "imp1" in created:
|
||||
return created["imp1"]["id"]
|
||||
not_created = result["methodResponses"][0][1].get("notCreated", {})
|
||||
if "imp1" in not_created:
|
||||
err = not_created["imp1"]
|
||||
raise RuntimeError(f"JMAP import failed: {err}")
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Migrate La Suite Messages → Stalwart JMAP")
|
||||
parser.add_argument("--db-host", default="127.0.0.1")
|
||||
parser.add_argument("--db-port", type=int, default=5432)
|
||||
parser.add_argument("--db-name", default="messages_db")
|
||||
parser.add_argument("--db-user", default="messages")
|
||||
parser.add_argument("--db-password", required=True)
|
||||
parser.add_argument("--s3-endpoint", default="http://127.0.0.1:8333")
|
||||
parser.add_argument("--s3-bucket", default="sunbeam-messages")
|
||||
parser.add_argument("--s3-access-key", required=True)
|
||||
parser.add_argument("--s3-secret-key", required=True)
|
||||
parser.add_argument("--jmap-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--jmap-user", default="admin")
|
||||
parser.add_argument("--jmap-password", required=True)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Count messages without importing")
|
||||
args = parser.parse_args()
|
||||
|
||||
checkpoint = load_checkpoint()
|
||||
|
||||
# Connect to messages_db
|
||||
print("Connecting to messages_db...")
|
||||
conn = psycopg2.connect(
|
||||
host=args.db_host,
|
||||
port=args.db_port,
|
||||
dbname=args.db_name,
|
||||
user=args.db_user,
|
||||
password=args.db_password,
|
||||
)
|
||||
|
||||
# Connect to SeaweedFS
|
||||
print("Connecting to SeaweedFS...")
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=args.s3_endpoint,
|
||||
aws_access_key_id=args.s3_access_key,
|
||||
aws_secret_access_key=args.s3_secret_key,
|
||||
region_name="us-east-1",
|
||||
)
|
||||
|
||||
# Get JMAP session
|
||||
if not args.dry_run:
|
||||
print("Connecting to Stalwart JMAP...")
|
||||
session, account_id = get_jmap_session(args.jmap_url, args.jmap_user, args.jmap_password)
|
||||
print(f" Account: {account_id}")
|
||||
mailboxes = jmap_get_mailboxes(args.jmap_url, account_id, args.jmap_user, args.jmap_password)
|
||||
print(f" Mailboxes: {list(mailboxes.keys())}")
|
||||
|
||||
# Query all messages from La Suite
|
||||
# NOTE: The actual table/column names depend on La Suite Messages' Django models.
|
||||
# You may need to adjust these queries after inspecting the actual schema.
|
||||
# Run `\dt` and `\d <table>` in psql against messages_db to find the real names.
|
||||
print("\nQuerying messages from La Suite database...")
|
||||
cur = conn.cursor()
|
||||
|
||||
# List all tables to help identify the right ones
|
||||
cur.execute("""
|
||||
SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public' ORDER BY table_name;
|
||||
""")
|
||||
tables = [row[0] for row in cur.fetchall()]
|
||||
print(f" Tables: {tables}")
|
||||
|
||||
if args.dry_run:
|
||||
# Just count messages per table that looks like it contains emails
|
||||
for table in tables:
|
||||
cur.execute(f"SELECT COUNT(*) FROM {table};")
|
||||
count = cur.fetchone()[0]
|
||||
if count > 0:
|
||||
print(f" {table}: {count} rows")
|
||||
print("\nDry run complete. Inspect the tables above and adjust the migration")
|
||||
print("queries in this script to match the actual La Suite Messages schema.")
|
||||
conn.close()
|
||||
return
|
||||
|
||||
# TODO: Replace with actual queries once schema is inspected.
|
||||
# The migration logic below is a template — run with --dry-run first
|
||||
# to see the actual table structure, then update these queries.
|
||||
#
|
||||
# Expected flow:
|
||||
# 1. Query user accounts
|
||||
# 2. For each user, query their mailboxes/folders
|
||||
# 3. For each mailbox, query messages (S3 key, flags, received date)
|
||||
# 4. Download .eml from S3
|
||||
# 5. Upload to Stalwart via JMAP Email/import
|
||||
#
|
||||
# Example (adjust table/column names):
|
||||
#
|
||||
# cur.execute("SELECT id, email FROM auth_user;")
|
||||
# for user_id, email in cur.fetchall():
|
||||
# cur.execute("SELECT id, name FROM mailbox WHERE user_id = %s;", (user_id,))
|
||||
# for mb_id, mb_name in cur.fetchall():
|
||||
# mailbox_jmap_id = ensure_mailbox(mb_name)
|
||||
# cur.execute("SELECT s3_key, is_read, received_at FROM message WHERE mailbox_id = %s;", (mb_id,))
|
||||
# for s3_key, is_read, received_at in cur.fetchall():
|
||||
# if s3_key in checkpoint["imported"]:
|
||||
# continue
|
||||
# eml = s3.get_object(Bucket=args.s3_bucket, Key=s3_key)["Body"].read()
|
||||
# keywords = {"$seen": True} if is_read else {}
|
||||
# jmap_import_email(..., eml, mailbox_jmap_id, keywords, received_at)
|
||||
# checkpoint["imported"][s3_key] = True
|
||||
# save_checkpoint(checkpoint)
|
||||
|
||||
print("\n⚠️ Schema inspection required!")
|
||||
print("Run with --dry-run first, then update the TODO section in this script")
|
||||
print("with the correct table and column names from the La Suite Messages schema.")
|
||||
|
||||
conn.close()
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user