✨(project) Django boilerplate

This commit introduces a boilerplate inspired by https://github.com/numerique-gouv/impress. The code has been cleaned to remove unnecessary Impress logic and dependencies. Changes made: - Removed Minio, WebRTC, and create bucket from the stack. - Removed the Next.js frontend (it will be replaced by Vite). - Cleaned up impress-specific backend logics. The whole stack remains functional: - All tests pass. - Linter checks pass. - Agent Connexion sources are already set-up. Why clear out the code? To adhere to the KISS principle, we aim to maintain a minimalist codebase. Cloning Impress allowed us to quickly inherit its code quality tools and deployment configurations for staging, pre-production, and production environments. What’s broken? - The tsclient is not functional anymore. - Some make commands need to be fixed. - Helm sources are outdated. - Naming across the project sources are inconsistent (impress, visio, etc.) - CI is not configured properly. This list might be incomplete. Let's grind it.
2024-01-09 15:30:36 +01:00
parent 2d81979b0a
commit 5b1a2b20de
146 changed files with 12668 additions and 1 deletions
--- a/bin/Tiltfile
+++ b/bin/Tiltfile
@@ -0,0 +1,68 @@
+load('ext://uibutton', 'cmd_button', 'bool_input', 'location')
+load('ext://namespace', 'namespace_create', 'namespace_inject')
+namespace_create('impress')
+
+docker_build(
+    'localhost:5001/impress-backend:latest',
+    context='..',
+    dockerfile='../Dockerfile',
+    only=['./src/backend', './src/mail', './docker'],
+    target = 'backend-production',
+    live_update=[
+        sync('../src/backend', '/app'),
+        run(
+            'pip install -r /app/requirements.txt',
+            trigger=['./api/requirements.txt']
+        )
+    ]
+)
+
+docker_build(
+    'localhost:5001/impress-y-webrtc-signaling:latest',
+    context='..',
+    dockerfile='../src/frontend/Dockerfile',
+    only=['./src/frontend/', './docker/', './dockerignore'],
+    target = 'y-webrtc-signaling',
+    live_update=[
+        sync('../src/frontend/apps/y-webrtc-signaling/src', '/home/frontend/apps/y-webrtc-signaling/src'),
+    ]
+)
+
+docker_build(
+    'localhost:5001/impress-frontend:latest',
+    context='..',
+    dockerfile='../src/frontend/Dockerfile',
+    only=['./src/frontend', './docker', './dockerignore'],
+    target = 'impress',
+    live_update=[
+        sync('../src/frontend', '/home/frontend'),
+    ]
+)
+
+k8s_yaml(local('cd ../src/helm && helmfile -n impress -e dev template .'))
+
+migration = '''
+set -eu
+# get k8s pod name from tilt resource name
+POD_NAME="$(tilt get kubernetesdiscovery impress-backend -ojsonpath='{.status.pods[0].name}')"
+kubectl -n impress exec "$POD_NAME" -- python manage.py makemigrations
+'''
+cmd_button('Make migration',
+           argv=['sh', '-c', migration],
+           resource='impress-backend',
+           icon_name='developer_board',
+           text='Run makemigration',
+)
+
+pod_migrate = '''
+set -eu
+# get k8s pod name from tilt resource name
+POD_NAME="$(tilt get kubernetesdiscovery impress-backend -ojsonpath='{.status.pods[0].name}')"
+kubectl -n impress exec "$POD_NAME" -- python manage.py migrate --no-input
+'''
+cmd_button('Migrate db',
+           argv=['sh', '-c', pod_migrate],
+           resource='impress-backend',
+           icon_name='developer_board',
+           text='Run database migration',
+)
--- a/bin/_config.sh
+++ b/bin/_config.sh
@@ -0,0 +1,157 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+REPO_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd)"
+UNSET_USER=0
+
+TERRAFORM_DIRECTORY="./env.d/terraform"
+COMPOSE_FILE="${REPO_DIR}/docker-compose.yml"
+COMPOSE_PROJECT="impress"
+
+
+# _set_user: set (or unset) default user id used to run docker commands
+#
+# usage: _set_user
+#
+# You can override default user ID (the current host user ID), by defining the
+# USER_ID environment variable.
+#
+# To avoid running docker commands with a custom user, please set the
+# $UNSET_USER environment variable to 1.
+function _set_user() {
+
+    if [ $UNSET_USER -eq 1 ]; then
+        USER_ID=""
+        return
+    fi
+
+    # USER_ID = USER_ID or `id -u` if USER_ID is not set
+    USER_ID=${USER_ID:-$(id -u)}
+
+    echo "🙋(user) ID: ${USER_ID}"
+}
+
+# docker_compose: wrap docker compose command
+#
+# usage: docker_compose [options] [ARGS...]
+#
+# options: docker compose command options
+# ARGS   : docker compose command arguments
+function _docker_compose() {
+
+    echo "🐳(compose) project: '${COMPOSE_PROJECT}' file: '${COMPOSE_FILE}'"
+    docker compose \
+        -p "${COMPOSE_PROJECT}" \
+        -f "${COMPOSE_FILE}" \
+        --project-directory "${REPO_DIR}" \
+        "$@"
+}
+
+# _dc_run: wrap docker compose run command
+#
+# usage: _dc_run [options] [ARGS...]
+#
+# options: docker compose run command options
+# ARGS   : docker compose run command arguments
+function _dc_run() {
+    _set_user
+
+    user_args="--user=$USER_ID"
+    if [ -z $USER_ID ]; then
+        user_args=""
+    fi
+
+    _docker_compose run --rm $user_args "$@"
+}
+
+# _dc_exec: wrap docker compose exec command
+#
+# usage: _dc_exec [options] [ARGS...]
+#
+# options: docker compose exec command options
+# ARGS   : docker compose exec command arguments
+function _dc_exec() {
+    _set_user
+
+    echo "🐳(compose) exec command: '\$@'"
+
+    user_args="--user=$USER_ID"
+    if [ -z $USER_ID ]; then
+        user_args=""
+    fi
+
+    _docker_compose exec $user_args "$@"
+}
+
+# _django_manage: wrap django's manage.py command with docker compose
+#
+# usage : _django_manage [ARGS...]
+#
+# ARGS : django's manage.py command arguments
+function _django_manage() {
+    _dc_run "app-dev" python manage.py "$@"
+}
+
+# _set_openstack_project: select an OpenStack project from the openrc files defined in the
+# terraform directory.
+#
+# usage: _set_openstack_project
+#
+# If necessary the script will prompt the user to choose a project from those available
+function _set_openstack_project() {
+
+    declare prompt
+    declare -a projects
+    declare -i default=1
+    declare -i choice=0
+    declare -i n_projects
+
+    # List projects by looking in the "./env.d/terraform" directory
+    # and store them in an array
+    read -r -a projects <<< "$(
+        find "${TERRAFORM_DIRECTORY}" -maxdepth 1 -mindepth 1 -type d |
+        sed 's|'"${TERRAFORM_DIRECTORY}\/"'||' |
+        xargs
+    )"
+    nb_projects=${#projects[@]}
+
+    if [[ ${nb_projects} -le 0 ]]; then
+        echo "There are no OpenStack projects defined..." >&2
+        echo "To add one, create a subdirectory in \"${TERRAFORM_DIRECTORY}\" with the name" \
+            "of your project and copy your \"openrc.sh\" file into it." >&2
+        exit 10
+    fi
+
+    if [[ ${nb_projects} -gt 1 ]]; then
+        prompt="Select an OpenStack project to target:\\n"
+        for (( i=0; i<nb_projects; i++ )); do
+            prompt+="[$((i+1))] ${projects[$i]}"
+            if [[ $((i+1)) -eq ${default} ]]; then
+                prompt+=" (default)"
+            fi
+            prompt+="\\n"
+        done
+        prompt+="If your OpenStack project is not listed, add it to the \"env.d/terraform\" directory.\\n"
+        prompt+="Your choice: "
+        read -r -p "$(echo -e "${prompt}")" choice
+
+        if [[ ${choice} -gt nb_projects ]]; then
+            (>&2 echo "Invalid choice ${choice} (should be <= ${nb_projects})")
+            exit 11
+        fi
+
+        if [[ ${choice} -le 0 ]]; then
+            choice=${default}
+        fi
+    fi
+
+    project=${projects[$((choice-1))]}
+    # Check that the openrc.sh file exists for this project
+    if [ ! -f "${TERRAFORM_DIRECTORY}/${project}/openrc.sh" ]; then
+        (>&2 echo "Missing \"openrc.sh\" file in \"${TERRAFORM_DIRECTORY}/${project}\". Check documentation.")
+        exit 12
+    fi
+
+    echo "${project}"
+}
--- a/bin/compose
+++ b/bin/compose
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/_config.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+_docker_compose "$@"
--- a/bin/manage
+++ b/bin/manage
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/_config.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+_django_manage "$@"
--- a/bin/pylint
+++ b/bin/pylint
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/_config.sh
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+declare diff_from
+declare -a paths
+declare -a args
+
+# Parse options
+for arg in "$@"
+do
+    case $arg in
+        --diff-only=*)
+            diff_from="${arg#*=}"
+            shift
+            ;;
+        -*)
+            args+=("$arg")
+            shift
+            ;;
+        *)
+            paths+=("$arg")
+            shift
+            ;;
+    esac
+done
+
+if [[ -n "${diff_from}" ]]; then
+    # Run pylint only on modified files located in src/backend
+    # (excluding deleted files and migration files)
+    # shellcheck disable=SC2207
+    paths=($(git diff "${diff_from}" --name-only --diff-filter=d -- src/backend ':!**/migrations/*.py' | grep -E '^src/backend/.*\.py$'))
+fi
+
+# Fix docker vs local path when project sources are mounted as a volume
+read -ra paths <<< "$(echo "${paths[@]}" | sed "s|src/backend/||g")"
+_dc_run app-dev pylint "${paths[@]}" "${args[@]}"
--- a/bin/pytest
+++ b/bin/pytest
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+_dc_run \
+    -e DJANGO_CONFIGURATION=Test \
+    app-dev \
+    pytest "$@"
--- a/bin/start-kind.sh
+++ b/bin/start-kind.sh
@@ -0,0 +1,103 @@
+#!/bin/sh
+set -o errexit
+
+CURRENT_DIR=$(pwd)
+
+echo "0. Create ca"
+# 0. Create ca
+mkcert -install
+cd /tmp
+mkcert "127.0.0.1.nip.io" "*.127.0.0.1.nip.io"
+cd $CURRENT_DIR
+
+echo "1. Create registry container unless it already exists"
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+	docker run \
+		-d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
+		registry:2
+fi
+
+echo "2. Create kind cluster with containerd registry config dir enabled"
+# 2. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+nodes:
+- role: control-plane
+  image: kindest/node:v1.27.3
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+- role: worker
+  image: kindest/node:v1.27.3
+- role: worker
+  image: kindest/node:v1.27.3
+EOF
+
+echo "3. Add the registry config to the nodes"
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes); do
+	docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+	cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+echo "4. Connect the registry to the cluster network if not already connected"
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+	docker network connect "kind" "${reg_name}"
+fi
+
+echo "5. Document the local registry"
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+echo "6. Install ingress-nginx"
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+kubectl -n ingress-nginx create secret tls mkcert --key /tmp/127.0.0.1.nip.io+1-key.pem --cert /tmp/127.0.0.1.nip.io+1.pem
+kubectl -n ingress-nginx patch deployments.apps ingress-nginx-controller --type 'json' -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value":"--default-ssl-certificate=ingress-nginx/mkcert"}]'
--- a/bin/state
+++ b/bin/state
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+project=$(_set_openstack_project)
+echo "Using \"${project}\" project..."
+
+source "${TERRAFORM_DIRECTORY}/${project}/openrc.sh"
+
+# Run Terraform commands in the Hashicorp docker container via docker compose
+# shellcheck disable=SC2068
+DOCKER_USER="$(id -u):$(id -g)" \
+    PROJECT="${project}" \
+    docker compose run --rm \
+    -e OS_AUTH_URL \
+    -e OS_IDENTITY_API_VERSION \
+    -e OS_USER_DOMAIN_NAME \
+    -e OS_PROJECT_DOMAIN_NAME \
+    -e OS_TENANT_ID \
+    -e OS_TENANT_NAME \
+    -e OS_USERNAME \
+    -e OS_PASSWORD \
+    -e OS_REGION_NAME \
+    terraform-state "$@"
--- a/bin/terraform
+++ b/bin/terraform
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+project=$(_set_openstack_project)
+echo "Using \"${project}\" project..."
+
+source "${TERRAFORM_DIRECTORY}/${project}/openrc.sh"
+
+# Run Terraform commands in the Hashicorp docker container via docker compose
+# shellcheck disable=SC2068
+DOCKER_USER="$(id -u):$(id -g)" \
+    PROJECT="${project}" \
+    docker compose run --rm \
+    -e OS_AUTH_URL \
+    -e OS_IDENTITY_API_VERSION \
+    -e OS_USER_DOMAIN_NAME \
+    -e OS_PROJECT_DOMAIN_NAME \
+    -e OS_TENANT_ID \
+    -e OS_TENANT_NAME \
+    -e OS_USERNAME \
+    -e OS_PASSWORD \
+    -e OS_REGION_NAME \
+    -e TF_VAR_user_name \
+    terraform "$@"
--- a/bin/update_openapi_schema
+++ b/bin/update_openapi_schema
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
+
+_dc_run \
+    -e DJANGO_CONFIGURATION=Test \
+    app-dev \
+    python manage.py spectacular \
+    --api-version 'v1.0' \
+    --urlconf 'impress.api_urls' \
+    --format openapi-json \
+    --file /app/core/tests/swagger/swagger.json
--- a/bin/updatekeys.sh
+++ b/bin/updatekeys.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+find . -name "*.enc.*" -exec sops updatekeys -y {} \;