From 1ec98f0948856d25740ebf4d209ecc33e51eb706 Mon Sep 17 00:00:00 2001 From: Quentin BEY Date: Fri, 14 Mar 2025 17:16:25 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=91=E2=80=8D=F0=9F=92=BB(tasks)=20run?= =?UTF-8?q?=20management=20commands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to run management commands from a celery task. --- CHANGELOG.md | 4 ++ src/backend/core/apps.py | 69 +++++++++++++++++-- .../tests/test_management_command_tasks.py | 36 ++++++++++ src/backend/core/utils/io.py | 49 +++++++++++++ src/backend/people/settings.py | 8 +++ 5 files changed, 159 insertions(+), 7 deletions(-) create mode 100644 src/backend/core/tests/test_management_command_tasks.py create mode 100644 src/backend/core/utils/io.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e2c75e3..f7ba538 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to ## [Unreleased] +### Added + +- 🧑‍💻(tasks) run management commands #814 + ## [1.14.1] - 2025-03-17 ## [1.14.0] - 2025-03-17 diff --git a/src/backend/core/apps.py b/src/backend/core/apps.py index d7da9f8..45fa3e9 100644 --- a/src/backend/core/apps.py +++ b/src/backend/core/apps.py @@ -1,11 +1,66 @@ """People Core application""" -# from django.apps import AppConfig -# from django.utils.translation import gettext_lazy as _ + +import logging + +from django.apps import AppConfig +from django.conf import settings +from django.core.management import call_command, get_commands +from django.utils.translation import gettext_lazy as _ + +from core.utils.io import TeeStringIO + +from people.celery_app import app as celery_app + +logger = logging.getLogger(__name__) -# class CoreConfig(AppConfig): -# """Configuration class for the People core app.""" +class CoreConfig(AppConfig): + """Configuration class for the People core app.""" -# name = "core" -# app_label = "core" -# verbose_name = _("People core application") + name = "core" + app_label = "core" + verbose_name = _("People core application") + + def ready(self): + """ + Register management command which are enabled via MANAGEMENT_COMMAND_AS_TASK setting. + """ + for command_name in settings.MANAGEMENT_COMMAND_AS_TASK: + # Check if the command is a valid management command + try: + app_name = get_commands()[command_name] + except KeyError: + logging.error( + "Command %s is not a valid management command.", command_name + ) + continue + + command_full_name = ".".join([app_name, command_name]) + + # Create a closure to capture the current value of command_full_name and command_name + def create_task(cmd_name, cmd_full_name): + @celery_app.task(name=cmd_full_name) + def task_wrapper(*command_args, **command_options): + stdout = TeeStringIO(logging.getLogger(cmd_full_name).info) + stderr = TeeStringIO(logging.getLogger(cmd_full_name).error) + + call_command( + cmd_name, + *command_args, + no_color=True, + stdout=stdout, + stderr=stderr, + **command_options, + ) + + stdout.seek(0) + stderr.seek(0) + return { + "stdout": str(stdout.read()), + "stderr": str(stderr.read()), + } + + return task_wrapper + + # Create the task with the current values + create_task(command_name, command_full_name) diff --git a/src/backend/core/tests/test_management_command_tasks.py b/src/backend/core/tests/test_management_command_tasks.py new file mode 100644 index 0000000..689153e --- /dev/null +++ b/src/backend/core/tests/test_management_command_tasks.py @@ -0,0 +1,36 @@ +"""Tests the core application loads the management command as tasks.""" + +from unittest.mock import patch + +from people.celery_app import app as celery_app + + +def test_fill_organization_metadata_as_task(settings): + """Check the fill_organization_metadata command is loaded as a task.""" + # Verify the command is configured to be loaded as a task + assert "fill_organization_metadata" in settings.MANAGEMENT_COMMAND_AS_TASK + + # The task should be registered in the format "app_name.command_name" + task_name = "core.fill_organization_metadata" + assert task_name in celery_app.tasks + + # Test that the task can be executed properly + with patch("core.apps.call_command") as mock_call_command: + # Get the registered task + task = celery_app.tasks[task_name] + + # Execute the task + result = task("arg1", "arg2", kwarg1="value1", kwarg2="value2") + + # Verify call_command was called with the correct command name + mock_call_command.assert_called_once() + assert mock_call_command.call_args[0][0] == "fill_organization_metadata" + assert mock_call_command.call_args[0][1] == "arg1" + assert mock_call_command.call_args[0][2] == "arg2" + assert mock_call_command.call_args[1]["kwarg1"] == "value1" + assert mock_call_command.call_args[1]["kwarg2"] == "value2" + + # Verify the task returns a dictionary with stdout and stderr + assert isinstance(result, dict) + assert "stdout" in result + assert "stderr" in result diff --git a/src/backend/core/utils/io.py b/src/backend/core/utils/io.py new file mode 100644 index 0000000..f55f3e2 --- /dev/null +++ b/src/backend/core/utils/io.py @@ -0,0 +1,49 @@ +"""Utility module providing I/O related classes and functions.""" + +from io import StringIO + + +class TeeStringIO: + """String IO implementation that captures output while preserving original logger output.""" + + def __init__(self, logger_output): + """Initialize a TeeStringIO instance. + + Args: + logger_output: A callable that will receive captured output. + """ + self.logger_output = logger_output + self.buffer = StringIO() + + def write(self, value): + """Write a string to both the logger and internal buffer. + + Args: + value: The string to write. + """ + self.logger_output(value.strip("\n")) + self.buffer.write(value) + + def read(self): + """Read the contents of the buffer. + + Returns: + The buffer contents as a string. + """ + return self.buffer.read() + + def seek(self, *args, **kwargs): + """Set the buffer's position. + + Args: + *args: Positional arguments passed to the underlying buffer. + **kwargs: Keyword arguments passed to the underlying buffer. + + Returns: + The new position in the buffer. + """ + return self.buffer.seek(*args, **kwargs) + + def flush(self): + """Flush the internal buffer.""" + self.buffer.flush() diff --git a/src/backend/people/settings.py b/src/backend/people/settings.py index 00d07b3..04fc83c 100755 --- a/src/backend/people/settings.py +++ b/src/backend/people/settings.py @@ -583,6 +583,14 @@ class Base(Configuration): environ_prefix=None, ) + MANAGEMENT_COMMAND_AS_TASK = [ + "fill_organization_metadata", + ] + values.ListValue( + default=[], + environ_name="MANAGEMENT_COMMAND_AS_TASK", + environ_prefix=None, + ) + # pylint: disable=invalid-name @property def ENVIRONMENT(self):