auto restart beat after 1h, #967

Dockerfile

@@ -70,6 +70,7 @@ RUN sed -i 's/^user www\-data\;$/user root\;/' /etc/nginx/nginx.conf
 COPY ./backend /app
 COPY ./docker_assets/run.sh /app
 COPY ./docker_assets/backend_start.py /app
+COPY ./docker_assets/beat_auto_spawn.sh /app
 
 COPY --from=node-builder ./frontend/dist /app/static
 

docker_assets/beat_auto_spawn.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# auto restart beat scheduler
+# https://github.com/celery/django-celery-beat/issues/894
+
+if [[ -n "$DJANGO_DEBUG" ]]; then
+  LOGLEVEL="DEBUG"
+else
+  LOGLEVEL="INFO"
+fi
+
+COMMAND="celery -A task beat --loglevel=$LOGLEVEL --scheduler django_celery_beat.schedulers:DatabaseScheduler"
+TIMEOUT=3600
+
+while true; do
+    echo "Starting process beat scheduler"
+
+    $COMMAND &
+    PID=$!
+
+    sleep $TIMEOUT
+
+    # Kill the process if still running
+    if kill -0 $PID 2>/dev/null; then
+        echo "Killing beat process after $TIMEOUT seconds"
+        kill $PID
+        # Wait a bit to allow graceful shutdown, then force kill if needed
+        sleep 10
+        kill -9 $PID 2>/dev/null
+    fi
+
+    echo "Restarting beat..."
+done

docker_assets/run.sh

@@ -3,6 +3,12 @@
 
 set -e
 
+if [[ -n "$DJANGO_DEBUG" ]]; then
+  LOGLEVEL="DEBUG"
+else
+  LOGLEVEL="INFO"
+fi
+
 # stop on pending manual migration
 python manage.py ta_stop_on_error
 
@@ -18,10 +24,11 @@ python manage.py ta_startup
 # start all tasks
 nginx &
 celery -A task.celery worker \
-    --loglevel=INFO \
+    --loglevel=$LOGLEVEL \
     --concurrency 4 \
     --max-tasks-per-child 5 \
     --max-memory-per-child 150000 &
-celery -A task beat --loglevel=INFO \
+
-    --scheduler django_celery_beat.schedulers:DatabaseScheduler &
+./beat_auto_spawn.sh &
+
 python backend_start.py