usable celery dev setup

commit 5438243b4874879196294a4072f245ebf22afd5d
Author: eneller <erikneller@gmx.de>
Date:   Sat Apr 5 00:12:55 2025 +0200

    minor celery

commit 3a03c498a7e6acff1afe350ce1ae196375d19029
Author: eneller <erikneller@gmx.de>
Date:   Fri Apr 4 23:40:23 2025 +0200

    begin celery

docker-compose.yml

@@ -0,0 +1,6 @@
+name: task-queue
+services:
+    redis:
+        ports:
+            - 6379:6379
+        image: redis

pyproject.toml

@@ -5,12 +5,13 @@ description = "Web Interface to epub2go.py"
 readme = "README.md"
 requires-python = ">=3.12"
 dependencies = [
-    "celery>=5.4.0",
+    "celery[redis]>=5.4.0",
     "django>=5.1.6",
     "epub2go",
     "gunicorn>=23.0.0",
     "python-dotenv>=1.0.1",
     "pywatchman>=2.0.0",
+    "redis>=5.2.1",
 ]
 
 [tool.uv.sources]

src/core/views.py

@@ -1,17 +1,16 @@
 from django.shortcuts import render
 from django.http import HttpRequest, HttpResponse, FileResponse, HttpResponseBadRequest
-from django.conf import settings
-from celery import shared_task
 
-from epub2go.convert import get_all_books, Book, GBConvert, allbooks_url
+from epub2go.convert import get_all_books, Book, allbooks_url
 
 import os
 from urllib.parse import urlparse
 import logging
 
+from epub2go_web.tasks import getEpub
+
 logger = logging.getLogger(__name__) #TODO configure logging
 
-converter = GBConvert(downloaddir=settings.MEDIA_ROOT)
 books = sorted(get_all_books(), key= lambda b: b.title)# TODO get from pickle
 gbnetloc = urlparse(allbooks_url).netloc
 
@@ -28,7 +27,8 @@ def index(request: HttpRequest):
     if targetParam:
         if validateUrl(targetParam):
             # download file
-            fpath = getEpub(targetParam)
+            result = getEpub.delay(targetParam)
+            fpath = result.get(timeout=60)
             fname = os.path.basename(fpath)
             file = open(fpath, 'rb')
             response = FileResponse(file)
@@ -46,9 +46,3 @@ def validateUrl(param)->bool :
     if(netloc == gbnetloc): return True
 
     return False
-
-# TODO make this async and show some indication of progress/loading
-#@shared_task
-def getEpub(book_url):
-    # TODO check for existing file and age
-    return converter.download(book_url)

src/epub2go_web/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

src/epub2go_web/celery.py

@@ -0,0 +1,23 @@
+# Start Celery workers from src/ using `celery -A epub2go_web worker --loglevel=INFO`
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'epub2go_web.settings')
+
+app = Celery('epub2go_web')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True, ignore_result=True)
+def debug_task(self):
+    print(f'Request: {self.request!r}')

src/epub2go_web/settings.py

@@ -69,6 +69,19 @@ TEMPLATES = [
 
 WSGI_APPLICATION = "epub2go_web.wsgi.application"
 
+# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": BASE_DIR / "db.sqlite3",
+    }
+}
+
+# Celery settings
+CELERY_BROKER_URL = "redis://localhost:6379/0"
+CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
+CELERY_TASK_ALWAYS_EAGER = True
+
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/5.1/howto/static-files/
 

src/epub2go_web/tasks.py

@@ -0,0 +1,11 @@
+from celery import shared_task
+from django.conf import settings
+
+from epub2go.convert import GBConvert
+
+converter = GBConvert(downloaddir=settings.MEDIA_ROOT)
+
+@shared_task
+def getEpub(book_url):
+    # TODO check for existing file and age
+    return converter.download(book_url)

uv.lock

@@ -63,6 +63,11 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/1a/98/d4eefe492ef810f304203cfff0f60554fab95514ec51b701a4529b66fa92/celery-5.5.0-py3-none-any.whl", hash = "sha256:f4170c6e5952281318448a899d9e9a15b9cbd007e002091766900dc8f71b9394", size = 438396 },
 ]
 
+[package.optional-dependencies]
+redis = [
+    { name = "redis" },
+]
+
 [[package]]
 name = "certifi"
 version = "2025.1.31"
@@ -197,22 +202,24 @@ name = "epub2go-py-web"
 version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
-    { name = "celery" },
+    { name = "celery", extra = ["redis"] },
     { name = "django" },
     { name = "epub2go" },
     { name = "gunicorn" },
     { name = "python-dotenv" },
     { name = "pywatchman" },
+    { name = "redis" },
 ]
 
 [package.metadata]
 requires-dist = [
-    { name = "celery", specifier = ">=5.4.0" },
+    { name = "celery", extras = ["redis"], specifier = ">=5.4.0" },
     { name = "django", specifier = ">=5.1.6" },
     { name = "epub2go", git = "https://github.com/eneller/epub2go.py" },
     { name = "gunicorn", specifier = ">=23.0.0" },
     { name = "python-dotenv", specifier = ">=1.0.1" },
     { name = "pywatchman", specifier = ">=2.0.0" },
+    { name = "redis", specifier = ">=5.2.1" },
 ]
 
 [[package]]
@@ -310,6 +317,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/51/7f/5d68d803489770cffa5d2b44be99b978c866f8a4d8e835f9da850415ed8a/pywatchman-2.0.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:51c2b4c72bea6b9fd90caf20759f5bc47febf0fd27bf2f247b87c66e2f6bab02", size = 52557 },
 ]
 
+[[package]]
+name = "redis"
+version = "5.2.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/47/da/d283a37303a995cd36f8b92db85135153dc4f7a8e4441aa827721b442cfb/redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f", size = 4608355 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3c/5f/fa26b9b2672cbe30e07d9a5bdf39cf16e3b80b42916757c5f92bca88e4ba/redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4", size = 261502 },
+]
+
 [[package]]
 name = "requests"
 version = "2.32.3"