diff --git a/collectoss/application/cli/backend.py b/collectoss/application/cli/backend.py index 3526a3c2c..0e66397b0 100644 --- a/collectoss/application/cli/backend.py +++ b/collectoss/application/cli/backend.py @@ -339,6 +339,7 @@ def stop_processes(signal, logger, engine): cleanup_collection_status_and_rabbit(logger, engine) + def assign_orphan_repos_to_default_user(session): query = s.sql.text(""" SELECT repo_id FROM repo WHERE repo_id NOT IN (SELECT repo_id FROM operations.user_repos) diff --git a/collectoss/application/service_manager.py b/collectoss/application/service_manager.py index 2497f37ef..48fc389b3 100644 --- a/collectoss/application/service_manager.py +++ b/collectoss/application/service_manager.py @@ -146,7 +146,10 @@ def clear_rabbitmq_messages(connection_string): logger.info("Clearing all messages from celery queue in rabbitmq") from collectoss.tasks.init.celery_app import celery_app - celery_app.control.purge() + try: + celery_app.control.purge() + except Exception as e: + logger.warning(f"Failed to purge celery queue (this is usually benign if the queue doesn't exist yet): {e}") clear_all_message_queues(connection_string) #rabbitmq_purge_command = f"sudo rabbitmqctl purge_queue celery -p {virtual_host_string}" @@ -162,5 +165,8 @@ def clear_all_message_queues(connection_string): parsed = urlparse(connection_string) for q in queues: - curl_cmd = f"curl -i -u {parsed.username}:{parsed.password} -XDELETE http://localhost:15672/api/queues/{virtual_host_string}/{q}" - subprocess.call(curl_cmd.split(" "),stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Use the hostname from the connection string instead of localhost + # to ensure it works in both docker and non-docker environments + hostname = parsed.hostname or "localhost" + curl_cmd = f"curl -is -u {parsed.username}:{parsed.password} -XDELETE http://{hostname}:15672/api/queues/{virtual_host_string}/{q}" + subprocess.call(curl_cmd.split(" "), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) diff --git a/collectoss/tasks/init/celery_app.py b/collectoss/tasks/init/celery_app.py index 4b10af18a..a9392eed7 100644 --- a/collectoss/tasks/init/celery_app.py +++ b/collectoss/tasks/init/celery_app.py @@ -8,7 +8,6 @@ import inspect import celery from celery import Celery -from celery import current_app from celery.signals import after_setup_logger @@ -264,7 +263,7 @@ def setup_periodic_tasks(sender, **kwargs): def setup_loggers(*args,**kwargs): """Override Celery loggers with our own.""" - all_celery_tasks = list(current_app.tasks.keys()) + all_celery_tasks = list(celery_app.tasks.keys()) tasks = [task for task in all_celery_tasks if 'celery.' not in task] diff --git a/pyproject.toml b/pyproject.toml index 8788c0fff..865f1e0b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "Beaker==1.11.0", "boto3==1.17.57", "bs4==0.0.1", - "celery~=5.5", + "celery>=5.6.0", "click~=8.1", "coloredlogs==15.0", "emoji==1.2.0", @@ -54,7 +54,7 @@ dependencies = [ "python-crfsuite>=0.9.8", "python-dotenv>=1.2.1", "pyYaml", - "redis==4.3.3", + "redis>=4.5.3", "scikit-image==0.19.1", "scikit-learn==1.5.0", "requests~=2.32", diff --git a/uv.lock b/uv.lock index f6a8bae45..f1622b01c 100644 --- a/uv.lock +++ b/uv.lock @@ -255,7 +255,7 @@ wheels = [ [[package]] name = "celery" -version = "5.5.3" +version = "5.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "billiard" }, @@ -263,13 +263,15 @@ dependencies = [ { name = "click-didyoumean" }, { name = "click-plugins" }, { name = "click-repl" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "kombu" }, { name = "python-dateutil" }, + { name = "tzlocal" }, { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bb/7d/6c289f407d219ba36d8b384b42489ebdd0c84ce9c413875a8aae0c85f35b/celery-5.5.3.tar.gz", hash = "sha256:6c972ae7968c2b5281227f01c3a3f984037d21c5129d07bf3550cc2afc6b10a5", size = 1667144, upload-time = "2025-06-01T11:08:12.563Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/af/0dcccc7fdcdf170f9a1585e5e96b6fb0ba1749ef6be8c89a6202284759bd/celery-5.5.3-py3-none-any.whl", hash = "sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525", size = 438775, upload-time = "2025-06-01T11:08:09.94Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, ] [[package]] @@ -501,7 +503,7 @@ requires-dist = [ { name = "beaker", specifier = "==1.11.0" }, { name = "boto3", specifier = "==1.17.57" }, { name = "bs4", specifier = "==0.0.1" }, - { name = "celery", specifier = "~=5.5" }, + { name = "celery", specifier = ">=5.6.0" }, { name = "click", specifier = "~=8.1" }, { name = "coloredlogs", specifier = "==15.0" }, { name = "emoji", specifier = "==1.2.0" }, @@ -532,7 +534,7 @@ requires-dist = [ { name = "python-crfsuite", specifier = ">=0.9.8" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "pyyaml" }, - { name = "redis", specifier = "==4.3.3" }, + { name = "redis", specifier = ">=4.5.3" }, { name = "requests", specifier = "~=2.32" }, { name = "scikit-image", specifier = "==0.19.1" }, { name = "scikit-learn", specifier = "==1.5.0" }, @@ -717,18 +719,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/82/e5d2c1c67d19841e9edc74954c827444ae826978499bde3dfc1d007c8c11/deepmerge-2.0-py3-none-any.whl", hash = "sha256:6de9ce507115cff0bed95ff0ce9ecc31088ef50cbdf09bc90a09349a318b3d00", size = 13475, upload-time = "2024-08-30T05:31:48.659Z" }, ] -[[package]] -name = "deprecated" -version = "1.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wrapt" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744, upload-time = "2025-01-27T10:46:25.7Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998, upload-time = "2025-01-27T10:46:09.186Z" }, -] - [[package]] name = "dill" version = "0.4.0" @@ -1576,7 +1566,7 @@ wheels = [ [[package]] name = "kombu" -version = "5.5.4" +version = "5.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "amqp" }, @@ -1584,9 +1574,9 @@ dependencies = [ { name = "tzdata" }, { name = "vine" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992, upload-time = "2025-06-01T10:19:22.281Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034, upload-time = "2025-06-01T10:19:20.436Z" }, + { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, ] [[package]] @@ -2630,16 +2620,14 @@ wheels = [ [[package]] name = "redis" -version = "4.3.3" +version = "8.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "async-timeout" }, - { name = "deprecated" }, - { name = "packaging" }, + { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/37/d58ce17fed05620b3ebfe6dff2b94367347b2fd01491b1a38bd4e74d9eeb/redis-4.3.3.tar.gz", hash = "sha256:2f7a57cf4af15cd543c4394bcbe2b9148db2606a37edba755368836e3a1d053e", size = 4564562, upload-time = "2022-06-02T14:49:00.987Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/c3/928b290c2c0ca99ab96eea5b4ff8f30be8112b075301a7d3ba214a3c8c12/redis-8.0.1.tar.gz", hash = "sha256:afc5a7a2f5a084f5b1880dec548dd45be17db7e43c82a30d84f952aefb05cfb0", size = 5114170, upload-time = "2026-06-23T14:52:37.728Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/a6/022ed2fb6d4526c60e7b7e8830e6d5797cd2711719aa4392f99eda787706/redis-4.3.3-py3-none-any.whl", hash = "sha256:f57f8df5d238a8ecf92f499b6b21467bfee6c13d89953c27edf1e2bc673622e7", size = 244585, upload-time = "2022-06-02T14:48:58.255Z" }, + { url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" }, ] [[package]] @@ -3670,6 +3658,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, ] +[[package]] +name = "tzlocal" +version = "5.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/55/15e2340963d2bfedcc6042da3911438fd336f8ae96b65bdbe3a29766da0c/tzlocal-5.4.3.tar.gz", hash = "sha256:3a8c9bc18cf47e1dcde252ea0e6a72a6cde320a400b6ac6db1f1f8cccd553c00", size = 30873, upload-time = "2026-06-17T04:17:41.764Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/28/fc144409c71569e928585f8f3c629d80d1ca3ef40175e9222f01588f98c9/tzlocal-5.4.3-py3-none-any.whl", hash = "sha256:24ce97bb58e2a973f7640ec2553ab4e6f6d5a0d0d1aa9dc43bca21d89e1feb82", size = 18039, upload-time = "2026-06-17T04:17:40.027Z" }, +] + [[package]] name = "uc-micro-py" version = "1.0.3"