From 35ea29b184ca4cf5b96a27429dfc62521873a2d3 Mon Sep 17 00:00:00 2001 From: Jan Kessler Date: Wed, 2 Apr 2025 21:50:00 +0200 Subject: [PATCH 1/3] prepare websocket redis sentinel code for upcoming native support of sentinel in python-socketio --- backend/open_webui/socket/main.py | 14 +----- backend/open_webui/utils/redis.py | 73 ++++--------------------------- 2 files changed, 11 insertions(+), 76 deletions(-) diff --git a/backend/open_webui/socket/main.py b/backend/open_webui/socket/main.py index 83dd74fff..c1ce42c79 100644 --- a/backend/open_webui/socket/main.py +++ b/backend/open_webui/socket/main.py @@ -9,9 +9,8 @@ from open_webui.models.users import Users, UserNameResponse from open_webui.models.channels import Channels from open_webui.models.chats import Chats from open_webui.utils.redis import ( - parse_redis_sentinel_url, get_sentinels_from_env, - AsyncRedisSentinelManager, + get_sentinel_url_from_env, ) from open_webui.env import ( @@ -38,16 +37,7 @@ log.setLevel(SRC_LOG_LEVELS["SOCKET"]) if WEBSOCKET_MANAGER == "redis": if WEBSOCKET_SENTINEL_HOSTS: - redis_config = parse_redis_sentinel_url(WEBSOCKET_REDIS_URL) - mgr = AsyncRedisSentinelManager( - WEBSOCKET_SENTINEL_HOSTS.split(","), - sentinel_port=int(WEBSOCKET_SENTINEL_PORT), - redis_port=redis_config["port"], - service=redis_config["service"], - db=redis_config["db"], - username=redis_config["username"], - password=redis_config["password"], - ) + mgr = socketio.AsyncRedisManager(get_sentinel_url_from_env(WEBSOCKET_REDIS_URL, WEBSOCKET_SENTINEL_HOSTS, WEBSOCKET_SENTINEL_PORT)) else: mgr = socketio.AsyncRedisManager(WEBSOCKET_REDIS_URL) sio = socketio.AsyncServer( diff --git a/backend/open_webui/utils/redis.py b/backend/open_webui/utils/redis.py index baccb16ad..715ac0d9b 100644 --- a/backend/open_webui/utils/redis.py +++ b/backend/open_webui/utils/redis.py @@ -4,7 +4,7 @@ from redis import asyncio as aioredis from urllib.parse import urlparse -def parse_redis_sentinel_url(redis_url): +def parse_redis_service_url(redis_url): parsed_url = urlparse(redis_url) if parsed_url.scheme != "redis": raise ValueError("Invalid Redis URL scheme. Must be 'redis'.") @@ -20,7 +20,7 @@ def parse_redis_sentinel_url(redis_url): def get_redis_connection(redis_url, redis_sentinels, decode_responses=True): if redis_sentinels: - redis_config = parse_redis_sentinel_url(redis_url) + redis_config = parse_redis_service_url(redis_url) sentinel = redis.sentinel.Sentinel( redis_sentinels, port=redis_config["port"], @@ -45,65 +45,10 @@ def get_sentinels_from_env(sentinel_hosts_env, sentinel_port_env): return [] -class AsyncRedisSentinelManager(socketio.AsyncRedisManager): - def __init__( - self, - sentinel_hosts, - sentinel_port=26379, - redis_port=6379, - service="mymaster", - db=0, - username=None, - password=None, - channel="socketio", - write_only=False, - logger=None, - redis_options=None, - ): - """ - Initialize the Redis Sentinel Manager. - This implementation mostly replicates the __init__ of AsyncRedisManager and - overrides _redis_connect() with a version that uses Redis Sentinel - - :param sentinel_hosts: List of Sentinel hosts - :param sentinel_port: Sentinel Port - :param redis_port: Redis Port (currently unsupported by aioredis!) - :param service: Master service name in Sentinel - :param db: Redis database to use - :param username: Redis username (if any) (currently unsupported by aioredis!) - :param password: Redis password (if any) - :param channel: The channel name on which the server sends and receives - notifications. Must be the same in all the servers. - :param write_only: If set to ``True``, only initialize to emit events. The - default of ``False`` initializes the class for emitting - and receiving. - :param redis_options: additional keyword arguments to be passed to - ``aioredis.from_url()``. - """ - self._sentinels = [(host, sentinel_port) for host in sentinel_hosts] - self._redis_port = redis_port - self._service = service - self._db = db - self._username = username - self._password = password - self._channel = channel - self.redis_options = redis_options or {} - - # connect and call grandparent constructor - self._redis_connect() - super(socketio.AsyncRedisManager, self).__init__( - channel=channel, write_only=write_only, logger=logger - ) - - def _redis_connect(self): - """Establish connections to Redis through Sentinel.""" - sentinel = aioredis.sentinel.Sentinel( - self._sentinels, - port=self._redis_port, - db=self._db, - password=self._password, - **self.redis_options, - ) - - self.redis = sentinel.master_for(self._service) - self.pubsub = self.redis.pubsub(ignore_subscribe_messages=True) +def get_sentinel_url_from_env(redis_url, sentinel_hosts_env, sentinel_port_env): + redis_config = parse_redis_service_url(redis_url) + username = redis_config["username"] or "" + password = redis_config["password"] or "" + auth_part = f"{username}:{password}" + hosts_part = ",".join(f"{host}:{sentinel_port_env}" for host in sentinel_hosts_env.split(",")) + return f"redis+sentinel://{auth_part}@{hosts_part}/{redis_config['db']}/{redis_config['service']}" From 257ca454569f1c4bd936a039c751f170ab3c6c38 Mon Sep 17 00:00:00 2001 From: Jan Kessler Date: Thu, 3 Apr 2025 08:24:24 +0200 Subject: [PATCH 2/3] leave out @ in redis+sentine url when no username/password is provided --- backend/open_webui/utils/redis.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/open_webui/utils/redis.py b/backend/open_webui/utils/redis.py index 715ac0d9b..24d3eefc7 100644 --- a/backend/open_webui/utils/redis.py +++ b/backend/open_webui/utils/redis.py @@ -49,6 +49,8 @@ def get_sentinel_url_from_env(redis_url, sentinel_hosts_env, sentinel_port_env): redis_config = parse_redis_service_url(redis_url) username = redis_config["username"] or "" password = redis_config["password"] or "" - auth_part = f"{username}:{password}" + auth_part = "" + if username or password: + auth_part = f"{username}:{password}@" hosts_part = ",".join(f"{host}:{sentinel_port_env}" for host in sentinel_hosts_env.split(",")) - return f"redis+sentinel://{auth_part}@{hosts_part}/{redis_config['db']}/{redis_config['service']}" + return f"redis+sentinel://{auth_part}{hosts_part}/{redis_config['db']}/{redis_config['service']}" From c6755f9151fe10295e25bab3212f4756ac675458 Mon Sep 17 00:00:00 2001 From: Jan Kessler Date: Sat, 12 Apr 2025 18:48:07 +0200 Subject: [PATCH 3/3] bump python-socketio to 5.13.0 (to support Redis Sentinel natively) --- backend/requirements.txt | 2 +- pyproject.toml | 2 +- uv.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index dd7c85932..02e0babae 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -3,7 +3,7 @@ uvicorn[standard]==0.34.0 pydantic==2.10.6 python-multipart==0.0.20 -python-socketio==5.11.3 +python-socketio==5.13.0 python-jose==3.4.0 passlib[bcrypt]==1.7.4 diff --git a/pyproject.toml b/pyproject.toml index 2e8537a77..5be1baf64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "pydantic==2.10.6", "python-multipart==0.0.18", - "python-socketio==5.11.3", + "python-socketio==5.13.0", "python-jose==3.4.0", "passlib[bcrypt]==1.7.4", diff --git a/uv.lock b/uv.lock index ab2e57ae4..557849959 100644 --- a/uv.lock +++ b/uv.lock @@ -2966,7 +2966,7 @@ requires-dist = [ { name = "python-jose", specifier = "==3.3.0" }, { name = "python-multipart", specifier = "==0.0.18" }, { name = "python-pptx", specifier = "==1.0.0" }, - { name = "python-socketio", specifier = "==5.11.3" }, + { name = "python-socketio", specifier = "==5.13.0" }, { name = "pytube", specifier = "==15.0.0" }, { name = "pyxlsb", specifier = "==1.0.10" }, { name = "qdrant-client", specifier = "~=1.12.0" }, @@ -4073,15 +4073,15 @@ wheels = [ [[package]] name = "python-socketio" -version = "5.11.3" +version = "5.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bidict" }, { name = "python-engineio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1e/74/b1e8787cea757e1f533a7878e94f929679ef7e07a2aaf44de6b71065b1f2/python_socketio-5.11.3.tar.gz", hash = "sha256:194af8cdbb7b0768c2e807ba76c7abc288eb5bb85559b7cddee51a6bc7a65737", size = 117702 } +sdist = { url = "https://files.pythonhosted.org/packages/21/1a/396d50ccf06ee539fa758ce5623b59a9cb27637fc4b2dc07ed08bf495e77/python_socketio-5.13.0.tar.gz", hash = "sha256:ac4e19a0302ae812e23b712ec8b6427ca0521f7c582d6abb096e36e24a263029", size = 121125 } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/59/5ee858d5736594d75385b9a8c0f65af6eca5da2b359ed3fb6a7486526399/python_socketio-5.11.3-py3-none-any.whl", hash = "sha256:2a923a831ff70664b7c502df093c423eb6aa93c1ce68b8319e840227a26d8b69", size = 76180 }, + { url = "https://files.pythonhosted.org/packages/3c/32/b4fb8585d1be0f68bde7e110dffbcf354915f77ad8c778563f0ad9655c02/python_socketio-5.13.0-py3-none-any.whl", hash = "sha256:51f68d6499f2df8524668c24bcec13ba1414117cfb3a90115c559b601ab10caf", size = 77800 }, ] [[package]]