011 fix redis
This commit is contained in:
+1
-1
@@ -13,7 +13,7 @@ from pydantic_settings import (
|
|||||||
YamlConfigSettingsSource,
|
YamlConfigSettingsSource,
|
||||||
)
|
)
|
||||||
|
|
||||||
DEFAULT_CONFIG_FILE = "config.yaml"
|
DEFAULT_CONFIG_FILE = "/config.yaml"
|
||||||
TEST_CONFIG_FILE = "config.test.yaml"
|
TEST_CONFIG_FILE = "config.test.yaml"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+17
-4
@@ -19,6 +19,10 @@ class RedisClientProtocol(Protocol):
|
|||||||
async def delete(self, key: str) -> Any: ...
|
async def delete(self, key: str) -> Any: ...
|
||||||
|
|
||||||
|
|
||||||
|
class PriceCacheRepositoryError(RuntimeError):
|
||||||
|
"""Deterministic repository error raised on Redis operation failures."""
|
||||||
|
|
||||||
|
|
||||||
class PriceCache:
|
class PriceCache:
|
||||||
"""Repository for cached provider payloads."""
|
"""Repository for cached provider payloads."""
|
||||||
|
|
||||||
@@ -41,7 +45,10 @@ class PriceCache:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def get(self, key: str) -> Any | None:
|
async def get(self, key: str) -> Any | None:
|
||||||
|
try:
|
||||||
payload = await self._redis_client.get(key)
|
payload = await self._redis_client.get(key)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PriceCacheRepositoryError("Redis cache read failed.") from exc
|
||||||
if payload is None:
|
if payload is None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -57,28 +64,34 @@ class PriceCache:
|
|||||||
async def set(self, key: str, value: Any, ttl: int | None = None) -> None:
|
async def set(self, key: str, value: Any, ttl: int | None = None) -> None:
|
||||||
serialized_payload = _serialize(value)
|
serialized_payload = _serialize(value)
|
||||||
resolved_ttl = self._ttl_seconds if ttl is None else ttl
|
resolved_ttl = self._ttl_seconds if ttl is None else ttl
|
||||||
|
try:
|
||||||
await self._redis_client.set(key, serialized_payload, ex=resolved_ttl)
|
await self._redis_client.set(key, serialized_payload, ex=resolved_ttl)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PriceCacheRepositoryError("Redis cache write failed.") from exc
|
||||||
|
|
||||||
async def invalidate(self, key: str) -> None:
|
async def invalidate(self, key: str) -> None:
|
||||||
|
try:
|
||||||
await self._redis_client.delete(key)
|
await self._redis_client.delete(key)
|
||||||
|
except Exception as exc:
|
||||||
|
raise PriceCacheRepositoryError("Redis cache invalidate failed.") from exc
|
||||||
|
|
||||||
|
|
||||||
def _default_redis_client_factory(redis_dsn: str) -> RedisClientProtocol:
|
def _default_redis_client_factory(redis_dsn: str) -> RedisClientProtocol:
|
||||||
"""Create async Redis client from DSN."""
|
"""Create async Redis client from DSN."""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import redis.asyncio as redis_asyncio # type: ignore[import-not-found]
|
from aioredis import from_url as aioredis_from_url # type: ignore
|
||||||
except ModuleNotFoundError:
|
except ModuleNotFoundError:
|
||||||
try:
|
try:
|
||||||
from aioredis import from_url as aioredis_from_url # type: ignore
|
import redis.asyncio as redis_asyncio # type: ignore[import-not-found]
|
||||||
except ModuleNotFoundError as exc:
|
except ModuleNotFoundError as exc:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"No async Redis client implementation is available."
|
"No async Redis client implementation is available."
|
||||||
) from exc
|
) from exc
|
||||||
return aioredis_from_url(redis_dsn, encoding="utf-8", decode_responses=False)
|
|
||||||
|
|
||||||
return redis_asyncio.from_url(redis_dsn, encoding="utf-8", decode_responses=False)
|
return redis_asyncio.from_url(redis_dsn, encoding="utf-8", decode_responses=False)
|
||||||
|
|
||||||
|
return aioredis_from_url(redis_dsn, encoding="utf-8", decode_responses=False)
|
||||||
|
|
||||||
|
|
||||||
def _serialize(value: Any) -> str:
|
def _serialize(value: Any) -> str:
|
||||||
normalized = _normalize_for_json(value)
|
normalized = _normalize_for_json(value)
|
||||||
|
|||||||
+5
-2
@@ -1,18 +1,20 @@
|
|||||||
services:
|
services:
|
||||||
app:
|
app:
|
||||||
image: yusupal1ev/g2s-aggregator:0.0.0
|
image: yusupal1ev/g2s-aggregator:0.0.0
|
||||||
|
container_name: aggregator
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "8000:8000"
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
- redis
|
||||||
- signoz
|
- signoz
|
||||||
|
volumes:
|
||||||
|
- ./config.yaml:/config.yaml
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
|
container_name: redis
|
||||||
command: ["redis-server", "--save", "", "--appendonly", "no"]
|
command: ["redis-server", "--save", "", "--appendonly", "no"]
|
||||||
ports:
|
ports:
|
||||||
- "6379:6379"
|
- "6379:6379"
|
||||||
|
|
||||||
signoz:
|
signoz:
|
||||||
image: signoz/signoz:latest
|
image: signoz/signoz:latest
|
||||||
container_name: signoz
|
container_name: signoz
|
||||||
@@ -22,6 +24,7 @@ services:
|
|||||||
SIGNOZ_ALERTS_CONTACT_POINT_FILE: /etc/signoz/alerts/signoz_contact_point_telegram.yaml
|
SIGNOZ_ALERTS_CONTACT_POINT_FILE: /etc/signoz/alerts/signoz_contact_point_telegram.yaml
|
||||||
SIGNOZ_ALERTS_RULES_FILE: /etc/signoz/alerts/signoz_alert_rules.yaml
|
SIGNOZ_ALERTS_RULES_FILE: /etc/signoz/alerts/signoz_alert_rules.yaml
|
||||||
ports:
|
ports:
|
||||||
|
- "8080:8080"
|
||||||
- "3301:3301"
|
- "3301:3301"
|
||||||
- "4317:4317"
|
- "4317:4317"
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
Generated
+20
-1
@@ -1091,6 +1091,25 @@ files = [
|
|||||||
{file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"},
|
{file = "pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "redis"
|
||||||
|
version = "7.3.0"
|
||||||
|
description = "Python client for Redis database and key-value store"
|
||||||
|
optional = false
|
||||||
|
python-versions = ">=3.10"
|
||||||
|
files = [
|
||||||
|
{file = "redis-7.3.0-py3-none-any.whl", hash = "sha256:9d4fcb002a12a5e3c3fbe005d59c48a2cc231f87fbb2f6b70c2d89bb64fec364"},
|
||||||
|
{file = "redis-7.3.0.tar.gz", hash = "sha256:4d1b768aafcf41b01022410b3cc4f15a07d9b3d6fe0c66fc967da2c88e551034"},
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.extras]
|
||||||
|
circuit-breaker = ["pybreaker (>=1.4.0)"]
|
||||||
|
hiredis = ["hiredis (>=3.2.0)"]
|
||||||
|
jwt = ["pyjwt (>=2.9.0)"]
|
||||||
|
ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)"]
|
||||||
|
otel = ["opentelemetry-api (>=1.39.1)", "opentelemetry-exporter-otlp-proto-http (>=1.39.1)", "opentelemetry-sdk (>=1.39.1)"]
|
||||||
|
xxhash = ["xxhash (>=3.6.0,<3.7.0)"]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "requests"
|
name = "requests"
|
||||||
version = "2.32.5"
|
version = "2.32.5"
|
||||||
@@ -1573,4 +1592,4 @@ type = ["pytest-mypy"]
|
|||||||
[metadata]
|
[metadata]
|
||||||
lock-version = "2.0"
|
lock-version = "2.0"
|
||||||
python-versions = "^3.14"
|
python-versions = "^3.14"
|
||||||
content-hash = "cf3bc6f669ae82e4110ed85fa24db240e9a2ec4f6cc48de1f1a8724937abe889"
|
content-hash = "954a8ff26e6abe5321bb634e27f6f458037ec59947f24ac7385623e5575fb817"
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ opentelemetry-exporter-otlp = "^1.40.0"
|
|||||||
opentelemetry-instrumentation-fastapi = "^0.61b0"
|
opentelemetry-instrumentation-fastapi = "^0.61b0"
|
||||||
opentelemetry-instrumentation-httpx = "^0.61b0"
|
opentelemetry-instrumentation-httpx = "^0.61b0"
|
||||||
pytest = "^9.0.2"
|
pytest = "^9.0.2"
|
||||||
|
redis = "^7.3.0"
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
pythonpath = ["."]
|
pythonpath = ["."]
|
||||||
|
|||||||
+4
-3
@@ -1,7 +1,7 @@
|
|||||||
# Spec Tasks Index
|
# Spec Tasks Index
|
||||||
|
|
||||||
> ⚠️ This file is generated. Do not edit manually.
|
> ⚠️ This file is generated. Do not edit manually.
|
||||||
> Generated at (UTC): `2026-03-08T13:21:58+00:00`
|
> Generated at (UTC): `2026-03-08T19:06:27+00:00`
|
||||||
|
|
||||||
## Tasks
|
## Tasks
|
||||||
|
|
||||||
@@ -18,9 +18,10 @@
|
|||||||
| 008 | DONE | 2026-03-07 | Add SigNoz to Telegram alerting configuration | `spec/tasks/008_add_signoz_telegram_alerting.md` |
|
| 008 | DONE | 2026-03-07 | Add SigNoz to Telegram alerting configuration | `spec/tasks/008_add_signoz_telegram_alerting.md` |
|
||||||
| 009 | DONE | 2026-03-07 | Add local infrastructure stack | `spec/tasks/009_add_local_infra_stack.md` |
|
| 009 | DONE | 2026-03-07 | Add local infrastructure stack | `spec/tasks/009_add_local_infra_stack.md` |
|
||||||
| 010 | DONE | 2026-03-08 | Migrate to YAML-only configuration loading | `spec/tasks/010_migrate_to_yaml_only_configuration.md` |
|
| 010 | DONE | 2026-03-08 | Migrate to YAML-only configuration loading | `spec/tasks/010_migrate_to_yaml_only_configuration.md` |
|
||||||
|
| 011 | DONE | 2026-03-08 | Migrate Redis repository client to aioredis | `spec/tasks/011_migrate_redis_repository_to_aioredis.md` |
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
- Total: **11**
|
- Total: **12**
|
||||||
- TODO: **0**
|
- TODO: **0**
|
||||||
- DONE: **11**
|
- DONE: **12**
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
---
|
||||||
|
id: 011
|
||||||
|
title: Migrate Redis repository client to aioredis
|
||||||
|
status: DONE
|
||||||
|
created: 2026-03-08
|
||||||
|
---
|
||||||
|
|
||||||
|
## Context
|
||||||
|
Текущая реализация кеш-репозитория использует клиент `redis`. Требуется перейти на `aioredis` для асинхронного доступа к Redis без изменения бизнес-поведения приложения.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
Перевести слой Repository (`PriceCache`) и связанные точки инициализации/конфигурации на `aioredis`, сохранив существующий контракт репозитория и runtime-поведение кеша.
|
||||||
|
|
||||||
|
## Constraints
|
||||||
|
- Изменения только в слоях Repository и инициализации зависимостей, необходимых для подключения клиента.
|
||||||
|
- Не изменять бизнес-правила, Controller/API-контракты и оркестрацию Service.
|
||||||
|
- SQL/внешний IO вне Repository и Adapter не добавлять.
|
||||||
|
- Публичный интерфейс `PriceCache` (`get`, `set`, `invalidate`) должен остаться совместимым.
|
||||||
|
- Не изменять файлы в `spec/` кроме этой задачи и сгенерированного `spec/index.md`.
|
||||||
|
|
||||||
|
## Acceptance criteria
|
||||||
|
- В кодовой базе для Redis repository используется `aioredis` вместо `redis`.
|
||||||
|
- Инициализация Redis-клиента и операции `get/set/invalidate` работают асинхронно через `aioredis`.
|
||||||
|
- Поведение TTL и сериализации кеша не изменено относительно текущего контракта.
|
||||||
|
- Конфигурация подключения к Redis продолжает читаться из секции `repository` YAML-конфига.
|
||||||
|
- Ошибки подключения/операций Redis обрабатываются детерминированно в рамках текущего repository-контракта.
|
||||||
|
|
||||||
|
## Definition of Done
|
||||||
|
- [ ] Обновлены зависимости проекта для использования `aioredis`.
|
||||||
|
- [ ] Репозиторий кеша переведен на `aioredis` без изменения публичного интерфейса.
|
||||||
|
- [ ] Обновлены/добавлены тесты репозитория для нового клиента.
|
||||||
|
- [ ] Все релевантные тесты проходят.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
- Обновить `tests/repositories/cache/test_redis_cache.py` под использование `aioredis`.
|
||||||
|
- Добавить/обновить тесты на cache hit/miss, `set/get` roundtrip, `invalidate`, TTL expiration.
|
||||||
|
- Добавить негативный тест на ошибку Redis-клиента при операции чтения или записи.
|
||||||
|
|
||||||
|
## Commands
|
||||||
|
- `poetry run pytest tests/repositories/cache/test_redis_cache.py -q`
|
||||||
|
- `poetry run pytest tests/services/test_aggregator.py -q`
|
||||||
+69
-2
@@ -1,8 +1,16 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
|
|
||||||
|
import app.config as app_config
|
||||||
|
import pytest
|
||||||
from app.config import Settings
|
from app.config import Settings
|
||||||
from app.repositories.cache.redis_cache import PriceCache
|
from app.repositories.cache.redis_cache import (
|
||||||
|
PriceCache,
|
||||||
|
PriceCacheRepositoryError,
|
||||||
|
_default_redis_client_factory,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class InMemoryRedisClient:
|
class InMemoryRedisClient:
|
||||||
@@ -39,6 +47,28 @@ class InMemoryRedisClient:
|
|||||||
return self._storage[key][0]
|
return self._storage[key][0]
|
||||||
|
|
||||||
|
|
||||||
|
class FailingGetRedisClient:
|
||||||
|
async def get(self, key: str) -> bytes | None: # noqa: ARG002
|
||||||
|
raise RuntimeError("redis get failed")
|
||||||
|
|
||||||
|
async def set(self, key: str, value: str, ex: int | None = None) -> bool: # noqa: ARG002
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def delete(self, key: str) -> int: # noqa: ARG002
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
class FailingSetRedisClient:
|
||||||
|
async def get(self, key: str) -> bytes | None: # noqa: ARG002
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def set(self, key: str, value: str, ex: int | None = None) -> bool: # noqa: ARG002
|
||||||
|
raise RuntimeError("redis set failed")
|
||||||
|
|
||||||
|
async def delete(self, key: str) -> int: # noqa: ARG002
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
def test_cache_miss_returns_none() -> None:
|
def test_cache_miss_returns_none() -> None:
|
||||||
cache = PriceCache(InMemoryRedisClient(), ttl_seconds=120)
|
cache = PriceCache(InMemoryRedisClient(), ttl_seconds=120)
|
||||||
|
|
||||||
@@ -130,7 +160,7 @@ repository:
|
|||||||
""".strip(),
|
""".strip(),
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
monkeypatch.setenv("G2S_CONFIG_FILE", str(config_file))
|
monkeypatch.setattr(app_config, "_resolve_runtime_config_file", lambda: str(config_file))
|
||||||
|
|
||||||
settings = Settings()
|
settings = Settings()
|
||||||
redis_client = InMemoryRedisClient()
|
redis_client = InMemoryRedisClient()
|
||||||
@@ -153,3 +183,40 @@ repository:
|
|||||||
assert seen_dsns == ["redis://redis.internal:6380/5"]
|
assert seen_dsns == ["redis://redis.internal:6380/5"]
|
||||||
assert redis_client.last_set == ("config-key", '{"ok":true}', 123)
|
assert redis_client.last_set == ("config-key", '{"ok":true}', 123)
|
||||||
assert result == {"ok": True}
|
assert result == {"ok": True}
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_client_factory_uses_aioredis_from_url(monkeypatch) -> None:
|
||||||
|
seen: dict[str, object] = {}
|
||||||
|
expected_client = object()
|
||||||
|
|
||||||
|
def fake_from_url(redis_dsn: str, *, encoding: str, decode_responses: bool) -> object:
|
||||||
|
seen["redis_dsn"] = redis_dsn
|
||||||
|
seen["encoding"] = encoding
|
||||||
|
seen["decode_responses"] = decode_responses
|
||||||
|
return expected_client
|
||||||
|
|
||||||
|
fake_aioredis_module = types.SimpleNamespace(from_url=fake_from_url)
|
||||||
|
monkeypatch.setitem(sys.modules, "aioredis", fake_aioredis_module)
|
||||||
|
|
||||||
|
actual_client = _default_redis_client_factory("redis://cache.internal:6381/9")
|
||||||
|
|
||||||
|
assert actual_client is expected_client
|
||||||
|
assert seen == {
|
||||||
|
"redis_dsn": "redis://cache.internal:6381/9",
|
||||||
|
"encoding": "utf-8",
|
||||||
|
"decode_responses": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_raises_deterministic_repository_error_on_redis_failure() -> None:
|
||||||
|
cache = PriceCache(FailingGetRedisClient(), ttl_seconds=10)
|
||||||
|
|
||||||
|
with pytest.raises(PriceCacheRepositoryError, match="Redis cache read failed."):
|
||||||
|
asyncio.run(cache.get("key"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_set_raises_deterministic_repository_error_on_redis_failure() -> None:
|
||||||
|
cache = PriceCache(FailingSetRedisClient(), ttl_seconds=10)
|
||||||
|
|
||||||
|
with pytest.raises(PriceCacheRepositoryError, match="Redis cache write failed."):
|
||||||
|
asyncio.run(cache.set("key", {"v": 1}))
|
||||||
|
|||||||
Reference in New Issue
Block a user