commit cddcc16d65644406a3e9432b5a84ef4fcaef8063 Author: mish Date: Mon Apr 27 22:47:36 2026 +0300 Add EvoSync v3 implementation plan diff --git a/EvoSync-v3-%E2%80%94-Implementation-Plan.md b/EvoSync-v3-%E2%80%94-Implementation-Plan.md new file mode 100644 index 0000000..51dc296 --- /dev/null +++ b/EvoSync-v3-%E2%80%94-Implementation-Plan.md @@ -0,0 +1,285 @@ +# EvoSync v3 — Implementation Plan + +## Context +The previous web app (Python/FastAPI v1.9 and a Node.js rewrite) has been deleted. The repo is in a clean state. The product is a **sync platform** (Evotor POS → VK Market), rebuilt from scratch with reliability, testability, and maintainability as primary goals. Admin panel is secondary. + +--- + +## Stack +- **Backend**: FastAPI 0.115 + Uvicorn (Python 3.12) +- **Task queue**: Celery 5.4 + Redis 7 (separate worker + beat containers) +- **Database**: MariaDB 11.4 + SQLAlchemy 2.0 (Mapped/DeclarativeBase) + Alembic +- **Templates**: Jinja2 (SSR, Bootstrap 5 CDN, no build step) +- **Monitoring**: Celery Flower (separate container, basic auth) +- **Testing**: pytest + pytest-asyncio + factory-boy, SQLite in-memory + +--- + +## Architecture + +``` +docker-compose: + web → FastAPI + uvicorn (port 8080) + worker → Celery worker (queues: sync, health, default) + beat → Celery beat (PersistentScheduler, single process) + flower → Flower UI (port 5555, basic auth) + db → MariaDB 11.4 + redis → Redis 7 alpine + +Single Dockerfile.web image — command: key differentiates services. +``` + +**Key architectural principle:** Sync logic is split across three layers: +1. `api_clients/` — pure HTTP, no ORM +2. `sync_core/` — pure business logic, no HTTP, no ORM → fully unit-testable +3. `tasks/` — orchestration: wires DB + clients + core, writes job history + +--- + +## Directory Structure + +``` +web/ +├── main.py # FastAPI app factory, router registration +├── config.py # pydantic-settings Settings +├── database.py # SQLAlchemy engine, SessionLocal, Base +├── auth.py # hash_password, verify_password, get_current_user +├── schemas.py # Pydantic form validation +├── templates_env.py # Jinja2Templates singleton +│ +├── models/ +│ ├── user.py +│ ├── evotor.py +│ ├── vk.py +│ ├── sync.py # SyncConfig, SyncFilter +│ ├── cache.py # CachedStore, CachedGroup, CachedProduct +│ └── history.py # SyncJob, HealthCheckLog +│ +├── api_clients/ +│ ├── evotor.py # fetch_stores/groups/products, token_refresh +│ └── vk.py # get/create/edit/delete products, albums, upload_photo +│ +├── sync_core/ +│ ├── filters.py # is_group_included, get_included_store_ids (pure) +│ ├── transform.py # calc_price, normalize_name, is_weight_measure (pure) +│ ├── differ.py # diff_products → SyncDiff dataclass (pure) +│ └── engine.py # sync_user(SyncContext) → SyncResult (pure, no I/O) +│ +├── tasks/ +│ ├── celery_app.py # Celery factory + beat_schedule +│ ├── sync.py # run_sync_all, run_sync_for_user +│ ├── catalog.py # refresh_all_catalogs, refresh_catalog_for_user +│ └── health.py # check_all_connections, prune_old_logs +│ +├── routes/ +│ ├── auth.py # /register /login /logout /confirm-email +│ ├── profile.py # /profile /profile/edit /password /delete +│ ├── reset.py # /forgot-password /reset-password +│ ├── connections.py # /connections +│ ├── evotor.py # /evotor /evotor/callback /evotor/token /evotor/disconnect +│ ├── vk.py # /vk /vk/token /vk/disconnect +│ ├── sync.py # /sync /sync/toggle /sync/confirm /sync/run-now +│ ├── catalog.py # /catalog /catalog/groups /catalog/products +│ └── admin.py # /admin/* (user list, job history, manual triggers) +│ +├── templates/ # Jinja2 .html files (Bootstrap 5) +│ ├── base.html +│ ├── admin/ +│ └── [21 page templates] +│ +├── static/style.css +│ +└── migrations/ + ├── env.py + └── versions/0001_initial.py + +tests/ +├── conftest.py # SQLite in-memory engine, db_session (rollback per test), sample_user +├── unit/ +│ ├── test_filters.py +│ ├── test_transform.py +│ └── test_differ.py +├── integration/ +│ ├── test_sync_engine.py +│ └── test_catalog_refresh.py +└── web/ + ├── test_auth_routes.py + └── test_evotor_webhook.py +``` + +--- + +## Database Schema + +### `users` +`id, first_name, last_name, email (UNIQUE), phone (UNIQUE), password_hash, is_email_confirmed, email_confirm_token, password_reset_token, password_reset_expires, is_admin, created_at, updated_at` + +### `evotor_connections` +`id, user_id (FK nullable), evotor_user_id (UNIQUE), access_token, refresh_token, token_expires_at, connection_type ENUM('webhook','manual'), is_online, last_checked_at, connected_at, updated_at` + +`user_id` is nullable — the webhook creates the row before a user claims it. + +### `vk_connections` +`id, user_id (FK UNIQUE), access_token, group_id, group_name, group_screen_name, is_online, last_checked_at, connected_at, updated_at` + +### `sync_configs` +`id, user_id (FK UNIQUE), is_enabled, confirmed_at, last_run_at (denorm), created_at, updated_at` + +### `sync_filters` +`id, sync_config_id (FK), entity_type ENUM('store','group','product'), entity_id, entity_name, filter_mode ENUM('include','exclude'), parent_entity_id, created_at` +`UNIQUE(sync_config_id, entity_type, entity_id)` + +### `cached_stores` +`id, user_id (FK), evotor_id, name, address, fetched_at` + +### `cached_groups` +`id, user_id (FK), evotor_id, store_evotor_id, name, fetched_at` + +### `cached_products` +`id, user_id (FK), evotor_id, store_evotor_id, group_evotor_id, name, price, cost_price, quantity, measure_name, article_number, description, allow_to_sell, product_type, is_excisable, is_age_limited, fetched_at, synced_at` + +### `sync_jobs` +`id, user_id (FK INDEX), triggered_by ENUM('schedule','manual','webhook'), status ENUM('pending','running','success','failed','partial'), started_at, finished_at, items_created, items_updated, items_deleted, items_skipped, error_count, error_detail (TEXT JSON), celery_task_id` + +### `health_check_logs` +`id, user_id (FK INDEX), service ENUM('evotor','vk'), event ENUM('online','offline','token_refreshed','token_refresh_failed'), detail, checked_at` +Pruned after 30 days by nightly beat task. + +--- + +## Celery Tasks & Schedule + +| Task | Queue | Schedule | Retry | +|------|-------|----------|-------| +| `sync.run_sync_all` | sync | every hour :00 | — | +| `sync.run_sync_for_user(user_id)` | sync | spawned by above | 3x, 60s delay | +| `catalog.refresh_all_catalogs` | default | every hour :30 | — | +| `catalog.refresh_catalog_for_user(user_id)` | default | spawned by above | 2x, 120s delay | +| `health.check_all_connections` | health | every 10 min | — | +| `health.prune_old_logs` | default | daily 03:00 | — | + +`run_sync_for_user` acquires a Redis lock `sync:user:{id}` (5 min TTL) to prevent overlap between scheduled and manual triggers. + +`task_acks_late=True` + `worker_prefetch_multiplier=1` — re-queue on worker crash, prevent task starvation. + +--- + +## SyncContext / SyncResult boundary + +```python +# sync_core/engine.py — zero imports from web.*, httpx, or sqlalchemy +@dataclass +class SyncContext: + user_id: int + evo_products: list[EvoProduct] # plain dataclasses + vk_products: list[VkProduct] + vk_albums: list[VkAlbum] + filters: list[FilterRule] + photo_path: str + +@dataclass +class SyncResult: + created: list[str] # evo_ids to create + updated: list[str] # evo_ids to update + deleted: list[int] # vk_ids to delete + skipped: list[str] + errors: list[SyncError] +``` + +`sync_user(ctx)` returns intent only — no I/O. The task layer (`tasks/sync.py`) executes VK API calls and writes the `SyncJob` row. This makes unit tests trivial: pass dataclasses in, assert on the result, no mocks needed. + +--- + +## Filter semantics (documented in `sync_core/filters.py`) +- No filters of a type → include everything +- Only exclude filters → include all except listed +- Any include filter → include only listed, exclude rest + +--- + +## Dockerfile.web +Single image for all services. `command:` in docker-compose selects role (web / worker / beat / flower). + +```dockerfile +FROM python:3.12-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --upgrade pip && pip install -r requirements.txt +COPY . . +CMD ["uvicorn", "web.main:app", "--host", "0.0.0.0", "--port", "8000"] +``` + +--- + +## docker-compose services +- `db`: mariadb:11.4, healthcheck, named volume +- `redis`: redis:7-alpine, `--save 60 1`, named volume, healthcheck +- `web`: port 8080→8000, `alembic upgrade head && uvicorn ...`, depends on db+redis healthy +- `worker`: `celery -A web.tasks.celery_app worker --loglevel=info --concurrency=2 --queues=default,sync,health` +- `beat`: `celery -A web.tasks.celery_app beat --scheduler celery.beat:PersistentScheduler --schedule /tmp/celerybeat-schedule` +- `flower`: port 5555, basic auth via `FLOWER_BASIC_AUTH` env var + +--- + +## requirements.txt +``` +fastapi==0.115.5 +uvicorn[standard]==0.32.1 +python-multipart==0.0.12 +jinja2==3.1.4 +sqlalchemy==2.0.36 +alembic==1.14.0 +pymysql==1.1.1 +cryptography>=44.0.0 +passlib[bcrypt]==1.7.4 +bcrypt==4.2.1 +pydantic-settings==2.6.1 +httpx==0.28.1 +celery[redis]==5.4.0 +redis==5.2.1 +flower==2.0.1 +python-json-logger==3.2.1 +pytest==8.3.4 +pytest-asyncio==0.24.0 +pytest-cov==6.0.0 +factory-boy==3.3.1 +``` + +--- + +## Test patterns + +**conftest.py:** +- SQLite in-memory engine (session-scoped) +- `db_session`: connection + transaction + rollback per test (clean slate, no truncation needed) +- `sample_user`: confirmed user pre-inserted +- Route tests: `httpx.AsyncClient(transport=ASGITransport(app=app))` + +**Unit tests** (`sync_core/`): zero patches — pass plain dataclasses, assert on SyncResult. + +**Integration tests**: mock only the HTTP clients (`api_clients/`), use real DB session. + +--- + +## Implementation sequence +1. **Foundation** — config, database, models, Alembic migration, Dockerfile.web, docker-compose.yml, requirements.txt +2. **Auth** — user model, routes, templates (register/login/confirm/reset) +3. **Connections** — evotor + vk models, routes, templates, webhook endpoint +4. **API clients** — `api_clients/evotor.py` + `api_clients/vk.py` with unit tests for response parsing +5. **Sync core** — filters → transform → differ → engine, all fully unit-tested before wiring +6. **Celery** — celery_app.py, beat schedule, health tasks +7. **Catalog** — catalog tasks + routes + templates +8. **Sync tasks** — tasks/sync.py + SyncJob writes + sync routes + job history UI +9. **Admin panel** — /admin routes + templates +10. **Integration tests** — written alongside steps 6–8 + +--- + +## Verification checklist +- `docker compose up` → all 6 services healthy +- `pytest tests/unit/` → 100% pass, no I/O, no patches +- `pytest tests/` → coverage ≥ 60% +- Flower at :5555 shows beat schedule + worker connected +- E2E: register → connect Evotor (webhook) + VK → enable sync → manual trigger → `sync_jobs` row with `status=success`