From a423e6847b271d554db6e995934e2bc291145c23 Mon Sep 17 00:00:00 2001 From: mish Date: Mon, 27 Apr 2026 22:55:40 +0300 Subject: [PATCH] Trim plan to Step 1: environment setup only --- EvoSync-v3-%E2%80%94-Implementation-Plan.md | 285 ------------------ ...3-%E2%80%94-Step-1%3A-Environment-Setup.md | 116 +++++++ 2 files changed, 116 insertions(+), 285 deletions(-) delete mode 100644 EvoSync-v3-%E2%80%94-Implementation-Plan.md create mode 100644 EvoSync-v3-%E2%80%94-Step-1%3A-Environment-Setup.md diff --git a/EvoSync-v3-%E2%80%94-Implementation-Plan.md b/EvoSync-v3-%E2%80%94-Implementation-Plan.md deleted file mode 100644 index 51dc296..0000000 --- a/EvoSync-v3-%E2%80%94-Implementation-Plan.md +++ /dev/null @@ -1,285 +0,0 @@ -# EvoSync v3 — Implementation Plan - -## Context -The previous web app (Python/FastAPI v1.9 and a Node.js rewrite) has been deleted. The repo is in a clean state. The product is a **sync platform** (Evotor POS → VK Market), rebuilt from scratch with reliability, testability, and maintainability as primary goals. Admin panel is secondary. - ---- - -## Stack -- **Backend**: FastAPI 0.115 + Uvicorn (Python 3.12) -- **Task queue**: Celery 5.4 + Redis 7 (separate worker + beat containers) -- **Database**: MariaDB 11.4 + SQLAlchemy 2.0 (Mapped/DeclarativeBase) + Alembic -- **Templates**: Jinja2 (SSR, Bootstrap 5 CDN, no build step) -- **Monitoring**: Celery Flower (separate container, basic auth) -- **Testing**: pytest + pytest-asyncio + factory-boy, SQLite in-memory - ---- - -## Architecture - -``` -docker-compose: - web → FastAPI + uvicorn (port 8080) - worker → Celery worker (queues: sync, health, default) - beat → Celery beat (PersistentScheduler, single process) - flower → Flower UI (port 5555, basic auth) - db → MariaDB 11.4 - redis → Redis 7 alpine - -Single Dockerfile.web image — command: key differentiates services. -``` - -**Key architectural principle:** Sync logic is split across three layers: -1. `api_clients/` — pure HTTP, no ORM -2. `sync_core/` — pure business logic, no HTTP, no ORM → fully unit-testable -3. `tasks/` — orchestration: wires DB + clients + core, writes job history - ---- - -## Directory Structure - -``` -web/ -├── main.py # FastAPI app factory, router registration -├── config.py # pydantic-settings Settings -├── database.py # SQLAlchemy engine, SessionLocal, Base -├── auth.py # hash_password, verify_password, get_current_user -├── schemas.py # Pydantic form validation -├── templates_env.py # Jinja2Templates singleton -│ -├── models/ -│ ├── user.py -│ ├── evotor.py -│ ├── vk.py -│ ├── sync.py # SyncConfig, SyncFilter -│ ├── cache.py # CachedStore, CachedGroup, CachedProduct -│ └── history.py # SyncJob, HealthCheckLog -│ -├── api_clients/ -│ ├── evotor.py # fetch_stores/groups/products, token_refresh -│ └── vk.py # get/create/edit/delete products, albums, upload_photo -│ -├── sync_core/ -│ ├── filters.py # is_group_included, get_included_store_ids (pure) -│ ├── transform.py # calc_price, normalize_name, is_weight_measure (pure) -│ ├── differ.py # diff_products → SyncDiff dataclass (pure) -│ └── engine.py # sync_user(SyncContext) → SyncResult (pure, no I/O) -│ -├── tasks/ -│ ├── celery_app.py # Celery factory + beat_schedule -│ ├── sync.py # run_sync_all, run_sync_for_user -│ ├── catalog.py # refresh_all_catalogs, refresh_catalog_for_user -│ └── health.py # check_all_connections, prune_old_logs -│ -├── routes/ -│ ├── auth.py # /register /login /logout /confirm-email -│ ├── profile.py # /profile /profile/edit /password /delete -│ ├── reset.py # /forgot-password /reset-password -│ ├── connections.py # /connections -│ ├── evotor.py # /evotor /evotor/callback /evotor/token /evotor/disconnect -│ ├── vk.py # /vk /vk/token /vk/disconnect -│ ├── sync.py # /sync /sync/toggle /sync/confirm /sync/run-now -│ ├── catalog.py # /catalog /catalog/groups /catalog/products -│ └── admin.py # /admin/* (user list, job history, manual triggers) -│ -├── templates/ # Jinja2 .html files (Bootstrap 5) -│ ├── base.html -│ ├── admin/ -│ └── [21 page templates] -│ -├── static/style.css -│ -└── migrations/ - ├── env.py - └── versions/0001_initial.py - -tests/ -├── conftest.py # SQLite in-memory engine, db_session (rollback per test), sample_user -├── unit/ -│ ├── test_filters.py -│ ├── test_transform.py -│ └── test_differ.py -├── integration/ -│ ├── test_sync_engine.py -│ └── test_catalog_refresh.py -└── web/ - ├── test_auth_routes.py - └── test_evotor_webhook.py -``` - ---- - -## Database Schema - -### `users` -`id, first_name, last_name, email (UNIQUE), phone (UNIQUE), password_hash, is_email_confirmed, email_confirm_token, password_reset_token, password_reset_expires, is_admin, created_at, updated_at` - -### `evotor_connections` -`id, user_id (FK nullable), evotor_user_id (UNIQUE), access_token, refresh_token, token_expires_at, connection_type ENUM('webhook','manual'), is_online, last_checked_at, connected_at, updated_at` - -`user_id` is nullable — the webhook creates the row before a user claims it. - -### `vk_connections` -`id, user_id (FK UNIQUE), access_token, group_id, group_name, group_screen_name, is_online, last_checked_at, connected_at, updated_at` - -### `sync_configs` -`id, user_id (FK UNIQUE), is_enabled, confirmed_at, last_run_at (denorm), created_at, updated_at` - -### `sync_filters` -`id, sync_config_id (FK), entity_type ENUM('store','group','product'), entity_id, entity_name, filter_mode ENUM('include','exclude'), parent_entity_id, created_at` -`UNIQUE(sync_config_id, entity_type, entity_id)` - -### `cached_stores` -`id, user_id (FK), evotor_id, name, address, fetched_at` - -### `cached_groups` -`id, user_id (FK), evotor_id, store_evotor_id, name, fetched_at` - -### `cached_products` -`id, user_id (FK), evotor_id, store_evotor_id, group_evotor_id, name, price, cost_price, quantity, measure_name, article_number, description, allow_to_sell, product_type, is_excisable, is_age_limited, fetched_at, synced_at` - -### `sync_jobs` -`id, user_id (FK INDEX), triggered_by ENUM('schedule','manual','webhook'), status ENUM('pending','running','success','failed','partial'), started_at, finished_at, items_created, items_updated, items_deleted, items_skipped, error_count, error_detail (TEXT JSON), celery_task_id` - -### `health_check_logs` -`id, user_id (FK INDEX), service ENUM('evotor','vk'), event ENUM('online','offline','token_refreshed','token_refresh_failed'), detail, checked_at` -Pruned after 30 days by nightly beat task. - ---- - -## Celery Tasks & Schedule - -| Task | Queue | Schedule | Retry | -|------|-------|----------|-------| -| `sync.run_sync_all` | sync | every hour :00 | — | -| `sync.run_sync_for_user(user_id)` | sync | spawned by above | 3x, 60s delay | -| `catalog.refresh_all_catalogs` | default | every hour :30 | — | -| `catalog.refresh_catalog_for_user(user_id)` | default | spawned by above | 2x, 120s delay | -| `health.check_all_connections` | health | every 10 min | — | -| `health.prune_old_logs` | default | daily 03:00 | — | - -`run_sync_for_user` acquires a Redis lock `sync:user:{id}` (5 min TTL) to prevent overlap between scheduled and manual triggers. - -`task_acks_late=True` + `worker_prefetch_multiplier=1` — re-queue on worker crash, prevent task starvation. - ---- - -## SyncContext / SyncResult boundary - -```python -# sync_core/engine.py — zero imports from web.*, httpx, or sqlalchemy -@dataclass -class SyncContext: - user_id: int - evo_products: list[EvoProduct] # plain dataclasses - vk_products: list[VkProduct] - vk_albums: list[VkAlbum] - filters: list[FilterRule] - photo_path: str - -@dataclass -class SyncResult: - created: list[str] # evo_ids to create - updated: list[str] # evo_ids to update - deleted: list[int] # vk_ids to delete - skipped: list[str] - errors: list[SyncError] -``` - -`sync_user(ctx)` returns intent only — no I/O. The task layer (`tasks/sync.py`) executes VK API calls and writes the `SyncJob` row. This makes unit tests trivial: pass dataclasses in, assert on the result, no mocks needed. - ---- - -## Filter semantics (documented in `sync_core/filters.py`) -- No filters of a type → include everything -- Only exclude filters → include all except listed -- Any include filter → include only listed, exclude rest - ---- - -## Dockerfile.web -Single image for all services. `command:` in docker-compose selects role (web / worker / beat / flower). - -```dockerfile -FROM python:3.12-slim -ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 -WORKDIR /app -RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* -COPY requirements.txt . -RUN pip install --upgrade pip && pip install -r requirements.txt -COPY . . -CMD ["uvicorn", "web.main:app", "--host", "0.0.0.0", "--port", "8000"] -``` - ---- - -## docker-compose services -- `db`: mariadb:11.4, healthcheck, named volume -- `redis`: redis:7-alpine, `--save 60 1`, named volume, healthcheck -- `web`: port 8080→8000, `alembic upgrade head && uvicorn ...`, depends on db+redis healthy -- `worker`: `celery -A web.tasks.celery_app worker --loglevel=info --concurrency=2 --queues=default,sync,health` -- `beat`: `celery -A web.tasks.celery_app beat --scheduler celery.beat:PersistentScheduler --schedule /tmp/celerybeat-schedule` -- `flower`: port 5555, basic auth via `FLOWER_BASIC_AUTH` env var - ---- - -## requirements.txt -``` -fastapi==0.115.5 -uvicorn[standard]==0.32.1 -python-multipart==0.0.12 -jinja2==3.1.4 -sqlalchemy==2.0.36 -alembic==1.14.0 -pymysql==1.1.1 -cryptography>=44.0.0 -passlib[bcrypt]==1.7.4 -bcrypt==4.2.1 -pydantic-settings==2.6.1 -httpx==0.28.1 -celery[redis]==5.4.0 -redis==5.2.1 -flower==2.0.1 -python-json-logger==3.2.1 -pytest==8.3.4 -pytest-asyncio==0.24.0 -pytest-cov==6.0.0 -factory-boy==3.3.1 -``` - ---- - -## Test patterns - -**conftest.py:** -- SQLite in-memory engine (session-scoped) -- `db_session`: connection + transaction + rollback per test (clean slate, no truncation needed) -- `sample_user`: confirmed user pre-inserted -- Route tests: `httpx.AsyncClient(transport=ASGITransport(app=app))` - -**Unit tests** (`sync_core/`): zero patches — pass plain dataclasses, assert on SyncResult. - -**Integration tests**: mock only the HTTP clients (`api_clients/`), use real DB session. - ---- - -## Implementation sequence -1. **Foundation** — config, database, models, Alembic migration, Dockerfile.web, docker-compose.yml, requirements.txt -2. **Auth** — user model, routes, templates (register/login/confirm/reset) -3. **Connections** — evotor + vk models, routes, templates, webhook endpoint -4. **API clients** — `api_clients/evotor.py` + `api_clients/vk.py` with unit tests for response parsing -5. **Sync core** — filters → transform → differ → engine, all fully unit-tested before wiring -6. **Celery** — celery_app.py, beat schedule, health tasks -7. **Catalog** — catalog tasks + routes + templates -8. **Sync tasks** — tasks/sync.py + SyncJob writes + sync routes + job history UI -9. **Admin panel** — /admin routes + templates -10. **Integration tests** — written alongside steps 6–8 - ---- - -## Verification checklist -- `docker compose up` → all 6 services healthy -- `pytest tests/unit/` → 100% pass, no I/O, no patches -- `pytest tests/` → coverage ≥ 60% -- Flower at :5555 shows beat schedule + worker connected -- E2E: register → connect Evotor (webhook) + VK → enable sync → manual trigger → `sync_jobs` row with `status=success` diff --git a/EvoSync-v3-%E2%80%94-Step-1%3A-Environment-Setup.md b/EvoSync-v3-%E2%80%94-Step-1%3A-Environment-Setup.md new file mode 100644 index 0000000..5fb1de5 --- /dev/null +++ b/EvoSync-v3-%E2%80%94-Step-1%3A-Environment-Setup.md @@ -0,0 +1,116 @@ +# EvoSync v3 — Step 1: Environment Setup + +## Context +Rebuilding EvoSync from scratch (Python sync platform, Evotor → VK Market). Repo is clean — no web app exists. Goal for this step: scaffold the project skeleton, wire up Docker, and verify all services start healthy with a working DB connection and a live FastAPI app. + +--- + +## Stack +- **Backend**: FastAPI 0.115 + Uvicorn (Python 3.12) +- **Task queue**: Celery 5.4 + Redis 7 +- **Database**: MariaDB 11.4 + SQLAlchemy 2.0 + Alembic +- **Templates**: Jinja2 + Bootstrap 5 CDN +- **Monitoring**: Celery Flower + +--- + +## What gets created in this step + +### Root-level files +- `Dockerfile.web` — single image for web/worker/beat/flower +- `docker-compose.yml` — 6 services: db, redis, web, worker, beat, flower +- `requirements.txt` — all pinned dependencies +- `.env.example` — template with all required vars +- `alembic.ini` — points to `web/migrations/` + +### `web/` package skeleton +``` +web/ +├── __init__.py +├── main.py # FastAPI app, single health route GET /health +├── config.py # pydantic-settings Settings (DATABASE_URL, REDIS_URL, SECRET_KEY, …) +├── database.py # SQLAlchemy engine + SessionLocal + Base +├── models/ +│ └── __init__.py # empty for now +├── tasks/ +│ ├── __init__.py +│ └── celery_app.py # Celery app factory, broker=REDIS_URL, no beat schedule yet +└── migrations/ + ├── env.py # Alembic env wired to web.database.Base + ├── script.py.mako + └── versions/ + └── 0001_initial.py # empty migration (no tables yet) +``` + +### `tests/` +``` +tests/ +├── conftest.py # SQLite in-memory engine, db_session fixture +└── test_health.py # GET /health returns 200 {"status": "ok"} +``` + +--- + +## docker-compose services + +| Service | Image | Port | Command | +|---------|-------|------|---------| +| `db` | mariadb:11.4 | — | default, healthcheck | +| `redis` | redis:7-alpine | — | `redis-server --save 60 1`, healthcheck | +| `web` | Dockerfile.web | 8080→8000 | `alembic upgrade head && uvicorn web.main:app --host 0.0.0.0 --port 8000` | +| `worker` | Dockerfile.web | — | `celery -A web.tasks.celery_app worker --loglevel=info` | +| `beat` | Dockerfile.web | — | `celery -A web.tasks.celery_app beat --scheduler celery.beat:PersistentScheduler --schedule /tmp/celerybeat-schedule` | +| `flower` | Dockerfile.web | 5555 | `celery -A web.tasks.celery_app flower --port=5555` | + +- `web` depends on `db` and `redis` being healthy before starting +- `worker` and `beat` depend on `redis` being healthy +- Single `Dockerfile.web` image, `command:` selects role + +--- + +## Dockerfile.web +```dockerfile +FROM python:3.12-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 +WORKDIR /app +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* +COPY requirements.txt . +RUN pip install --upgrade pip && pip install -r requirements.txt +COPY . . +CMD ["uvicorn", "web.main:app", "--host", "0.0.0.0", "--port", "8000"] +``` + +--- + +## requirements.txt +``` +fastapi==0.115.5 +uvicorn[standard]==0.32.1 +python-multipart==0.0.12 +jinja2==3.1.4 +sqlalchemy==2.0.36 +alembic==1.14.0 +pymysql==1.1.1 +cryptography>=44.0.0 +passlib[bcrypt]==1.7.4 +bcrypt==4.2.1 +pydantic-settings==2.6.1 +httpx==0.28.1 +celery[redis]==5.4.0 +redis==5.2.1 +flower==2.0.1 +python-json-logger==3.2.1 +pytest==8.3.4 +pytest-asyncio==0.24.0 +pytest-cov==6.0.0 +factory-boy==3.3.1 +``` + +--- + +## Verification checklist +1. `docker compose up --build` → all 6 services reach healthy/running state +2. `curl http://localhost:8080/health` → `{"status": "ok"}` +3. Flower at `http://localhost:5555` → shows worker connected +4. `docker compose exec web alembic upgrade head` → no error +5. `pytest tests/` → `test_health.py` passes