feat: Evotor + VK catalog sync, connections, and store/group filters
- Evotor catalog: background Celery task syncing stores/groups/products from Evotor API; UI pages with per-store and per-group sync toggles - VK connection: manual token + group ID entry with inline test button - Evotor connection: inline test button (calls /stores) - VK catalog: background task syncing VK Market albums and products; separate catalog UI at /vk-catalog/albums - SyncFilter extended to support entity_type=group with parent_entity_id - Migration 0004: vk_cached_albums + vk_cached_products tables - Beat schedule updated to run both refresh_catalog and refresh_vk_catalog - README updated with new schema, routes, tasks, and config Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
mguschin
167
web/tasks/vk_catalog.py
Normal file
167
web/tasks/vk_catalog.py
Normal file
@@ -0,0 +1,167 @@
|
||||
"""
|
||||
Periodic VK catalog sync: fetch albums and products from VK Market
|
||||
for every connected user and upsert into vk_cached_* tables.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
from celery import shared_task
|
||||
|
||||
from web.config import settings
|
||||
from web.database import SessionLocal
|
||||
from web.models.connections import VkCachedAlbum, VkCachedProduct, VkConnection
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VK_API = "https://api.vk.com/method"
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
def _vk_get(method: str, params: dict, token: str) -> dict:
|
||||
params = {**params, "access_token": token, "v": settings.VK_API_VERSION}
|
||||
r = httpx.get(f"{VK_API}/{method}", params=params, timeout=20)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
def _sync_user(db, user_id: int, token: str, group_id: str) -> None:
|
||||
now = _now()
|
||||
owner_id = f"-{group_id}"
|
||||
|
||||
# ── albums ────────────────────────────────────────────────────────────────
|
||||
try:
|
||||
data = _vk_get("market.getAlbums", {"owner_id": owner_id, "count": 100}, token)
|
||||
except Exception as e:
|
||||
logger.warning("user=%s vk fetch albums failed: %s", user_id, e)
|
||||
return
|
||||
|
||||
if "error" in data:
|
||||
logger.warning("user=%s vk albums error: %s", user_id, data["error"])
|
||||
return
|
||||
|
||||
albums = data.get("response", {}).get("items", [])
|
||||
album_ids = []
|
||||
for a in albums:
|
||||
aid = str(a["id"])
|
||||
album_ids.append(aid)
|
||||
row = db.query(VkCachedAlbum).filter_by(user_id=user_id, vk_group_id=group_id, album_id=aid).first()
|
||||
if row:
|
||||
row.title = a.get("title", "")
|
||||
row.count = a.get("count")
|
||||
row.fetched_at = now
|
||||
else:
|
||||
db.add(VkCachedAlbum(
|
||||
user_id=user_id,
|
||||
vk_group_id=group_id,
|
||||
album_id=aid,
|
||||
title=a.get("title", ""),
|
||||
count=a.get("count"),
|
||||
fetched_at=now,
|
||||
))
|
||||
db.flush()
|
||||
|
||||
# ── products (extended=1 gives albums_ids per product) ───────────────────
|
||||
offset = 0
|
||||
all_products = []
|
||||
while True:
|
||||
try:
|
||||
data = _vk_get(
|
||||
"market.get",
|
||||
{"owner_id": owner_id, "count": 200, "offset": offset, "extended": 1},
|
||||
token,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("user=%s vk fetch products (extended) failed: %s", user_id, e)
|
||||
break
|
||||
|
||||
if "error" in data:
|
||||
logger.warning("user=%s vk products (extended) error: %s", user_id, data["error"])
|
||||
break
|
||||
|
||||
items = data.get("response", {}).get("items", [])
|
||||
all_products.extend(items)
|
||||
if len(items) < 200:
|
||||
break
|
||||
offset += 200
|
||||
|
||||
for p in all_products:
|
||||
pid = str(p["id"])
|
||||
album_id = str(p["albums_ids"][0]) if p.get("albums_ids") else None
|
||||
price_raw = p.get("price", {}).get("amount")
|
||||
price = float(price_raw) / 100 if price_raw is not None else None
|
||||
thumb = None
|
||||
if p.get("thumb_photo"):
|
||||
sizes = p["thumb_photo"].get("sizes", [])
|
||||
if sizes:
|
||||
thumb = sizes[-1].get("url")
|
||||
|
||||
row = db.query(VkCachedProduct).filter_by(
|
||||
user_id=user_id, vk_group_id=group_id, vk_product_id=pid,
|
||||
).first()
|
||||
if row:
|
||||
row.album_id = album_id
|
||||
row.name = p.get("title", "")
|
||||
row.description = p.get("description")
|
||||
row.price = price
|
||||
row.availability = p.get("availability")
|
||||
row.thumb_url = thumb
|
||||
row.fetched_at = now
|
||||
else:
|
||||
db.add(VkCachedProduct(
|
||||
user_id=user_id,
|
||||
vk_group_id=group_id,
|
||||
vk_product_id=pid,
|
||||
album_id=album_id,
|
||||
name=p.get("title", ""),
|
||||
description=p.get("description"),
|
||||
price=price,
|
||||
availability=p.get("availability"),
|
||||
thumb_url=thumb,
|
||||
fetched_at=now,
|
||||
))
|
||||
|
||||
db.commit()
|
||||
logger.info(
|
||||
"user=%s vk catalog synced: group=%s albums=%d products=%d",
|
||||
user_id, group_id, len(albums), len(all_products),
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
name="web.tasks.vk_catalog.refresh_vk_catalog",
|
||||
queue="default",
|
||||
bind=True,
|
||||
max_retries=2,
|
||||
default_retry_delay=60,
|
||||
)
|
||||
def refresh_vk_catalog(self) -> dict:
|
||||
"""Fetch and cache VK Market albums and products for all connected users."""
|
||||
db = SessionLocal()
|
||||
results = {"ok": 0, "failed": 0}
|
||||
try:
|
||||
connections = (
|
||||
db.query(VkConnection)
|
||||
.filter(
|
||||
VkConnection.user_id.isnot(None),
|
||||
VkConnection.access_token.isnot(None),
|
||||
VkConnection.access_token != "",
|
||||
VkConnection.vk_user_id.isnot(None),
|
||||
VkConnection.vk_user_id != "",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
for conn in connections:
|
||||
try:
|
||||
_sync_user(db, conn.user_id, conn.access_token, conn.vk_user_id)
|
||||
results["ok"] += 1
|
||||
except Exception as exc:
|
||||
logger.error("vk catalog sync failed for user=%s: %s", conn.user_id, exc)
|
||||
results["failed"] += 1
|
||||
finally:
|
||||
db.close()
|
||||
logger.info("refresh_vk_catalog done: %s", results)
|
||||
return results
|
||||
Reference in New Issue
Block a user