Mc Smog
9164527f58
- Add streaming mode for memory-efficient large catalog scraping - Implement retry logic with exponential backoff - Add auto session reinitialization on 403 errors - Add configurable options (pageSize, maxProducts, rateLimitDelay) - Add maxIterations protection against infinite loops - Add retry.ts utility module with withRetry and withRetryAndReinit - Update .env.example with new scraping options - Add pgAdmin and CloudBeaver to docker-compose Co-Authored-By: Claude <noreply@anthropic.com>
20 lines
920 B
Plaintext
20 lines
920 B
Plaintext
# Database
|
|
DATABASE_URL="postgresql://user:password@localhost:5432/supermarket?schema=public"
|
|
|
|
# Magnit API Configuration
|
|
MAGNIT_STORE_CODE="992301"
|
|
MAGNIT_STORE_TYPE="6"
|
|
MAGNIT_CATALOG_TYPE="1"
|
|
|
|
# Scraping Options
|
|
MAGNIT_USE_STREAMING=true # true = streaming mode (рекомендуется), false = legacy
|
|
MAGNIT_PAGE_SIZE=50 # Размер страницы API (max 50)
|
|
MAGNIT_MAX_PRODUCTS= # Лимит товаров (пусто = без лимита)
|
|
MAGNIT_RATE_LIMIT_DELAY=300 # Задержка между запросами (ms)
|
|
MAGNIT_MAX_ITERATIONS=10000 # Защита от бесконечного цикла
|
|
MAGNIT_HEADLESS=true # Headless режим браузера
|
|
|
|
# Resilience Options
|
|
MAGNIT_RETRY_ATTEMPTS=3 # Количество попыток retry
|
|
MAGNIT_REQUEST_TIMEOUT=30000 # Timeout запросов (ms)
|