refactor: reorganize scripts - move debug code to experiments/

- Move debug/test scripts from src/scripts/ to experiments/
- Remove test-detail-endpoint from package.json
- Delete temp-product-page.html
- Move E2E_GUIDE.md to docs/
- Add experiments/README.md with documentation
- Keep only production scripts in src/scripts/
- Clean up tsconfig.json exclude list (experiments are now outside src/)

Co-Authored-By: Claude <noreply@anthropic.com>

experiments/magnit-detail-endpoints/debug-detail-response.ts

@@ -0,0 +1,81 @@
+import 'dotenv/config';
+import { chromium } from 'playwright';
+import axios from 'axios';
+import { Logger } from '../utils/logger.js';
+
+async function main() {
+  Logger.info('=== Debug: Смотрим что возвращает API ===\n');
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  await page.goto('https://magnit.ru/', { waitUntil: 'domcontentloaded' });
+  const cookies = await context.cookies();
+  const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+
+  const mgUdiCookie = cookies.find(c => c.name === 'mg_udi');
+  const deviceId = mgUdiCookie?.value || '';
+
+  const httpClient = axios.create({
+    baseURL: 'https://magnit.ru',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': '*/*',
+      'Cookie': cookieStr,
+      'x-device-id': deviceId,
+      'x-client-name': 'magnit',
+      'x-device-platform': 'Web',
+      'x-new-magnit': 'true',
+    },
+  });
+
+  await browser.close();
+
+  // Тестируем на конкретном товаре из логов
+  const productId = '1000201813'; // Презервативы Durex - там был brand
+
+  const endpoint = `/webgate/v2/goods/${productId}/stores/992301?storetype=2&catalogtype=1`;
+
+  Logger.info(`Запрос: ${endpoint}`);
+
+  try {
+    const response = await httpClient.get(endpoint);
+    Logger.info(`Status: ${response.status}\n`);
+
+    const data = response.data;
+    console.log(JSON.stringify(data, null, 2));
+
+    // Анализируем что есть в ответе
+    if (data.details && data.details.length > 0) {
+      Logger.info(`\n=== АНАЛИЗ details массива (${data.details.length} элементов) ===\n`);
+
+      for (let i = 0; i < Math.min(data.details.length, 15); i++) {
+        const detail = data.details[i];
+        Logger.info(`${i + 1}. name: "${detail.name}" | value: "${detail.value}"`);
+
+        // Проверяем парсинг
+        const name = detail.name.toLowerCase();
+
+        if (name.includes('бренд') || name === 'brand') {
+          Logger.info(`   → Это БРЕНД!`);
+        } else if (name.includes('описание') || name === 'description') {
+          Logger.info(`   → Это ОПИСАНИЕ!`);
+        } else if (name.includes('вес') || name.includes('weight')) {
+          Logger.info(`   → Это ВЕС!`);
+        } else if (name.includes('единица') || name.includes('unit')) {
+          Logger.info(`   → Это ЕДИНИЦА!`);
+        }
+      }
+    }
+
+    if (data.categories && data.categories.length > 0) {
+      Logger.info(`\nCategories: ${data.categories.join(', ')}`);
+    }
+
+  } catch (error: any) {
+    Logger.error('Ошибка:', error.message);
+  }
+}
+
+main();

experiments/magnit-detail-endpoints/find-product-detail-api.ts

@@ -0,0 +1,149 @@
+import 'dotenv/config';
+import { chromium } from 'playwright';
+import axios from 'axios';
+import { Logger } from '../utils/logger.js';
+
+async function findDetailApiViaDirectRequest() {
+  Logger.info('=== МЕТОД 1: Прямой GET запрос к API ===\n');
+
+  const productId = '1000233138';
+  const storeCode = process.env.MAGNIT_STORE_CODE || '992301';
+
+  // Сначала получим cookies через Playwright
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  await page.goto('https://magnit.ru/', { waitUntil: 'domcontentloaded' });
+  const cookies = await context.cookies();
+  const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+
+  const mgUdiCookie = cookies.find(c => c.name === 'mg_udi');
+  const deviceId = mgUdiCookie?.value || '';
+
+  await browser.close();
+
+  // Теперь пробуем разные endpoints
+  const httpClient = axios.create({
+    baseURL: 'https://magnit.ru',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': '*/*',
+      'Cookie': cookieStr,
+      'x-device-id': deviceId,
+      'x-client-name': 'magnit',
+      'x-device-platform': 'Web',
+      'x-new-magnit': 'true',
+    },
+  });
+
+  const endpoints = [
+    `/webgate/v2/goods/${productId}?storeCode=${storeCode}&storeType=6`,
+    `/webgate/v2/goods/${productId}?shopCode=${storeCode}&shopType=6`,
+    `/webgate/v2/products/${productId}?storeCode=${storeCode}`,
+    `/webgate/v2/catalog/product/${productId}?storeCode=${storeCode}`,
+  ];
+
+  for (const endpoint of endpoints) {
+    try {
+      Logger.info(`Пробую: GET ${endpoint}`);
+      const response = await httpClient.get(endpoint);
+      Logger.info(`✅ Status: ${response.status}`);
+      if (response.data) {
+        const json = JSON.stringify(response.data);
+        if (json.length < 2000) {
+          Logger.info(`Response: ${json}`);
+        } else {
+          Logger.info(`Response (preview): ${json.substring(0, 500)}...`);
+        }
+      }
+      break; // Если успешно, выходим
+    } catch (error: any) {
+      if (error.response?.status === 404) {
+        Logger.info(`   ❌ 404 Not Found`);
+      } else if (error.response?.status === 403) {
+        Logger.info(`   ❌ 403 Forbidden`);
+      } else {
+        Logger.info(`   ❌ ${error.message}`);
+      }
+    }
+  }
+}
+
+async function extractFromSSR() {
+  Logger.info('\n=== МЕТОД 2: Извлечение данных из SSR HTML ===\n');
+
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  const productUrl = 'https://magnit.ru/product/1000233138-podguzniki_la_fresh_dlya_vzroslykh_l_10sht?shopCode=992301&shopType=6';
+
+  Logger.info(`Загружаю страницу (без networkidle): ${productUrl}`);
+
+  try {
+    await page.goto(productUrl, {
+      waitUntil: 'domcontentloaded',
+      timeout: 15000,
+    });
+
+    // Ждем немного для рендеринга
+    await page.waitForTimeout(2000);
+
+    // Проверяем данные в HTML
+    const productData = await page.evaluate(() => {
+      // Ищем данные в window.__NUXT__
+      if ((window as any).__NUXT__) {
+        const nuxtData = (window as any).__NUXT__;
+        return {
+          source: '__NUXT__',
+          keys: Object.keys(nuxtData),
+          // Проверяем ключи с данными о товаре
+          data: nuxtData.data || nuxtData.pinia || null,
+        };
+      }
+
+      // Ищем JSON в скриптах
+      const scripts = Array.from(document.querySelectorAll('script[type="application/json"]'));
+      for (const script of scripts) {
+        try {
+          const json = JSON.parse(script.textContent || '');
+          if (json.product || json.goods || json.data?.product) {
+            return { source: 'application/json script', data: json };
+          }
+        } catch (e) {}
+      }
+
+      return { found: false };
+    });
+
+    if (productData.source) {
+      Logger.info(`✅ Данные найдены в: ${productData.source}`);
+      Logger.info(`Ключи: ${JSON.stringify(productData.keys || Object.keys(productData.data || {}))}`);
+      if (productData.data) {
+        Logger.info(`Данные (превью): ${JSON.stringify(productData.data).substring(0, 1000)}...`);
+      }
+    } else {
+      Logger.info(`❌ Данные не найдены`);
+    }
+
+    // Также сохраним HTML для анализа
+    const html = await page.content();
+    Logger.info(`\nHTML размер: ${html.length} символов`);
+    Logger.info(`HTML содержит "brand": ${html.includes('"brand"')} раз`);
+    Logger.info(`HTML содержит "description": ${html.includes('"description"')} раз`);
+    Logger.info(`HTML содержит "weight": ${html.includes('"weight"')} раз`);
+
+  } catch (error) {
+    Logger.error(`Ошибка: ${error}`);
+  } finally {
+    await browser.close();
+  }
+}
+
+async function main() {
+  await findDetailApiViaDirectRequest();
+  await extractFromSSR();
+}
+
+main();

experiments/magnit-detail-endpoints/find-product-detail-endpoint-v1.ts

@@ -0,0 +1,93 @@
+import 'dotenv/config';
+import { chromium } from 'playwright';
+import axios from 'axios';
+import { Logger } from '../utils/logger.js';
+
+async function main() {
+  Logger.info('=== Поиск endpoint для ДЕТАЛЕЙ товара ===\n');
+
+  // Получаем cookies
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  await page.goto('https://magnit.ru/', { waitUntil: 'domcontentloaded' });
+  const cookies = await context.cookies();
+  const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+
+  const mgUdiCookie = cookies.find(c => c.name === 'mg_udi');
+  const deviceId = mgUdiCookie?.value || '';
+
+  const httpClient = axios.create({
+    baseURL: 'https://magnit.ru',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': '*/*',
+      'Cookie': cookieStr,
+      'x-device-id': deviceId,
+      'x-client-name': 'magnit',
+      'x-device-platform': 'Web',
+      'x-new-magnit': 'true',
+    },
+  });
+
+  await browser.close();
+
+  const productId = '1000233138';
+
+  // Разные возможные endpoints для деталей товара
+  const endpoints = [
+    // v1 API (пользователь нашел reviews через v1)
+    `/webgate/v1/goods/${productId}?storeCode=992301&storeType=6`,
+    `/webgate/v1/products/${productId}?storeCode=992301`,
+    `/webgate/v1/catalog/product/${productId}?storeCode=992301`,
+    `/webgate/v1/listing/goods/${productId}?storeCode=992301`,
+    `/webgate/v1/listing/product/${productId}?storeCode=992301`,
+    // Другие варианты
+    `/webgate/v1/products/detail/${productId}?storeCode=992301`,
+    `/webgate/v1/object?productId=${productId}&storeCode=992301`,
+    `/webgate/v1/item/${productId}?storeCode=992301`,
+  ];
+
+  for (const endpoint of endpoints) {
+    try {
+      Logger.info(`Пробую: ${endpoint}`);
+      const response = await httpClient.get(endpoint);
+
+      if (response.status === 200) {
+        Logger.info(`✅ Status: ${response.status}`);
+
+        const json = JSON.stringify(response.data);
+        if (json.length < 3000) {
+          Logger.info(`Response:\n${json}`);
+        } else {
+          // Проверяем, есть ли полезные поля
+          const data = response.data;
+          const hasDetails = data.brand || data.description || data.weight || data.unit;
+
+          if (hasDetails) {
+            Logger.info(`=== НАЙДЕНЫ ДЕТАЛИ ТОВАРА ===`);
+            Logger.info(`brand: ${data.brand}`);
+            Logger.info(`description: ${data.description?.substring(0, 100)}`);
+            Logger.info(`weight: ${data.weight}`);
+            Logger.info(`unit: ${data.unit}`);
+            break;
+          } else {
+            Logger.info(`Response без деталей товара (preview):`);
+            Logger.info(json.substring(0, 500) + '...');
+          }
+        }
+      }
+    } catch (error: any) {
+      if (error.response?.status === 404) {
+        Logger.info(`   ❌ 404 Not Found`);
+      } else if (error.response?.status === 403) {
+        Logger.info(`   ❌ 403 Forbidden`);
+      } else {
+        Logger.info(`   ❌ ${error.message}`);
+      }
+    }
+  }
+}
+
+main();

experiments/magnit-detail-endpoints/test-all-detail-endpoints.ts

@@ -0,0 +1,84 @@
+import 'dotenv/config';
+import { chromium } from 'playwright';
+import axios from 'axios';
+import { Logger } from '../utils/logger.js';
+
+async function main() {
+  Logger.info('=== Тестирование всех endpoints для деталей товара ===\n');
+
+  // Получаем cookies
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  await page.goto('https://magnit.ru/', { waitUntil: 'domcontentloaded' });
+  const cookies = await context.cookies();
+  const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+
+  const mgUdiCookie = cookies.find(c => c.name === 'mg_udi');
+  const deviceId = mgUdiCookie?.value || '';
+
+  const httpClient = axios.create({
+    baseURL: 'https://magnit.ru',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': '*/*',
+      'Cookie': cookieStr,
+      'x-device-id': deviceId,
+      'x-client-name': 'magnit',
+      'x-device-platform': 'Web',
+      'x-new-magnit': 'true',
+    },
+  });
+
+  await browser.close();
+
+  const endpoints = [
+    {
+      name: '🔍 user-reviews-and-object-info (ДЕТАЛИ ТОВАРА)',
+      url: '/webgate/v1/listing/user-reviews-and-object-info?service=dostavka&objectType=product&objectId=1000530495',
+    },
+    {
+      name: '🏷️ promotions/type',
+      url: '/webgate/v1/promotions/type?adult=true&type=19&limit=10&storeCode=996609',
+    },
+    {
+      name: '🏪 goods/{id}/stores/{storeCode}',
+      url: '/webgate/v2/goods/1000530495/stores/996609?storetype=2&catalogtype=1',
+    },
+  ];
+
+  for (const { name, url } of endpoints) {
+    try {
+      Logger.info(`\n${name}`);
+      Logger.info(`URL: ${url}`);
+
+      const response = await httpClient.get(url);
+      Logger.info(`✅ Status: ${response.status}`);
+
+      const data = response.data;
+      const json = JSON.stringify(data, null, 2);
+
+      // Показываем только ключевые поля
+      if (json.length < 2000) {
+        Logger.info(`Response:\n${json}`);
+      } else {
+        Logger.info(`Response (превью):`);
+        console.log(json.substring(0, 800) + '...');
+      }
+
+      // Проверяем наличие ключевых полей
+      const hasDetails = data.brand || data.description || data.weight || data.unit ||
+                         data.objectInfo?.brand || data.objectInfo?.description ||
+                         data.product?.brand || data.product?.description;
+
+      if (hasDetails) {
+        Logger.info(`\n⭐️⭐️ НАЙДЕНЫ ДЕТАЛИ ТОВАРА! ⭐️⭐️`);
+      }
+    } catch (error: any) {
+      Logger.info(`❌ Error: ${error.response?.status || error.message}`);
+    }
+  }
+}
+
+main();

experiments/magnit-detail-endpoints/test-detail-endpoint.ts

@@ -0,0 +1,55 @@
+import 'dotenv/config';
+import { MagnitApiScraper } from '../scrapers/api/magnit/MagnitApiScraper.js';
+import { Logger } from '../utils/logger.js';
+
+async function main() {
+  const storeCode = process.env.MAGNIT_STORE_CODE || '992301';
+  const productId = '1000233138'; // Из inspect скрипта
+
+  const scraper = new MagnitApiScraper({
+    storeCode,
+    storeType: process.env.MAGNIT_STORE_TYPE || '6',
+    catalogType: process.env.MAGNIT_CATALOG_TYPE || '1',
+    headless: process.env.MAGNIT_HEADLESS !== 'false',
+  });
+
+  try {
+    await scraper.initialize();
+
+    Logger.info(`Попытка получить детали товара ${productId}...\n`);
+
+    // Пробуем разные возможные endpoints для деталей товара
+    const endpoints = [
+      `/webgate/v2/goods/${productId}`,
+      `/webgate/v2/products/${productId}`,
+      `/webgate/v2/catalog/product/${productId}`,
+      `/webgate/v2/goods/detail/${productId}`,
+    ];
+
+    for (const endpoint of endpoints) {
+      try {
+        Logger.info(`Пробую: ${endpoint}`);
+        const response = await (scraper as any).httpClient.get(endpoint);
+        Logger.info(`✅ Успех! Status: ${response.status}`);
+        Logger.info(JSON.stringify(response.data, null, 2));
+        break; // Если успешно, выходим из цикла
+      } catch (error: any) {
+        if (error.response?.status === 404) {
+          Logger.info(`   ❌ 404 Not Found`);
+        } else if (error.response?.status === 403) {
+          Logger.info(`   ❌ 403 Forbidden (нужна аутентификация)`);
+        } else {
+          Logger.info(`   ❌ ${error.response?.status || error.message}`);
+        }
+      }
+    }
+
+  } catch (error) {
+    Logger.error('❌ Ошибка:', error);
+    process.exit(1);
+  } finally {
+    await scraper.close();
+  }
+}
+
+main();

experiments/magnit-detail-endpoints/test-object-reviews-endpoint.ts

@@ -0,0 +1,55 @@
+import 'dotenv/config';
+import { chromium } from 'playwright';
+import axios from 'axios';
+import { Logger } from '../utils/logger.js';
+
+async function main() {
+  Logger.info('=== Тестирование API endpoint для деталей товара ===\n');
+
+  // Получаем cookies через Playwright
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext();
+  const page = await context.newPage();
+
+  await page.goto('https://magnit.ru/', { waitUntil: 'domcontentloaded' });
+  const cookies = await context.cookies();
+  const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ');
+
+  const mgUdiCookie = cookies.find(c => c.name === 'mg_udi');
+  const deviceId = mgUdiCookie?.value || '';
+
+  await browser.close();
+
+  // Создаем HTTP клиент
+  const httpClient = axios.create({
+    baseURL: 'https://magnit.ru',
+    headers: {
+      'Content-Type': 'application/json',
+      'Accept': '*/*',
+      'Cookie': cookieStr,
+      'x-device-id': deviceId,
+      'x-client-name': 'magnit',
+      'x-device-platform': 'Web',
+      'x-new-magnit': 'true',
+    },
+  });
+
+  // Пробуем endpoint который нашел пользователь
+  const endpoint = '/webgate/v1/listing/object-reviews?service=dostavka&objectId=1000530495&objectType=product&page=0&size=10';
+
+  try {
+    Logger.info(`Запрос: ${endpoint}`);
+    const response = await httpClient.get(endpoint);
+    Logger.info(`✅ Status: ${response.status}`);
+    Logger.info(`\n=== ОТВЕТ API ===`);
+    console.log(JSON.stringify(response.data, null, 2));
+  } catch (error: any) {
+    Logger.error(`Ошибка: ${error.message}`);
+    if (error.response) {
+      Logger.error(`Status: ${error.response.status}`);
+      Logger.error(`Data:`, error.response.data);
+    }
+  }
+}
+
+main();