v2.1

Ship configurator updates with catalog DB included for server deploy.

Co-authored-by: Cursor <[email protected]>

server/services/leubaHiagCrawler.js

@@ -0,0 +1,533 @@
+/**
+ * Leuba Hiag catalog crawler.
+ *
+ * The Leuba Hiag webshop (https://www.leubahiag.ch) is a Laravel/Bagisto shop whose
+ * product catalog is served by Algolia. The public catalog search page embeds a
+ * short-lived "secured" Algolia API key (guest scope) inside the page markup. We read
+ * that key live on every run (it expires), then page through the Algolia index and
+ * upsert each product into the shared `formex_materials` table under the `leubahiag`
+ * supplier.
+ *
+ * Notes:
+ * - Algolia exposes the product catalog but NOT prices. Customer-specific prices come
+ *   from the authenticated `/fr/sales/prices-and-availability` endpoint: we log in with
+ *   the account credentials (LEUBAHIAG_EMAIL/PASSWORD) and, after importing each line,
+ *   fetch the per-SKU panel price and store it in price_chf.
+ * - Imported items are inactive by default so the catalog rows don't flood the
+ *   configurator finish picker. Activate the wanted decors from the Matériaux admin page.
+ */
+
+import pool, { run } from '../db/index.js';
+
+const BASE_URL = 'https://www.leubahiag.ch';
+const SEARCH_URL = `${BASE_URL}/fr/catalog/search`;
+const CRAWL_DELAY = parseInt(process.env.LEUBAHIAG_CRAWL_DELAY_MS || '300', 10);
+const ALGOLIA_HITS_PER_PAGE = 1000;
+
+/**
+ * Curated catalog lines to import (per the supplier's agreed assortment).
+ * Each target maps to an Algolia query and/or brand facet filter. The `label` is used
+ * as the material `collection` so the configurator groups items by recognisable line.
+ */
+export const LEUBA_HIAG_TARGETS = [
+  { key: 'pfleiderer-xpsm', label: 'Pfleiderer XPSM (Antifingerprint)', tier: 'Entrée de gamme', query: 'XPSM', brands: ['Pfleiderer'] },
+  { key: 'swiss-krono', label: 'Swiss Krono', tier: 'Entrée de gamme', query: '', brands: ['Swiss Krono'] },
+  { key: 'unilin-master-oak', label: 'Unilin Master Oak', tier: 'Milieu de gamme', query: 'Master Oak', brands: ['UNILIN'] },
+  { key: 'shinnoki', label: 'Shinnoki', tier: 'Milieu de gamme', query: 'Shinnoki', brands: [] },
+  { key: 'cleaf', label: 'Cleaf', tier: 'Haut de gamme', query: '', brands: ['CLEAF'] },
+  { key: 'valchromat', label: 'Valchromat', tier: 'Haut de gamme', query: '', brands: ['Valchromat'] },
+];
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function browserHeaders(extra = {}) {
+  return {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'fr-CH,fr;q=0.9,en;q=0.5',
+    ...extra,
+  };
+}
+
+function absoluteUrl(url) {
+  if (!url) return null;
+  if (url.startsWith('http://') || url.startsWith('https://')) return url;
+  return `${BASE_URL}${url.startsWith('/') ? '' : '/'}${url}`;
+}
+
+/** Parse a "CHF 1'234.50" string into a number, or null. */
+function parseCHF(text) {
+  if (typeof text !== 'string') return null;
+  const clean = text.replace(/CHF/gi, '').replace(/[\u00a0\u202f'’\s]/g, '').replace(',', '.');
+  const n = parseFloat(clean);
+  return Number.isFinite(n) ? n : null;
+}
+
+/**
+ * Authenticated Leuba Hiag (Laravel) session. Used to read customer-specific prices
+ * from the `prices-and-availability` endpoint (axios sends the XSRF-TOKEN cookie back
+ * as an X-XSRF-TOKEN header — we replicate that here).
+ */
+class LeubaHiagSession {
+  constructor() {
+    this.cookies = new Map();
+    this.loggedIn = false;
+  }
+
+  captureCookies(response) {
+    let raw = [];
+    if (typeof response.headers.getSetCookie === 'function') {
+      raw = response.headers.getSetCookie();
+    } else {
+      const header = response.headers.get('set-cookie');
+      if (header) raw = header.split(/,\s*(?=[A-Za-z0-9_\-]+=)/);
+    }
+    for (const entry of raw) {
+      const [pair] = entry.split(';');
+      const eq = pair.indexOf('=');
+      if (eq > 0) this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
+    }
+  }
+
+  cookieHeader() {
+    return [...this.cookies.entries()].map(([k, v]) => `${k}=${v}`).join('; ');
+  }
+
+  xsrfToken() {
+    const raw = this.cookies.get('XSRF-TOKEN');
+    return raw ? decodeURIComponent(raw) : '';
+  }
+
+  async login(emit = () => {}, credentials = {}) {
+    const email = credentials.username || credentials.email
+      || process.env.LEUBAHIAG_EMAIL || process.env.LEUBAHIAG_USER;
+    const password = credentials.password || process.env.LEUBAHIAG_PASSWORD;
+    if (!email || !password) {
+      emit({ type: 'log', message: 'Identifiants Leuba Hiag absents : import sans prix.' });
+      return false;
+    }
+
+    const loginUrl = `${BASE_URL}/fr/customer/auth/login`;
+    try {
+      const getRes = await fetch(loginUrl, { headers: browserHeaders(), redirect: 'follow' });
+      this.captureCookies(getRes);
+      const html = await getRes.text();
+      const token = (html.match(/name="_token"\s+value="([^"]+)"/) || [])[1];
+      if (!token) {
+        emit({ type: 'log', message: 'Jeton CSRF Leuba Hiag introuvable : import sans prix.' });
+        return false;
+      }
+
+      const body = new URLSearchParams();
+      body.set('_token', token);
+      body.set('email', email);
+      body.set('password', password);
+
+      // redirect:'manual' so we capture Set-Cookie from the post-login 302 (the
+      // LoggedIn/session cookies live on the redirect response, not the final page).
+      const postRes = await fetch(loginUrl, {
+        method: 'POST',
+        headers: browserHeaders({
+          'Content-Type': 'application/x-www-form-urlencoded',
+          Referer: loginUrl,
+          Origin: BASE_URL,
+          Cookie: this.cookieHeader(),
+        }),
+        body: body.toString(),
+        redirect: 'manual',
+      });
+      this.captureCookies(postRes);
+
+      // Confirm the session and refresh the XSRF-TOKEN from the authenticated catalog page.
+      const verifyRes = await fetch(`${BASE_URL}/fr/catalog/search`, {
+        headers: browserHeaders({ Cookie: this.cookieHeader() }),
+        redirect: 'follow',
+      });
+      this.captureCookies(verifyRes);
+      const verifyHtml = await verifyRes.text();
+      const isLoggedIn = /\\u0022isLoggedIn\\u0022:true/.test(verifyHtml)
+        || this.cookies.get('LoggedIn') === '1';
+
+      this.loggedIn = isLoggedIn;
+      emit({
+        type: 'log',
+        message: this.loggedIn
+          ? `Connecté à Leuba Hiag (${email}).`
+          : 'Connexion Leuba Hiag non confirmée : import sans prix.',
+      });
+      return this.loggedIn;
+    } catch (err) {
+      emit({ type: 'log', message: `Erreur de connexion Leuba Hiag : ${err.message}` });
+      return false;
+    }
+  }
+
+  /** POST {skus} to the prices endpoint; returns the per-sku price map (or {}). */
+  async fetchPrices(skus) {
+    if (!this.loggedIn || skus.length === 0) return {};
+    const res = await fetch(`${BASE_URL}/fr/sales/prices-and-availability`, {
+      method: 'POST',
+      headers: browserHeaders({
+        'Content-Type': 'application/json',
+        Accept: 'application/json, text/plain, */*',
+        'X-Requested-With': 'XMLHttpRequest',
+        'X-XSRF-TOKEN': this.xsrfToken(),
+        Referer: `${BASE_URL}/fr/catalog/search`,
+        Origin: BASE_URL,
+        Cookie: this.cookieHeader(),
+      }),
+      body: JSON.stringify({ skus }),
+    });
+    this.captureCookies(res);
+    if (!res.ok) {
+      throw new Error(`prices-and-availability HTTP ${res.status}`);
+    }
+    return res.json();
+  }
+}
+
+/**
+ * Read the live Algolia connection config (appId, index, secured apiKey) from the
+ * catalog search page. The key is embedded as `\u0022apiKey\u0022:\u0022...\u0022`
+ * inside the `window.globalSearchConfigBase` JSON blob.
+ */
+async function fetchAlgoliaConfig(emit = () => {}) {
+  const res = await fetch(SEARCH_URL, { headers: browserHeaders(), redirect: 'follow' });
+  if (!res.ok) {
+    throw new Error(`Impossible de charger la page catalogue Leuba Hiag (HTTP ${res.status}).`);
+  }
+  const html = await res.text();
+
+  const appId = (html.match(/\\u0022appId\\u0022:\\u0022([A-Za-z0-9]+)\\u0022/) || [])[1]
+    || process.env.LEUBAHIAG_ALGOLIA_APP_ID;
+  const indexName = (html.match(/\\u0022indexName\\u0022:\\u0022([A-Za-z0-9_]+)\\u0022/) || [])[1]
+    || process.env.LEUBAHIAG_ALGOLIA_INDEX;
+  const apiKey = (html.match(/\\u0022apiKey\\u0022:\\u0022([A-Za-z0-9+/=]+)\\u0022/) || [])[1];
+
+  if (!appId || !indexName || !apiKey) {
+    throw new Error('Clé Algolia introuvable sur la page catalogue Leuba Hiag (structure du site modifiée ?).');
+  }
+
+  emit({ type: 'log', message: `Catalogue Algolia détecté: index ${indexName} (app ${appId}).` });
+  return { appId, indexName, apiKey };
+}
+
+/**
+ * Run one Algolia query page. `facetFilters` is an array of arrays per Algolia syntax.
+ */
+async function queryAlgolia({ appId, indexName, apiKey }, { query = '', page = 0, hitsPerPage = ALGOLIA_HITS_PER_PAGE, facetFilters = [] } = {}) {
+  const url = `https://${appId}-dsn.algolia.net/1/indexes/${indexName}/query`;
+  const params = new URLSearchParams();
+  params.set('query', query);
+  params.set('hitsPerPage', String(hitsPerPage));
+  params.set('page', String(page));
+  if (facetFilters.length > 0) {
+    params.set('facetFilters', JSON.stringify(facetFilters));
+  }
+
+  const res = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'X-Algolia-Application-Id': appId,
+      'X-Algolia-API-Key': apiKey,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ params: params.toString() }),
+  });
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => '');
+    throw new Error(`Requête Algolia échouée (HTTP ${res.status})${body ? `: ${body.slice(0, 160)}` : ''}`);
+  }
+  return res.json();
+}
+
+function firstString(value) {
+  if (Array.isArray(value)) return value.find((v) => typeof v === 'string') || null;
+  return typeof value === 'string' ? value : null;
+}
+
+function toNumber(value) {
+  const n = Number(value);
+  return Number.isFinite(n) ? n : null;
+}
+
+/**
+ * Map an Algolia product record to a `formex_materials` row.
+ * Dimension fields from the index: thickness (mm), length (mm), width (mm).
+ * @param {string} [collectionOverride] Force the `collection` (curated line label).
+ */
+function mapHit(hit, collectionOverride) {
+  const sku = String(hit.sku || hit.objectID || '').trim();
+  if (!sku) return null;
+
+  const categoryPath = Array.isArray(hit.categories_with_path) ? hit.categories_with_path : [];
+  const category = categoryPath.length > 0
+    ? categoryPath[categoryPath.length - 1]
+    : (firstString(hit.categories?.level0) || null);
+
+  const collection = collectionOverride
+    || firstString(hit.brand)
+    || firstString(hit.categories?.level0)
+    || 'Leuba Hiag';
+
+  return {
+    sku,
+    materialCode: `LH-${sku}`,
+    name: hit.name || hit.group_name || sku,
+    collection,
+    category,
+    width_mm: toNumber(hit.width),
+    height_mm: toNumber(hit.length),
+    thickness_mm: toNumber(hit.thickness),
+    image_url: absoluteUrl(hit.image_url),
+    formex_url: absoluteUrl(hit.url),
+    price_chf: null,
+    price_unit: null,
+  };
+}
+
+async function ensureSupplier() {
+  const existing = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['leubahiag']);
+  if (existing.rows.length > 0) return existing.rows[0].id;
+
+  await run(
+    'INSERT INTO suppliers (code, name, email, type, active) VALUES (?, ?, ?, ?, ?)',
+    ['leubahiag', 'Leuba Hiag', '[email protected]', 'materials', 1]
+  );
+  const created = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['leubahiag']);
+  return created.rows[0]?.id || null;
+}
+
+/**
+ * Upsert a material. Existing rows keep their `active` flag (so admin activations are
+ * preserved across re-syncs); new rows are inserted with the provided `active` value.
+ */
+async function upsertMaterial(material, supplierId, activeForNew) {
+  const existing = await pool.query(
+    'SELECT id FROM formex_materials WHERE formex_code = ?',
+    [material.materialCode]
+  );
+
+  if (existing.rows.length > 0) {
+    await run(
+      `UPDATE formex_materials
+       SET name = ?, collection = ?, category = ?, width_mm = ?, height_mm = ?, thickness_mm = ?,
+           image_url = ?, formex_url = ?,
+           supplier_id = COALESCE(?, supplier_id), last_synced_at = datetime('now')
+       WHERE formex_code = ?`,
+      [
+        material.name,
+        material.collection,
+        material.category,
+        material.width_mm,
+        material.height_mm,
+        material.thickness_mm,
+        material.image_url,
+        material.formex_url,
+        supplierId,
+        material.materialCode,
+      ]
+    );
+    return { id: existing.rows[0].id, created: false };
+  }
+
+  await run(
+    `INSERT INTO formex_materials
+      (formex_code, name, collection, category, width_mm, height_mm, thickness_mm,
+       image_url, formex_url, price_chf, price_unit, supplier_id, active, last_synced_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
+    [
+      material.materialCode,
+      material.name,
+      material.collection,
+      material.category,
+      material.width_mm,
+      material.height_mm,
+      material.thickness_mm,
+      material.image_url,
+      material.formex_url,
+      material.price_chf ?? null,
+      material.price_unit ?? null,
+      supplierId,
+      activeForNew ? 1 : 0,
+    ]
+  );
+
+  const created = await pool.query('SELECT id FROM formex_materials WHERE formex_code = ?', [material.materialCode]);
+  return { id: created.rows[0]?.id || null, created: true };
+}
+
+/** Update the price of an already-imported material by its SKU. */
+async function updateMaterialPrice(sku, priceChf, priceUnit) {
+  await run(
+    `UPDATE formex_materials
+     SET price_chf = ?, price_unit = COALESCE(?, price_unit), last_synced_at = datetime('now')
+     WHERE formex_code = ?`,
+    [priceChf, priceUnit, `LH-${sku}`]
+  );
+}
+
+async function logSync(status, itemsCount, error = null) {
+  await run(
+    'INSERT INTO sync_logs (source, status, items_count, error) VALUES (?, ?, ?, ?)',
+    ['leubahiag', status, itemsCount, error]
+  );
+}
+
+/**
+ * Import all products matching one curated target (a single catalog line).
+ */
+async function importTarget(config, supplierId, target, activate, emit) {
+  const facetFilters = Array.isArray(target.brands) && target.brands.length > 0
+    ? [target.brands.map((b) => `brand:${b}`)]
+    : [];
+
+  let synced = 0;
+  let created = 0;
+  let page = 0;
+  let nbPages = 1;
+  let nbHits = 0;
+  const skus = [];
+
+  do {
+    if (page > 0) await sleep(CRAWL_DELAY);
+    const data = await queryAlgolia(config, { query: target.query, page, facetFilters });
+    nbPages = data.nbPages || 1;
+    nbHits = data.nbHits || 0;
+
+    if (page === 0) {
+      emit({ type: 'log', message: `→ ${target.label} (${target.tier}) : ${nbHits} produit(s).` });
+    }
+
+    const hits = Array.isArray(data.hits) ? data.hits : [];
+    for (const hit of hits) {
+      const material = mapHit(hit, target.label);
+      if (!material) continue;
+      try {
+        const result = await upsertMaterial(material, supplierId, activate);
+        synced++;
+        if (result.created) created++;
+        skus.push(material.sku);
+      } catch (err) {
+        emit({ type: 'log', message: `  ✗ ${material.materialCode}: ${err.message}` });
+      }
+    }
+    page++;
+  } while (page < nbPages);
+
+  emit({ type: 'log', message: `  ✓ ${target.label} : ${synced} importé(s) (${created} nouveau(x)).` });
+  return { synced, created, skus };
+}
+
+/**
+ * Fetch and store customer prices for a list of SKUs (in batches).
+ * Returns the number of items priced.
+ */
+async function importPrices(session, skus, emit) {
+  const BATCH = 50;
+  let priced = 0;
+  for (let i = 0; i < skus.length; i += BATCH) {
+    const batch = skus.slice(i, i + BATCH);
+    try {
+      const data = await session.fetchPrices(batch);
+      for (const sku of batch) {
+        const info = data?.[sku];
+        if (!info) continue;
+        const tier = Array.isArray(info.tierprices) ? info.tierprices[0] : null;
+        const priceChf = parseCHF(tier?.price);
+        if (priceChf == null) continue;
+        await updateMaterialPrice(sku, priceChf, 'pièce');
+        priced++;
+      }
+    } catch (err) {
+      emit({ type: 'log', message: `  ✗ Prix lot ${i / BATCH + 1}: ${err.message}` });
+    }
+    emit({ type: 'log', message: `  Prix: ${priced}/${skus.length} récupérés…` });
+    if (i + BATCH < skus.length) await sleep(CRAWL_DELAY);
+  }
+  return priced;
+}
+
+/**
+ * Synchronise the curated Leuba Hiag catalog lines via Algolia.
+ *
+ * @param {object}   options
+ * @param {string[]} [options.targets]      Subset of target keys to import (default: all).
+ * @param {boolean}  [options.activate]     Mark newly imported items active (default false).
+ * @param {boolean}  [options.fetchPrices]  Log in and import customer prices (default true).
+ * @param {object}   [options.credentials]  { username/email, password } (else env vars).
+ * @param {function} emit                   SSE-style event emitter.
+ */
+export async function syncLeubaHiag(options = {}, emit = () => {}) {
+  const { targets = null, activate = false, fetchPrices = true, credentials = {} } = options;
+
+  const selected = Array.isArray(targets) && targets.length > 0
+    ? LEUBA_HIAG_TARGETS.filter((t) => targets.includes(t.key))
+    : LEUBA_HIAG_TARGETS;
+
+  const start = Date.now();
+  let synced = 0;
+  let created = 0;
+  let priced = 0;
+
+  try {
+    if (selected.length === 0) {
+      throw new Error('Aucune gamme Leuba Hiag sélectionnée.');
+    }
+
+    const config = await fetchAlgoliaConfig(emit);
+    const supplierId = await ensureSupplier();
+
+    // Log in up-front so prices can be fetched after each line is imported.
+    const session = new LeubaHiagSession();
+    if (fetchPrices) {
+      await session.login(emit, credentials);
+    }
+
+    emit({ type: 'log', message: `${selected.length} gamme(s) à importer.` });
+    emit({ type: 'progress', current: 0, total: selected.length, collection: 'Leuba Hiag' });
+
+    const allSkus = [];
+    for (let i = 0; i < selected.length; i++) {
+      const target = selected[i];
+      emit({ type: 'progress', current: i, total: selected.length, collection: target.label });
+      try {
+        const res = await importTarget(config, supplierId, target, activate, emit);
+        synced += res.synced;
+        created += res.created;
+        allSkus.push(...res.skus);
+      } catch (err) {
+        emit({ type: 'log', message: `  ✗ ${target.label}: ${err.message}` });
+      }
+      if (i < selected.length - 1) await sleep(CRAWL_DELAY);
+    }
+
+    if (session.loggedIn && allSkus.length > 0) {
+      emit({ type: 'log', message: `Récupération des prix client pour ${allSkus.length} article(s)…` });
+      priced = await importPrices(session, allSkus, emit);
+      emit({ type: 'log', message: `✓ ${priced} prix client enregistrés.` });
+    }
+
+    const duration = ((Date.now() - start) / 1000).toFixed(1);
+    await logSync('success', synced, null);
+    const result = {
+      success: true,
+      itemsCount: synced,
+      createdCount: created,
+      updatedCount: synced - created,
+      pricedCount: priced,
+      duration,
+      error: null,
+    };
+    emit({ type: 'progress', current: selected.length, total: selected.length, collection: 'Leuba Hiag' });
+    emit({ type: 'complete', result });
+    return result;
+  } catch (err) {
+    await logSync('error', synced, err.message);
+    emit({ type: 'error', error: err.message });
+    return { success: false, itemsCount: synced, error: err.message };
+  }
+}