apex/server/services/leubaHiagCrawler.js

/**
 * Leuba Hiag catalog crawler.
 *
 * The Leuba Hiag webshop (https://www.leubahiag.ch) is a Laravel/Bagisto shop whose
 * product catalog is served by Algolia. The public catalog search page embeds a
 * short-lived "secured" Algolia API key (guest scope) inside the page markup. We read
 * that key live on every run (it expires), then page through the Algolia index and
 * upsert each product into the shared `formex_materials` table under the `leubahiag`
 * supplier.
 *
 * Notes:
 * - Algolia exposes the product catalog but NOT prices. Customer-specific prices come
 *   from the authenticated `/fr/sales/prices-and-availability` endpoint: we log in with
 *   the account credentials (LEUBAHIAG_EMAIL/PASSWORD) and, after importing each line,
 *   fetch the per-SKU panel price and store it in price_chf.
 * - Imported items are inactive by default so the catalog rows don't flood the
 *   configurator finish picker. Activate the wanted decors from the Matériaux admin page.
 */

import pool, { run } from '../db/index.js';

const BASE_URL = 'https://www.leubahiag.ch';
const SEARCH_URL = `${BASE_URL}/fr/catalog/search`;
const CRAWL_DELAY = parseInt(process.env.LEUBAHIAG_CRAWL_DELAY_MS || '300', 10);
const ALGOLIA_HITS_PER_PAGE = 1000;

/**
 * Curated catalog lines to import (per the supplier's agreed assortment).
 * Each target maps to an Algolia query and/or brand facet filter. The `label` is used
 * as the material `collection` so the configurator groups items by recognisable line.
 */
export const LEUBA_HIAG_TARGETS = [
  { key: 'pfleiderer-xpsm', label: 'Pfleiderer XPSM (Antifingerprint)', tier: 'Entrée de gamme', query: 'XPSM', brands: ['Pfleiderer'] },
  { key: 'swiss-krono', label: 'Swiss Krono', tier: 'Entrée de gamme', query: '', brands: ['Swiss Krono'] },
  { key: 'unilin-master-oak', label: 'Unilin Master Oak', tier: 'Milieu de gamme', query: 'Master Oak', brands: ['UNILIN'] },
  { key: 'shinnoki', label: 'Shinnoki', tier: 'Milieu de gamme', query: 'Shinnoki', brands: [] },
  { key: 'cleaf', label: 'Cleaf', tier: 'Haut de gamme', query: '', brands: ['CLEAF'] },
  { key: 'valchromat', label: 'Valchromat', tier: 'Haut de gamme', query: '', brands: ['Valchromat'] },
];

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function browserHeaders(extra = {}) {
  return {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'fr-CH,fr;q=0.9,en;q=0.5',
    ...extra,
  };
}

function absoluteUrl(url) {
  if (!url) return null;
  if (url.startsWith('http://') || url.startsWith('https://')) return url;
  return `${BASE_URL}${url.startsWith('/') ? '' : '/'}${url}`;
}

/** Parse a "CHF 1'234.50" string into a number, or null. */
function parseCHF(text) {
  if (typeof text !== 'string') return null;
  const clean = text.replace(/CHF/gi, '').replace(/[\u00a0\u202f'’\s]/g, '').replace(',', '.');
  const n = parseFloat(clean);
  return Number.isFinite(n) ? n : null;
}

/**
 * Authenticated Leuba Hiag (Laravel) session. Used to read customer-specific prices
 * from the `prices-and-availability` endpoint (axios sends the XSRF-TOKEN cookie back
 * as an X-XSRF-TOKEN header — we replicate that here).
 */
class LeubaHiagSession {
  constructor() {
    this.cookies = new Map();
    this.loggedIn = false;
  }

  captureCookies(response) {
    let raw = [];
    if (typeof response.headers.getSetCookie === 'function') {
      raw = response.headers.getSetCookie();
    } else {
      const header = response.headers.get('set-cookie');
      if (header) raw = header.split(/,\s*(?=[A-Za-z0-9_\-]+=)/);
    }
    for (const entry of raw) {
      const [pair] = entry.split(';');
      const eq = pair.indexOf('=');
      if (eq > 0) this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim());
    }
  }

  cookieHeader() {
    return [...this.cookies.entries()].map(([k, v]) => `${k}=${v}`).join('; ');
  }

  xsrfToken() {
    const raw = this.cookies.get('XSRF-TOKEN');
    return raw ? decodeURIComponent(raw) : '';
  }

  async login(emit = () => {}, credentials = {}) {
    const email = credentials.username || credentials.email
      || process.env.LEUBAHIAG_EMAIL || process.env.LEUBAHIAG_USER;
    const password = credentials.password || process.env.LEUBAHIAG_PASSWORD;
    if (!email || !password) {
      emit({ type: 'log', message: 'Identifiants Leuba Hiag absents : import sans prix.' });
      return false;
    }

    const loginUrl = `${BASE_URL}/fr/customer/auth/login`;
    try {
      const getRes = await fetch(loginUrl, { headers: browserHeaders(), redirect: 'follow' });
      this.captureCookies(getRes);
      const html = await getRes.text();
      const token = (html.match(/name="_token"\s+value="([^"]+)"/) || [])[1];
      if (!token) {
        emit({ type: 'log', message: 'Jeton CSRF Leuba Hiag introuvable : import sans prix.' });
        return false;
      }

      const body = new URLSearchParams();
      body.set('_token', token);
      body.set('email', email);
      body.set('password', password);

      // redirect:'manual' so we capture Set-Cookie from the post-login 302 (the
      // LoggedIn/session cookies live on the redirect response, not the final page).
      const postRes = await fetch(loginUrl, {
        method: 'POST',
        headers: browserHeaders({
          'Content-Type': 'application/x-www-form-urlencoded',
          Referer: loginUrl,
          Origin: BASE_URL,
          Cookie: this.cookieHeader(),
        }),
        body: body.toString(),
        redirect: 'manual',
      });
      this.captureCookies(postRes);

      // Confirm the session and refresh the XSRF-TOKEN from the authenticated catalog page.
      const verifyRes = await fetch(`${BASE_URL}/fr/catalog/search`, {
        headers: browserHeaders({ Cookie: this.cookieHeader() }),
        redirect: 'follow',
      });
      this.captureCookies(verifyRes);
      const verifyHtml = await verifyRes.text();
      const isLoggedIn = /\\u0022isLoggedIn\\u0022:true/.test(verifyHtml)
        || this.cookies.get('LoggedIn') === '1';

      this.loggedIn = isLoggedIn;
      emit({
        type: 'log',
        message: this.loggedIn
          ? `Connecté à Leuba Hiag (${email}).`
          : 'Connexion Leuba Hiag non confirmée : import sans prix.',
      });
      return this.loggedIn;
    } catch (err) {
      emit({ type: 'log', message: `Erreur de connexion Leuba Hiag : ${err.message}` });
      return false;
    }
  }

  /** POST {skus} to the prices endpoint; returns the per-sku price map (or {}). */
  async fetchPrices(skus) {
    if (!this.loggedIn || skus.length === 0) return {};
    const res = await fetch(`${BASE_URL}/fr/sales/prices-and-availability`, {
      method: 'POST',
      headers: browserHeaders({
        'Content-Type': 'application/json',
        Accept: 'application/json, text/plain, */*',
        'X-Requested-With': 'XMLHttpRequest',
        'X-XSRF-TOKEN': this.xsrfToken(),
        Referer: `${BASE_URL}/fr/catalog/search`,
        Origin: BASE_URL,
        Cookie: this.cookieHeader(),
      }),
      body: JSON.stringify({ skus }),
    });
    this.captureCookies(res);
    if (!res.ok) {
      throw new Error(`prices-and-availability HTTP ${res.status}`);
    }
    return res.json();
  }
}

/**
 * Read the live Algolia connection config (appId, index, secured apiKey) from the
 * catalog search page. The key is embedded as `\u0022apiKey\u0022:\u0022...\u0022`
 * inside the `window.globalSearchConfigBase` JSON blob.
 */
async function fetchAlgoliaConfig(emit = () => {}) {
  const res = await fetch(SEARCH_URL, { headers: browserHeaders(), redirect: 'follow' });
  if (!res.ok) {
    throw new Error(`Impossible de charger la page catalogue Leuba Hiag (HTTP ${res.status}).`);
  }
  const html = await res.text();

  const appId = (html.match(/\\u0022appId\\u0022:\\u0022([A-Za-z0-9]+)\\u0022/) || [])[1]
    || process.env.LEUBAHIAG_ALGOLIA_APP_ID;
  const indexName = (html.match(/\\u0022indexName\\u0022:\\u0022([A-Za-z0-9_]+)\\u0022/) || [])[1]
    || process.env.LEUBAHIAG_ALGOLIA_INDEX;
  const apiKey = (html.match(/\\u0022apiKey\\u0022:\\u0022([A-Za-z0-9+/=]+)\\u0022/) || [])[1];

  if (!appId || !indexName || !apiKey) {
    throw new Error('Clé Algolia introuvable sur la page catalogue Leuba Hiag (structure du site modifiée ?).');
  }

  emit({ type: 'log', message: `Catalogue Algolia détecté: index ${indexName} (app ${appId}).` });
  return { appId, indexName, apiKey };
}

/**
 * Run one Algolia query page. `facetFilters` is an array of arrays per Algolia syntax.
 */
async function queryAlgolia({ appId, indexName, apiKey }, { query = '', page = 0, hitsPerPage = ALGOLIA_HITS_PER_PAGE, facetFilters = [] } = {}) {
  const url = `https://${appId}-dsn.algolia.net/1/indexes/${indexName}/query`;
  const params = new URLSearchParams();
  params.set('query', query);
  params.set('hitsPerPage', String(hitsPerPage));
  params.set('page', String(page));
  if (facetFilters.length > 0) {
    params.set('facetFilters', JSON.stringify(facetFilters));
  }

  const res = await fetch(url, {
    method: 'POST',
    headers: {
      'X-Algolia-Application-Id': appId,
      'X-Algolia-API-Key': apiKey,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({ params: params.toString() }),
  });

  if (!res.ok) {
    const body = await res.text().catch(() => '');
    throw new Error(`Requête Algolia échouée (HTTP ${res.status})${body ? `: ${body.slice(0, 160)}` : ''}`);
  }
  return res.json();
}

function firstString(value) {
  if (Array.isArray(value)) return value.find((v) => typeof v === 'string') || null;
  return typeof value === 'string' ? value : null;
}

function toNumber(value) {
  const n = Number(value);
  return Number.isFinite(n) ? n : null;
}

/**
 * Map an Algolia product record to a `formex_materials` row.
 * Dimension fields from the index: thickness (mm), length (mm), width (mm).
 * @param {string} [collectionOverride] Force the `collection` (curated line label).
 */
function mapHit(hit, collectionOverride) {
  const sku = String(hit.sku || hit.objectID || '').trim();
  if (!sku) return null;

  const categoryPath = Array.isArray(hit.categories_with_path) ? hit.categories_with_path : [];
  const category = categoryPath.length > 0
    ? categoryPath[categoryPath.length - 1]
    : (firstString(hit.categories?.level0) || null);

  const collection = collectionOverride
    || firstString(hit.brand)
    || firstString(hit.categories?.level0)
    || 'Leuba Hiag';

  return {
    sku,
    materialCode: `LH-${sku}`,
    name: hit.name || hit.group_name || sku,
    collection,
    category,
    width_mm: toNumber(hit.width),
    height_mm: toNumber(hit.length),
    thickness_mm: toNumber(hit.thickness),
    image_url: absoluteUrl(hit.image_url),
    formex_url: absoluteUrl(hit.url),
    price_chf: null,
    price_unit: null,
  };
}

async function ensureSupplier() {
  const existing = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['leubahiag']);
  if (existing.rows.length > 0) return existing.rows[0].id;

  await run(
    'INSERT INTO suppliers (code, name, email, type, active) VALUES (?, ?, ?, ?, ?)',
    ['leubahiag', 'Leuba Hiag', '[email protected]', 'materials', 1]
  );
  const created = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['leubahiag']);
  return created.rows[0]?.id || null;
}

/**
 * Upsert a material. Existing rows keep their `active` flag (so admin activations are
 * preserved across re-syncs); new rows are inserted with the provided `active` value.
 */
async function upsertMaterial(material, supplierId, activeForNew) {
  const existing = await pool.query(
    'SELECT id FROM formex_materials WHERE formex_code = ?',
    [material.materialCode]
  );

  if (existing.rows.length > 0) {
    await run(
      `UPDATE formex_materials
       SET name = ?, collection = ?, category = ?, width_mm = ?, height_mm = ?, thickness_mm = ?,
           image_url = ?, formex_url = ?,
           supplier_id = COALESCE(?, supplier_id), last_synced_at = datetime('now')
       WHERE formex_code = ?`,
      [
        material.name,
        material.collection,
        material.category,
        material.width_mm,
        material.height_mm,
        material.thickness_mm,
        material.image_url,
        material.formex_url,
        supplierId,
        material.materialCode,
      ]
    );
    return { id: existing.rows[0].id, created: false };
  }

  await run(
    `INSERT INTO formex_materials
      (formex_code, name, collection, category, width_mm, height_mm, thickness_mm,
       image_url, formex_url, price_chf, price_unit, supplier_id, active, last_synced_at)
     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
    [
      material.materialCode,
      material.name,
      material.collection,
      material.category,
      material.width_mm,
      material.height_mm,
      material.thickness_mm,
      material.image_url,
      material.formex_url,
      material.price_chf ?? null,
      material.price_unit ?? null,
      supplierId,
      activeForNew ? 1 : 0,
    ]
  );

  const created = await pool.query('SELECT id FROM formex_materials WHERE formex_code = ?', [material.materialCode]);
  return { id: created.rows[0]?.id || null, created: true };
}

/** Update the price of an already-imported material by its SKU. */
async function updateMaterialPrice(sku, priceChf, priceUnit) {
  await run(
    `UPDATE formex_materials
     SET price_chf = ?, price_unit = COALESCE(?, price_unit), last_synced_at = datetime('now')
     WHERE formex_code = ?`,
    [priceChf, priceUnit, `LH-${sku}`]
  );
}

async function logSync(status, itemsCount, error = null) {
  await run(
    'INSERT INTO sync_logs (source, status, items_count, error) VALUES (?, ?, ?, ?)',
    ['leubahiag', status, itemsCount, error]
  );
}

/**
 * Import all products matching one curated target (a single catalog line).
 */
async function importTarget(config, supplierId, target, activate, emit) {
  const facetFilters = Array.isArray(target.brands) && target.brands.length > 0
    ? [target.brands.map((b) => `brand:${b}`)]
    : [];

  let synced = 0;
  let created = 0;
  let page = 0;
  let nbPages = 1;
  let nbHits = 0;
  const skus = [];

  do {
    if (page > 0) await sleep(CRAWL_DELAY);
    const data = await queryAlgolia(config, { query: target.query, page, facetFilters });
    nbPages = data.nbPages || 1;
    nbHits = data.nbHits || 0;

    if (page === 0) {
      emit({ type: 'log', message: `→ ${target.label} (${target.tier}) : ${nbHits} produit(s).` });
    }

    const hits = Array.isArray(data.hits) ? data.hits : [];
    for (const hit of hits) {
      const material = mapHit(hit, target.label);
      if (!material) continue;
      try {
        const result = await upsertMaterial(material, supplierId, activate);
        synced++;
        if (result.created) created++;
        skus.push(material.sku);
      } catch (err) {
        emit({ type: 'log', message: `  ✗ ${material.materialCode}: ${err.message}` });
      }
    }
    page++;
  } while (page < nbPages);

  emit({ type: 'log', message: `  ✓ ${target.label} : ${synced} importé(s) (${created} nouveau(x)).` });
  return { synced, created, skus };
}

/**
 * Fetch and store customer prices for a list of SKUs (in batches).
 * Returns the number of items priced.
 */
async function importPrices(session, skus, emit) {
  const BATCH = 50;
  let priced = 0;
  for (let i = 0; i < skus.length; i += BATCH) {
    const batch = skus.slice(i, i + BATCH);
    try {
      const data = await session.fetchPrices(batch);
      for (const sku of batch) {
        const info = data?.[sku];
        if (!info) continue;
        const tier = Array.isArray(info.tierprices) ? info.tierprices[0] : null;
        const priceChf = parseCHF(tier?.price);
        if (priceChf == null) continue;
        await updateMaterialPrice(sku, priceChf, 'pièce');
        priced++;
      }
    } catch (err) {
      emit({ type: 'log', message: `  ✗ Prix lot ${i / BATCH + 1}: ${err.message}` });
    }
    emit({ type: 'log', message: `  Prix: ${priced}/${skus.length} récupérés…` });
    if (i + BATCH < skus.length) await sleep(CRAWL_DELAY);
  }
  return priced;
}

/**
 * Synchronise the curated Leuba Hiag catalog lines via Algolia.
 *
 * @param {object}   options
 * @param {string[]} [options.targets]      Subset of target keys to import (default: all).
 * @param {boolean}  [options.activate]     Mark newly imported items active (default false).
 * @param {boolean}  [options.fetchPrices]  Log in and import customer prices (default true).
 * @param {object}   [options.credentials]  { username/email, password } (else env vars).
 * @param {function} emit                   SSE-style event emitter.
 */
export async function syncLeubaHiag(options = {}, emit = () => {}) {
  const { targets = null, activate = false, fetchPrices = true, credentials = {} } = options;

  const selected = Array.isArray(targets) && targets.length > 0
    ? LEUBA_HIAG_TARGETS.filter((t) => targets.includes(t.key))
    : LEUBA_HIAG_TARGETS;

  const start = Date.now();
  let synced = 0;
  let created = 0;
  let priced = 0;

  try {
    if (selected.length === 0) {
      throw new Error('Aucune gamme Leuba Hiag sélectionnée.');
    }

    const config = await fetchAlgoliaConfig(emit);
    const supplierId = await ensureSupplier();

    // Log in up-front so prices can be fetched after each line is imported.
    const session = new LeubaHiagSession();
    if (fetchPrices) {
      await session.login(emit, credentials);
    }

    emit({ type: 'log', message: `${selected.length} gamme(s) à importer.` });
    emit({ type: 'progress', current: 0, total: selected.length, collection: 'Leuba Hiag' });

    const allSkus = [];
    for (let i = 0; i < selected.length; i++) {
      const target = selected[i];
      emit({ type: 'progress', current: i, total: selected.length, collection: target.label });
      try {
        const res = await importTarget(config, supplierId, target, activate, emit);
        synced += res.synced;
        created += res.created;
        allSkus.push(...res.skus);
      } catch (err) {
        emit({ type: 'log', message: `  ✗ ${target.label}: ${err.message}` });
      }
      if (i < selected.length - 1) await sleep(CRAWL_DELAY);
    }

    if (session.loggedIn && allSkus.length > 0) {
      emit({ type: 'log', message: `Récupération des prix client pour ${allSkus.length} article(s)…` });
      priced = await importPrices(session, allSkus, emit);
      emit({ type: 'log', message: `✓ ${priced} prix client enregistrés.` });
    }

    const duration = ((Date.now() - start) / 1000).toFixed(1);
    await logSync('success', synced, null);
    const result = {
      success: true,
      itemsCount: synced,
      createdCount: created,
      updatedCount: synced - created,
      pricedCount: priced,
      duration,
      error: null,
    };
    emit({ type: 'progress', current: selected.length, total: selected.length, collection: 'Leuba Hiag' });
    emit({ type: 'complete', result });
    return result;
  } catch (err) {
    await logSync('error', synced, err.message);
    emit({ type: 'error', error: err.message });
    return { success: false, itemsCount: synced, error: err.message };
  }
}