/** * Leuba Hiag catalog crawler. * * The Leuba Hiag webshop (https://www.leubahiag.ch) is a Laravel/Bagisto shop whose * product catalog is served by Algolia. The public catalog search page embeds a * short-lived "secured" Algolia API key (guest scope) inside the page markup. We read * that key live on every run (it expires), then page through the Algolia index and * upsert each product into the shared `formex_materials` table under the `leubahiag` * supplier. * * Notes: * - Algolia exposes the product catalog but NOT prices. Customer-specific prices come * from the authenticated `/fr/sales/prices-and-availability` endpoint: we log in with * the account credentials (LEUBAHIAG_EMAIL/PASSWORD) and, after importing each line, * fetch the per-SKU panel price and store it in price_chf. * - Imported items are inactive by default so the catalog rows don't flood the * configurator finish picker. Activate the wanted decors from the Matériaux admin page. */ import pool, { run } from '../db/index.js'; const BASE_URL = 'https://www.leubahiag.ch'; const SEARCH_URL = `${BASE_URL}/fr/catalog/search`; const CRAWL_DELAY = parseInt(process.env.LEUBAHIAG_CRAWL_DELAY_MS || '300', 10); const ALGOLIA_HITS_PER_PAGE = 1000; /** * Curated catalog lines to import (per the supplier's agreed assortment). * Each target maps to an Algolia query and/or brand facet filter. The `label` is used * as the material `collection` so the configurator groups items by recognisable line. */ export const LEUBA_HIAG_TARGETS = [ { key: 'pfleiderer-xpsm', label: 'Pfleiderer XPSM (Antifingerprint)', tier: 'Entrée de gamme', query: 'XPSM', brands: ['Pfleiderer'] }, { key: 'swiss-krono', label: 'Swiss Krono', tier: 'Entrée de gamme', query: '', brands: ['Swiss Krono'] }, { key: 'unilin-master-oak', label: 'Unilin Master Oak', tier: 'Milieu de gamme', query: 'Master Oak', brands: ['UNILIN'] }, { key: 'shinnoki', label: 'Shinnoki', tier: 'Milieu de gamme', query: 'Shinnoki', brands: [] }, { key: 'cleaf', label: 'Cleaf', tier: 'Haut de gamme', query: '', brands: ['CLEAF'] }, { key: 'valchromat', label: 'Valchromat', tier: 'Haut de gamme', query: '', brands: ['Valchromat'] }, ]; function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } function browserHeaders(extra = {}) { return { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'fr-CH,fr;q=0.9,en;q=0.5', ...extra, }; } function absoluteUrl(url) { if (!url) return null; if (url.startsWith('http://') || url.startsWith('https://')) return url; return `${BASE_URL}${url.startsWith('/') ? '' : '/'}${url}`; } /** Parse a "CHF 1'234.50" string into a number, or null. */ function parseCHF(text) { if (typeof text !== 'string') return null; const clean = text.replace(/CHF/gi, '').replace(/[\u00a0\u202f'’\s]/g, '').replace(',', '.'); const n = parseFloat(clean); return Number.isFinite(n) ? n : null; } /** * Authenticated Leuba Hiag (Laravel) session. Used to read customer-specific prices * from the `prices-and-availability` endpoint (axios sends the XSRF-TOKEN cookie back * as an X-XSRF-TOKEN header — we replicate that here). */ class LeubaHiagSession { constructor() { this.cookies = new Map(); this.loggedIn = false; } captureCookies(response) { let raw = []; if (typeof response.headers.getSetCookie === 'function') { raw = response.headers.getSetCookie(); } else { const header = response.headers.get('set-cookie'); if (header) raw = header.split(/,\s*(?=[A-Za-z0-9_\-]+=)/); } for (const entry of raw) { const [pair] = entry.split(';'); const eq = pair.indexOf('='); if (eq > 0) this.cookies.set(pair.slice(0, eq).trim(), pair.slice(eq + 1).trim()); } } cookieHeader() { return [...this.cookies.entries()].map(([k, v]) => `${k}=${v}`).join('; '); } xsrfToken() { const raw = this.cookies.get('XSRF-TOKEN'); return raw ? decodeURIComponent(raw) : ''; } async login(emit = () => {}, credentials = {}) { const email = credentials.username || credentials.email || process.env.LEUBAHIAG_EMAIL || process.env.LEUBAHIAG_USER; const password = credentials.password || process.env.LEUBAHIAG_PASSWORD; if (!email || !password) { emit({ type: 'log', message: 'Identifiants Leuba Hiag absents : import sans prix.' }); return false; } const loginUrl = `${BASE_URL}/fr/customer/auth/login`; try { const getRes = await fetch(loginUrl, { headers: browserHeaders(), redirect: 'follow' }); this.captureCookies(getRes); const html = await getRes.text(); const token = (html.match(/name="_token"\s+value="([^"]+)"/) || [])[1]; if (!token) { emit({ type: 'log', message: 'Jeton CSRF Leuba Hiag introuvable : import sans prix.' }); return false; } const body = new URLSearchParams(); body.set('_token', token); body.set('email', email); body.set('password', password); // redirect:'manual' so we capture Set-Cookie from the post-login 302 (the // LoggedIn/session cookies live on the redirect response, not the final page). const postRes = await fetch(loginUrl, { method: 'POST', headers: browserHeaders({ 'Content-Type': 'application/x-www-form-urlencoded', Referer: loginUrl, Origin: BASE_URL, Cookie: this.cookieHeader(), }), body: body.toString(), redirect: 'manual', }); this.captureCookies(postRes); // Confirm the session and refresh the XSRF-TOKEN from the authenticated catalog page. const verifyRes = await fetch(`${BASE_URL}/fr/catalog/search`, { headers: browserHeaders({ Cookie: this.cookieHeader() }), redirect: 'follow', }); this.captureCookies(verifyRes); const verifyHtml = await verifyRes.text(); const isLoggedIn = /\\u0022isLoggedIn\\u0022:true/.test(verifyHtml) || this.cookies.get('LoggedIn') === '1'; this.loggedIn = isLoggedIn; emit({ type: 'log', message: this.loggedIn ? `Connecté à Leuba Hiag (${email}).` : 'Connexion Leuba Hiag non confirmée : import sans prix.', }); return this.loggedIn; } catch (err) { emit({ type: 'log', message: `Erreur de connexion Leuba Hiag : ${err.message}` }); return false; } } /** POST {skus} to the prices endpoint; returns the per-sku price map (or {}). */ async fetchPrices(skus) { if (!this.loggedIn || skus.length === 0) return {}; const res = await fetch(`${BASE_URL}/fr/sales/prices-and-availability`, { method: 'POST', headers: browserHeaders({ 'Content-Type': 'application/json', Accept: 'application/json, text/plain, */*', 'X-Requested-With': 'XMLHttpRequest', 'X-XSRF-TOKEN': this.xsrfToken(), Referer: `${BASE_URL}/fr/catalog/search`, Origin: BASE_URL, Cookie: this.cookieHeader(), }), body: JSON.stringify({ skus }), }); this.captureCookies(res); if (!res.ok) { throw new Error(`prices-and-availability HTTP ${res.status}`); } return res.json(); } } /** * Read the live Algolia connection config (appId, index, secured apiKey) from the * catalog search page. The key is embedded as `\u0022apiKey\u0022:\u0022...\u0022` * inside the `window.globalSearchConfigBase` JSON blob. */ async function fetchAlgoliaConfig(emit = () => {}) { const res = await fetch(SEARCH_URL, { headers: browserHeaders(), redirect: 'follow' }); if (!res.ok) { throw new Error(`Impossible de charger la page catalogue Leuba Hiag (HTTP ${res.status}).`); } const html = await res.text(); const appId = (html.match(/\\u0022appId\\u0022:\\u0022([A-Za-z0-9]+)\\u0022/) || [])[1] || process.env.LEUBAHIAG_ALGOLIA_APP_ID; const indexName = (html.match(/\\u0022indexName\\u0022:\\u0022([A-Za-z0-9_]+)\\u0022/) || [])[1] || process.env.LEUBAHIAG_ALGOLIA_INDEX; const apiKey = (html.match(/\\u0022apiKey\\u0022:\\u0022([A-Za-z0-9+/=]+)\\u0022/) || [])[1]; if (!appId || !indexName || !apiKey) { throw new Error('Clé Algolia introuvable sur la page catalogue Leuba Hiag (structure du site modifiée ?).'); } emit({ type: 'log', message: `Catalogue Algolia détecté: index ${indexName} (app ${appId}).` }); return { appId, indexName, apiKey }; } /** * Run one Algolia query page. `facetFilters` is an array of arrays per Algolia syntax. */ async function queryAlgolia({ appId, indexName, apiKey }, { query = '', page = 0, hitsPerPage = ALGOLIA_HITS_PER_PAGE, facetFilters = [] } = {}) { const url = `https://${appId}-dsn.algolia.net/1/indexes/${indexName}/query`; const params = new URLSearchParams(); params.set('query', query); params.set('hitsPerPage', String(hitsPerPage)); params.set('page', String(page)); if (facetFilters.length > 0) { params.set('facetFilters', JSON.stringify(facetFilters)); } const res = await fetch(url, { method: 'POST', headers: { 'X-Algolia-Application-Id': appId, 'X-Algolia-API-Key': apiKey, 'Content-Type': 'application/json', }, body: JSON.stringify({ params: params.toString() }), }); if (!res.ok) { const body = await res.text().catch(() => ''); throw new Error(`Requête Algolia échouée (HTTP ${res.status})${body ? `: ${body.slice(0, 160)}` : ''}`); } return res.json(); } function firstString(value) { if (Array.isArray(value)) return value.find((v) => typeof v === 'string') || null; return typeof value === 'string' ? value : null; } function toNumber(value) { const n = Number(value); return Number.isFinite(n) ? n : null; } /** * Map an Algolia product record to a `formex_materials` row. * Dimension fields from the index: thickness (mm), length (mm), width (mm). * @param {string} [collectionOverride] Force the `collection` (curated line label). */ function mapHit(hit, collectionOverride) { const sku = String(hit.sku || hit.objectID || '').trim(); if (!sku) return null; const categoryPath = Array.isArray(hit.categories_with_path) ? hit.categories_with_path : []; const category = categoryPath.length > 0 ? categoryPath[categoryPath.length - 1] : (firstString(hit.categories?.level0) || null); const collection = collectionOverride || firstString(hit.brand) || firstString(hit.categories?.level0) || 'Leuba Hiag'; return { sku, materialCode: `LH-${sku}`, name: hit.name || hit.group_name || sku, collection, category, width_mm: toNumber(hit.width), height_mm: toNumber(hit.length), thickness_mm: toNumber(hit.thickness), image_url: absoluteUrl(hit.image_url), formex_url: absoluteUrl(hit.url), price_chf: null, price_unit: null, }; } async function ensureSupplier() { const existing = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['leubahiag']); if (existing.rows.length > 0) return existing.rows[0].id; await run( 'INSERT INTO suppliers (code, name, email, type, active) VALUES (?, ?, ?, ?, ?)', ['leubahiag', 'Leuba Hiag', 'info@leubahiag.ch', 'materials', 1] ); const created = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['leubahiag']); return created.rows[0]?.id || null; } /** * Upsert a material. Existing rows keep their `active` flag (so admin activations are * preserved across re-syncs); new rows are inserted with the provided `active` value. */ async function upsertMaterial(material, supplierId, activeForNew) { const existing = await pool.query( 'SELECT id FROM formex_materials WHERE formex_code = ?', [material.materialCode] ); if (existing.rows.length > 0) { await run( `UPDATE formex_materials SET name = ?, collection = ?, category = ?, width_mm = ?, height_mm = ?, thickness_mm = ?, image_url = ?, formex_url = ?, supplier_id = COALESCE(?, supplier_id), last_synced_at = datetime('now') WHERE formex_code = ?`, [ material.name, material.collection, material.category, material.width_mm, material.height_mm, material.thickness_mm, material.image_url, material.formex_url, supplierId, material.materialCode, ] ); return { id: existing.rows[0].id, created: false }; } await run( `INSERT INTO formex_materials (formex_code, name, collection, category, width_mm, height_mm, thickness_mm, image_url, formex_url, price_chf, price_unit, supplier_id, active, last_synced_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`, [ material.materialCode, material.name, material.collection, material.category, material.width_mm, material.height_mm, material.thickness_mm, material.image_url, material.formex_url, material.price_chf ?? null, material.price_unit ?? null, supplierId, activeForNew ? 1 : 0, ] ); const created = await pool.query('SELECT id FROM formex_materials WHERE formex_code = ?', [material.materialCode]); return { id: created.rows[0]?.id || null, created: true }; } /** Update the price of an already-imported material by its SKU. */ async function updateMaterialPrice(sku, priceChf, priceUnit) { await run( `UPDATE formex_materials SET price_chf = ?, price_unit = COALESCE(?, price_unit), last_synced_at = datetime('now') WHERE formex_code = ?`, [priceChf, priceUnit, `LH-${sku}`] ); } async function logSync(status, itemsCount, error = null) { await run( 'INSERT INTO sync_logs (source, status, items_count, error) VALUES (?, ?, ?, ?)', ['leubahiag', status, itemsCount, error] ); } /** * Import all products matching one curated target (a single catalog line). */ async function importTarget(config, supplierId, target, activate, emit) { const facetFilters = Array.isArray(target.brands) && target.brands.length > 0 ? [target.brands.map((b) => `brand:${b}`)] : []; let synced = 0; let created = 0; let page = 0; let nbPages = 1; let nbHits = 0; const skus = []; do { if (page > 0) await sleep(CRAWL_DELAY); const data = await queryAlgolia(config, { query: target.query, page, facetFilters }); nbPages = data.nbPages || 1; nbHits = data.nbHits || 0; if (page === 0) { emit({ type: 'log', message: `→ ${target.label} (${target.tier}) : ${nbHits} produit(s).` }); } const hits = Array.isArray(data.hits) ? data.hits : []; for (const hit of hits) { const material = mapHit(hit, target.label); if (!material) continue; try { const result = await upsertMaterial(material, supplierId, activate); synced++; if (result.created) created++; skus.push(material.sku); } catch (err) { emit({ type: 'log', message: ` ✗ ${material.materialCode}: ${err.message}` }); } } page++; } while (page < nbPages); emit({ type: 'log', message: ` ✓ ${target.label} : ${synced} importé(s) (${created} nouveau(x)).` }); return { synced, created, skus }; } /** * Fetch and store customer prices for a list of SKUs (in batches). * Returns the number of items priced. */ async function importPrices(session, skus, emit) { const BATCH = 50; let priced = 0; for (let i = 0; i < skus.length; i += BATCH) { const batch = skus.slice(i, i + BATCH); try { const data = await session.fetchPrices(batch); for (const sku of batch) { const info = data?.[sku]; if (!info) continue; const tier = Array.isArray(info.tierprices) ? info.tierprices[0] : null; const priceChf = parseCHF(tier?.price); if (priceChf == null) continue; await updateMaterialPrice(sku, priceChf, 'pièce'); priced++; } } catch (err) { emit({ type: 'log', message: ` ✗ Prix lot ${i / BATCH + 1}: ${err.message}` }); } emit({ type: 'log', message: ` Prix: ${priced}/${skus.length} récupérés…` }); if (i + BATCH < skus.length) await sleep(CRAWL_DELAY); } return priced; } /** * Synchronise the curated Leuba Hiag catalog lines via Algolia. * * @param {object} options * @param {string[]} [options.targets] Subset of target keys to import (default: all). * @param {boolean} [options.activate] Mark newly imported items active (default false). * @param {boolean} [options.fetchPrices] Log in and import customer prices (default true). * @param {object} [options.credentials] { username/email, password } (else env vars). * @param {function} emit SSE-style event emitter. */ export async function syncLeubaHiag(options = {}, emit = () => {}) { const { targets = null, activate = false, fetchPrices = true, credentials = {} } = options; const selected = Array.isArray(targets) && targets.length > 0 ? LEUBA_HIAG_TARGETS.filter((t) => targets.includes(t.key)) : LEUBA_HIAG_TARGETS; const start = Date.now(); let synced = 0; let created = 0; let priced = 0; try { if (selected.length === 0) { throw new Error('Aucune gamme Leuba Hiag sélectionnée.'); } const config = await fetchAlgoliaConfig(emit); const supplierId = await ensureSupplier(); // Log in up-front so prices can be fetched after each line is imported. const session = new LeubaHiagSession(); if (fetchPrices) { await session.login(emit, credentials); } emit({ type: 'log', message: `${selected.length} gamme(s) à importer.` }); emit({ type: 'progress', current: 0, total: selected.length, collection: 'Leuba Hiag' }); const allSkus = []; for (let i = 0; i < selected.length; i++) { const target = selected[i]; emit({ type: 'progress', current: i, total: selected.length, collection: target.label }); try { const res = await importTarget(config, supplierId, target, activate, emit); synced += res.synced; created += res.created; allSkus.push(...res.skus); } catch (err) { emit({ type: 'log', message: ` ✗ ${target.label}: ${err.message}` }); } if (i < selected.length - 1) await sleep(CRAWL_DELAY); } if (session.loggedIn && allSkus.length > 0) { emit({ type: 'log', message: `Récupération des prix client pour ${allSkus.length} article(s)…` }); priced = await importPrices(session, allSkus, emit); emit({ type: 'log', message: `✓ ${priced} prix client enregistrés.` }); } const duration = ((Date.now() - start) / 1000).toFixed(1); await logSync('success', synced, null); const result = { success: true, itemsCount: synced, createdCount: created, updatedCount: synced - created, pricedCount: priced, duration, error: null, }; emit({ type: 'progress', current: selected.length, total: selected.length, collection: 'Leuba Hiag' }); emit({ type: 'complete', result }); return result; } catch (err) { await logSync('error', synced, err.message); emit({ type: 'error', error: err.message }); return { success: false, itemsCount: synced, error: err.message }; } }