852 lines
30 KiB
JavaScript
852 lines
30 KiB
JavaScript
import * as cheerio from 'cheerio';
|
||
import pool, { run } from '../db/index.js';
|
||
|
||
const BASE_URL = 'https://www.haefele.ch';
|
||
const CRAWL_DELAY = parseInt(process.env.HAFELE_CRAWL_DELAY_MS || '500', 10);
|
||
const BROWSER_TIMEOUT_MS = parseInt(process.env.HAFELE_BROWSER_TIMEOUT_MS || '90000', 10);
|
||
|
||
function sleep(ms) {
|
||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||
}
|
||
|
||
function normalizeSpaces(value) {
|
||
return (value || '').replace(/\s+/g, ' ').trim();
|
||
}
|
||
|
||
function absoluteUrl(url) {
|
||
if (!url) return null;
|
||
if (url.startsWith('http://') || url.startsWith('https://')) return url;
|
||
return `${BASE_URL}${url.startsWith('/') ? '' : '/'}${url}`;
|
||
}
|
||
|
||
function parsePriceCHF(text) {
|
||
const clean = normalizeSpaces(text)
|
||
.replace(/[\u00a0\u202f]/g, ' ')
|
||
.replace(/[’']/g, '');
|
||
const match = clean.match(/([\d]+(?:[.,][\d]+)?)\s*CHF|CHF\s*([\d]+(?:[.,][\d]+)?)/i);
|
||
if (!match) return null;
|
||
return parseFloat((match[1] || match[2]).replace(',', '.'));
|
||
}
|
||
|
||
function parseFirstMm(text) {
|
||
const match = normalizeSpaces(text).match(/(\d+(?:[.,]\d+)?)\s*mm/i);
|
||
if (!match) return null;
|
||
return parseFloat(match[1].replace(',', '.'));
|
||
}
|
||
|
||
function parseBoreDiameter(text) {
|
||
const clean = normalizeSpaces(text);
|
||
const patterns = [
|
||
/[Øø∅]\s*(\d+(?:[.,]\d+)?)\s*mm/i,
|
||
/(?:trou|per[çc]age|diam[eè]tre|boring|bohrung)\s*[:\s]*[Øø∅]?\s*(\d+(?:[.,]\d+)?)\s*mm/i,
|
||
/(\d+(?:[.,]\d+)?)\s*mm\s*(?:trou|per[çc]age|boring|bohrung)/i,
|
||
];
|
||
for (const pat of patterns) {
|
||
const m = clean.match(pat);
|
||
if (m) {
|
||
const val = parseFloat((m[1] || m[2]).replace(',', '.'));
|
||
if (val > 0 && val <= 50) return val;
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
function parseBoreDepths(specs) {
|
||
const baseKeys = [
|
||
'profondeur de perçage dans le fond',
|
||
'profondeur de perçage fond',
|
||
'profondeur perçage fond',
|
||
'profondeur percage fond',
|
||
'bohrtiefe in boden',
|
||
'bohrtiefe boden',
|
||
'drilling depth base',
|
||
'profondeur de perçage dans la partie inférieure',
|
||
];
|
||
const pieceKeys = [
|
||
'profondeur de perçage dans la pièce',
|
||
'profondeur de perçage pièce',
|
||
'profondeur perçage pièce',
|
||
'profondeur percage piece',
|
||
'profondeur de perçage dans le panneau latéral',
|
||
'profondeur de perçage dans la pièce fixée',
|
||
'bohrtiefe in seitenteil',
|
||
'bohrtiefe seitenteil',
|
||
'drilling depth piece',
|
||
'profondeur de perçage dans la partie latérale',
|
||
];
|
||
|
||
let depthBase = null;
|
||
let depthPiece = null;
|
||
|
||
for (const [key, value] of Object.entries(specs)) {
|
||
const k = key.toLowerCase();
|
||
if (!depthBase) {
|
||
for (const bk of baseKeys) {
|
||
if (k.includes(bk)) { depthBase = parseFirstMm(value); break; }
|
||
}
|
||
}
|
||
if (!depthPiece) {
|
||
for (const pk of pieceKeys) {
|
||
if (k.includes(pk)) { depthPiece = parseFirstMm(value); break; }
|
||
}
|
||
}
|
||
if (depthBase && depthPiece) break;
|
||
}
|
||
|
||
// Fallback: look for a generic "profondeur de perçage" with two values
|
||
if (!depthBase && !depthPiece) {
|
||
for (const [key, value] of Object.entries(specs)) {
|
||
if (key.toLowerCase().includes('profondeur') && key.toLowerCase().includes('per')) {
|
||
const allMm = [...value.matchAll(/(\d+(?:[.,]\d+)?)\s*mm/gi)].map(m => parseFloat(m[1].replace(',', '.')));
|
||
if (allMm.length >= 2) {
|
||
depthBase = allMm[0];
|
||
depthPiece = allMm[1];
|
||
} else if (allMm.length === 1 && !depthBase) {
|
||
depthBase = allMm[0];
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
return { depthBase, depthPiece };
|
||
}
|
||
|
||
function isCloudflareChallenge(html = '') {
|
||
const lower = normalizeSpaces(html).toLowerCase();
|
||
return lower.includes('just a moment') || lower.includes('cloudflare') || lower.includes('cf-challenge');
|
||
}
|
||
|
||
class HaefeleSession {
|
||
constructor() {
|
||
this.cookies = new Map();
|
||
this.loggedIn = false;
|
||
}
|
||
|
||
captureCookies(response) {
|
||
let rawCookies = [];
|
||
|
||
if (typeof response.headers.getSetCookie === 'function') {
|
||
rawCookies = response.headers.getSetCookie();
|
||
} else {
|
||
const raw = response.headers.get('set-cookie');
|
||
if (raw) {
|
||
rawCookies = raw.split(/,\s*(?=[A-Za-z0-9_\-]+=)/);
|
||
}
|
||
}
|
||
|
||
for (const header of rawCookies) {
|
||
const [pair] = header.split(';');
|
||
const eqIdx = pair.indexOf('=');
|
||
if (eqIdx > 0) {
|
||
this.cookies.set(pair.slice(0, eqIdx).trim(), pair.slice(eqIdx + 1).trim());
|
||
}
|
||
}
|
||
}
|
||
|
||
cookieHeader() {
|
||
return [...this.cookies.entries()].map(([k, v]) => `${k}=${v}`).join('; ');
|
||
}
|
||
|
||
headers(extra = {}) {
|
||
return {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||
'Accept-Language': 'fr-CH,fr;q=0.9,en;q=0.5',
|
||
'Accept-Encoding': 'gzip, deflate, br',
|
||
'Connection': 'keep-alive',
|
||
'Cache-Control': 'no-cache',
|
||
'Pragma': 'no-cache',
|
||
'Upgrade-Insecure-Requests': '1',
|
||
'Sec-Fetch-Dest': 'document',
|
||
'Sec-Fetch-Mode': 'navigate',
|
||
'Sec-Fetch-Site': 'none',
|
||
'Sec-Fetch-User': '?1',
|
||
'sec-ch-ua': '"Google Chrome";v="125", "Chromium";v="125", "Not.A/Brand";v="24"',
|
||
'sec-ch-ua-mobile': '?0',
|
||
'sec-ch-ua-platform': '"Windows"',
|
||
...(this.cookies.size > 0 ? { Cookie: this.cookieHeader() } : {}),
|
||
...extra,
|
||
};
|
||
}
|
||
|
||
async login(emit = () => {}, credentials = {}) {
|
||
const username = credentials.username || process.env.HAFELE_USERNAME || process.env.HAFELE_USER;
|
||
const password = credentials.password || process.env.HAFELE_PASSWORD;
|
||
const customerNo = credentials.customerNo || process.env.HAFELE_CUSTOMER_NO || username;
|
||
|
||
if (!username || !password) {
|
||
emit({ type: 'log', message: 'Identifiants Häfele absents (sync publique uniquement).' });
|
||
return false;
|
||
}
|
||
|
||
const homeUrl = `${BASE_URL}/fr/`;
|
||
const accountStartUrl = `${BASE_URL}/fr/account/start/`;
|
||
|
||
try {
|
||
// 1) Load FR home page (contains header login form + synchronizer token)
|
||
const getRes = await fetch(homeUrl, {
|
||
headers: this.headers(),
|
||
redirect: 'follow',
|
||
});
|
||
this.captureCookies(getRes);
|
||
const html = await getRes.text();
|
||
const $ = cheerio.load(html);
|
||
|
||
// Prefer the known header login form from the provided markup.
|
||
let form = $('#loginWithExistingAccountForm').first();
|
||
if (!form.length) {
|
||
form = $('form').filter((_, el) => {
|
||
const id = ($(el).attr('id') || '').toLowerCase();
|
||
const action = ($(el).attr('action') || '').toLowerCase();
|
||
return id.includes('login') || action.includes('processlogin') || action.includes('/account/login');
|
||
}).first();
|
||
}
|
||
|
||
if (!form.length) {
|
||
emit({ type: 'log', message: 'Formulaire de login Häfele introuvable, continuation sans session.' });
|
||
return false;
|
||
}
|
||
|
||
const action = form.attr('action') || '/fr/account/login/';
|
||
const loginAction = absoluteUrl(action);
|
||
const payload = new URLSearchParams();
|
||
|
||
form.find('input').each((_, input) => {
|
||
const name = ($(input).attr('name') || '').trim();
|
||
const type = (($(input).attr('type') || 'text')).toLowerCase();
|
||
const value = ($(input).attr('value') || '').trim();
|
||
if (!name) return;
|
||
|
||
if (type === 'hidden') {
|
||
payload.set(name, value);
|
||
return;
|
||
}
|
||
|
||
const lowered = name.toLowerCase();
|
||
if (lowered.includes('pass')) {
|
||
payload.set(name, password);
|
||
} else if (lowered.includes('kunden') || lowered.includes('customer')) {
|
||
payload.set(name, customerNo);
|
||
} else if (lowered.includes('user') || lowered.includes('login') || lowered.includes('mitbenutzer')) {
|
||
payload.set(name, username);
|
||
}
|
||
});
|
||
|
||
// Explicit Häfele CH header login fields from current markup.
|
||
if (!payload.has('ShopLoginForm_Login_headerItemLogin')) {
|
||
payload.set('ShopLoginForm_Login_headerItemLogin', customerNo || username);
|
||
}
|
||
if (!payload.has('ShopLoginForm_Password_headerItemLogin')) {
|
||
payload.set('ShopLoginForm_Password_headerItemLogin', password);
|
||
}
|
||
if (!payload.has('ShopLoginForm_RememberLogin_headerItemLogin')) {
|
||
payload.set('ShopLoginForm_RememberLogin_headerItemLogin', 'on');
|
||
}
|
||
|
||
// Optional co-user login flow.
|
||
if (username && customerNo && username !== customerNo) {
|
||
payload.set('ShopLoginForm_LoginAsJointUser_headerItemLogin', 'on');
|
||
if (!payload.has('ShopLoginForm_UserName_headerItemLogin')) {
|
||
payload.set('ShopLoginForm_UserName_headerItemLogin', username);
|
||
}
|
||
}
|
||
|
||
const postRes = await fetch(loginAction, {
|
||
method: 'POST',
|
||
headers: this.headers({
|
||
'Content-Type': 'application/x-www-form-urlencoded',
|
||
Referer: homeUrl,
|
||
Origin: BASE_URL,
|
||
}),
|
||
body: payload.toString(),
|
||
redirect: 'follow',
|
||
});
|
||
this.captureCookies(postRes);
|
||
|
||
// 2) Verify with account start page (authenticated area)
|
||
const checkRes = await fetch(accountStartUrl, {
|
||
headers: this.headers(),
|
||
redirect: 'follow',
|
||
});
|
||
this.captureCookies(checkRes);
|
||
const finalUrl = checkRes.url || '';
|
||
const finalHtml = await checkRes.text();
|
||
const pageText = normalizeSpaces(finalHtml).toLowerCase();
|
||
const stillLogin =
|
||
finalUrl.includes('/account/login') ||
|
||
pageText.includes('mot de passe') ||
|
||
pageText.includes('passwort') ||
|
||
pageText.includes('connexion') && pageText.includes('numéro de client');
|
||
|
||
this.loggedIn = !stillLogin;
|
||
emit({
|
||
type: 'log',
|
||
message: this.loggedIn
|
||
? `Connecté à Häfele (${this.cookies.size} cookies).`
|
||
: 'Login Häfele non confirmé, continuation de la sync.',
|
||
});
|
||
return this.loggedIn;
|
||
} catch (err) {
|
||
emit({ type: 'log', message: `Erreur login Häfele: ${err.message}` });
|
||
return false;
|
||
}
|
||
}
|
||
}
|
||
|
||
let session = new HaefeleSession();
|
||
let currentCredentials = {};
|
||
let currentEmit = () => {};
|
||
|
||
async function waitForCloudflare(page, maxSeconds = 30) {
|
||
for (let elapsed = 0; elapsed < maxSeconds; elapsed += 3) {
|
||
const title = await page.title().catch(() => '');
|
||
const html = await page.content().catch(() => '');
|
||
if (!isCloudflareChallenge(html) && !title.toLowerCase().includes('just a moment')) {
|
||
return true;
|
||
}
|
||
currentEmit({ type: 'log', message: ` Cloudflare en cours… (${elapsed}s)` });
|
||
await page.waitForTimeout(3000);
|
||
}
|
||
return false;
|
||
}
|
||
|
||
let browserInstance = null;
|
||
let browserContext = null;
|
||
|
||
async function getBrowserPage() {
|
||
if (browserInstance && browserContext) {
|
||
return browserContext.newPage();
|
||
}
|
||
|
||
let chromium;
|
||
try {
|
||
({ chromium } = await import('playwright'));
|
||
} catch {
|
||
throw new Error('Cloudflare challenge détecté et Playwright indisponible.');
|
||
}
|
||
|
||
const headless = process.env.HAFELE_BROWSER_HEADLESS !== '0';
|
||
browserInstance = await chromium.launch({
|
||
headless,
|
||
args: [
|
||
'--disable-blink-features=AutomationControlled',
|
||
'--no-sandbox'
|
||
]
|
||
});
|
||
|
||
browserContext = await browserInstance.newContext({
|
||
locale: 'fr-CH',
|
||
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
|
||
viewport: { width: 1920, height: 1080 }
|
||
});
|
||
|
||
return browserContext.newPage();
|
||
}
|
||
|
||
async function closeBrowser() {
|
||
if (browserInstance) {
|
||
await browserInstance.close().catch(() => {});
|
||
browserInstance = null;
|
||
browserContext = null;
|
||
}
|
||
}
|
||
|
||
let browserLoggedIn = false;
|
||
|
||
async function ensureBrowserLogin() {
|
||
if (browserLoggedIn) return;
|
||
|
||
const username = currentCredentials.username || process.env.HAFELE_USERNAME || process.env.HAFELE_USER;
|
||
const password = currentCredentials.password || process.env.HAFELE_PASSWORD;
|
||
const customerNo = currentCredentials.customerNo || process.env.HAFELE_CUSTOMER_NO || username;
|
||
|
||
if (!username || !password) {
|
||
currentEmit({ type: 'log', message: 'Pas d\'identifiants Häfele, navigation sans login.' });
|
||
return;
|
||
}
|
||
|
||
const page = await getBrowserPage();
|
||
try {
|
||
currentEmit({ type: 'log', message: 'Ouverture de la page de login Häfele…' });
|
||
await page.goto(`${BASE_URL}/fr/account/login/`, { waitUntil: 'load', timeout: BROWSER_TIMEOUT_MS });
|
||
|
||
const cfCleared = await waitForCloudflare(page, 45);
|
||
if (!cfCleared) {
|
||
currentEmit({ type: 'log', message: 'Cloudflare non résolu sur la page login.' });
|
||
const title = await page.title().catch(() => '?');
|
||
currentEmit({ type: 'log', message: ` Page titre: "${title}", URL: ${page.url()}` });
|
||
return;
|
||
}
|
||
|
||
const currentUrl = page.url();
|
||
const title = await page.title().catch(() => '');
|
||
currentEmit({ type: 'log', message: `Page login chargée: "${title}" (${currentUrl})` });
|
||
|
||
// Use exact ID selectors — the form has #ShopLoginForm_Login (text input)
|
||
// and #ShopLoginForm_LoginAsJointUser (hidden checkbox). Substring selectors
|
||
// like [name*=] would match the checkbox first and break .first().
|
||
const customerField = page.locator('#ShopLoginForm_Login');
|
||
const pwField = page.locator('#ShopLoginForm_Password');
|
||
|
||
try {
|
||
await customerField.waitFor({ state: 'visible', timeout: 15000 });
|
||
} catch {
|
||
currentEmit({ type: 'log', message: 'Champ #ShopLoginForm_Login non visible après attente.' });
|
||
const bodySnippet = normalizeSpaces(await page.content().catch(() => '')).slice(0, 500);
|
||
currentEmit({ type: 'log', message: ` Début HTML: ${bodySnippet}` });
|
||
return;
|
||
}
|
||
|
||
currentEmit({ type: 'log', message: 'Formulaire de login trouvé, saisie des identifiants…' });
|
||
await customerField.fill(customerNo || username);
|
||
await pwField.fill(password);
|
||
|
||
// Handle co-user mode if customer number differs from username.
|
||
if (username && customerNo && username !== customerNo) {
|
||
const coUserCheck = page.locator('#ShopLoginForm_LoginAsJointUser');
|
||
const coUserCheckDiv = page.locator('#divShopLoginForm_LoginAsJointUser');
|
||
// The real checkbox is hidden; the visible <div> acts as the toggle.
|
||
if (await coUserCheckDiv.count()) {
|
||
await coUserCheckDiv.click().catch(() => {});
|
||
await page.waitForTimeout(800);
|
||
} else if (await coUserCheck.count()) {
|
||
await page.evaluate(() => {
|
||
document.querySelector('#ShopLoginForm_LoginAsJointUser').checked = true;
|
||
document.querySelector('#ShopLoginForm_LoginAsJointUser').dispatchEvent(new Event('change', { bubbles: true }));
|
||
});
|
||
await page.waitForTimeout(800);
|
||
}
|
||
const coUserName = page.locator('#ShopLoginForm_UserName');
|
||
if (await coUserName.isVisible({ timeout: 2000 }).catch(() => false)) {
|
||
await coUserName.fill(username);
|
||
}
|
||
}
|
||
|
||
const submitBtn = page.locator('button[data-testid="accountLoginFormBtn"]');
|
||
|
||
await Promise.all([
|
||
page.waitForNavigation({ waitUntil: 'load', timeout: 30000 }).catch(() => {}),
|
||
submitBtn.click()
|
||
]);
|
||
|
||
await page.waitForTimeout(3000);
|
||
const afterUrl = page.url();
|
||
browserLoggedIn = !afterUrl.includes('/account/login');
|
||
currentEmit({
|
||
type: 'log',
|
||
message: browserLoggedIn
|
||
? `Connecté à Häfele via navigateur (${afterUrl}).`
|
||
: `Login navigateur non confirmé (${afterUrl}), continuation sans prix.`
|
||
});
|
||
} finally {
|
||
await page.close();
|
||
}
|
||
}
|
||
|
||
async function fetchPageWithBrowser(url) {
|
||
await ensureBrowserLogin();
|
||
|
||
const page = await getBrowserPage();
|
||
try {
|
||
currentEmit({ type: 'log', message: `Navigation vers ${url}…` });
|
||
await page.goto(url, { waitUntil: 'load', timeout: BROWSER_TIMEOUT_MS });
|
||
|
||
const cfCleared = await waitForCloudflare(page, 45);
|
||
if (!cfCleared) {
|
||
throw new Error('Cloudflare challenge non résolu automatiquement dans le navigateur.');
|
||
}
|
||
|
||
// Wait for product content to appear.
|
||
try {
|
||
await page.locator('.productDetailPage, .productHeadline, h1').first()
|
||
.waitFor({ state: 'visible', timeout: 15000 });
|
||
} catch {
|
||
// Page loaded but no product selector found — continue anyway.
|
||
}
|
||
|
||
// Price is loaded via AJAX after page load — wait for it.
|
||
try {
|
||
await page.locator('#productPriceInformation span.price').first()
|
||
.waitFor({ state: 'visible', timeout: 15000 });
|
||
currentEmit({ type: 'log', message: 'Prix détecté dans la page.' });
|
||
} catch {
|
||
currentEmit({ type: 'log', message: 'Prix non chargé après 15s, récupération du contenu actuel.' });
|
||
}
|
||
|
||
const html = await page.content();
|
||
currentEmit({ type: 'log', message: 'Contenu récupéré via navigateur (Playwright).' });
|
||
return html;
|
||
} finally {
|
||
await page.close();
|
||
}
|
||
}
|
||
|
||
async function fetchPage(url) {
|
||
const referer = `${BASE_URL}/fr/`;
|
||
let response = await fetch(url, {
|
||
headers: session.headers({
|
||
Referer: referer,
|
||
Origin: BASE_URL,
|
||
'Sec-Fetch-Site': 'same-origin'
|
||
}),
|
||
redirect: 'follow',
|
||
});
|
||
session.captureCookies(response);
|
||
let firstBody = '';
|
||
if (response.status === 403) {
|
||
firstBody = await response.text().catch(() => '');
|
||
if (isCloudflareChallenge(firstBody)) {
|
||
currentEmit({ type: 'log', message: 'Cloudflare détecté, tentative de fallback navigateur…' });
|
||
return fetchPageWithBrowser(url);
|
||
}
|
||
}
|
||
|
||
// Retry once after refreshing home page cookies when anti-bot blocks first hit.
|
||
if (response.status === 403) {
|
||
const warmup = await fetch(referer, {
|
||
headers: session.headers({
|
||
Referer: referer,
|
||
Origin: BASE_URL,
|
||
'Sec-Fetch-Site': 'same-origin'
|
||
}),
|
||
redirect: 'follow',
|
||
});
|
||
session.captureCookies(warmup);
|
||
|
||
response = await fetch(url, {
|
||
headers: session.headers({
|
||
Referer: referer,
|
||
Origin: BASE_URL,
|
||
'Sec-Fetch-Site': 'same-origin'
|
||
}),
|
||
redirect: 'follow',
|
||
});
|
||
session.captureCookies(response);
|
||
if (response.status === 403) {
|
||
const secondBody = await response.text().catch(() => '');
|
||
if (isCloudflareChallenge(secondBody)) {
|
||
currentEmit({ type: 'log', message: 'Challenge persistant, fallback navigateur…' });
|
||
return fetchPageWithBrowser(url);
|
||
}
|
||
throw new Error(`Failed to fetch ${url}: 403`);
|
||
}
|
||
}
|
||
|
||
if (!response.ok) {
|
||
const body = await response.text().catch(() => '');
|
||
const sample = normalizeSpaces(body).slice(0, 180);
|
||
throw new Error(`Failed to fetch ${url}: ${response.status}${sample ? ` (${sample})` : ''}`);
|
||
}
|
||
return response.text();
|
||
}
|
||
|
||
function extractKeyValueSpecs($) {
|
||
const specs = {};
|
||
$('.productPropertiesSection.details-product-body').each((_, el) => {
|
||
const key = normalizeSpaces($(el).find('.productPropertiesSectionHeader').first().text()).toLowerCase();
|
||
const value = normalizeSpaces($(el).find('.productPropertiesSectionBody').first().text());
|
||
if (key && value) specs[key] = value;
|
||
});
|
||
return specs;
|
||
}
|
||
|
||
function extractArticleNumber($) {
|
||
const text = normalizeSpaces($('.product-sku-title').first().text() || $.text());
|
||
const match = text.match(/N°\s*d['’]art\.\s*([\d.]+)/i);
|
||
return match ? match[1] : null;
|
||
}
|
||
|
||
function buildMaterialCode(url, articleNumber) {
|
||
if (articleNumber) {
|
||
return `HAF-${articleNumber.replace(/[^\dA-Za-z]/g, '')}`;
|
||
}
|
||
const skuMatch = url.match(/\/(P-\d+)\//i);
|
||
if (skuMatch) return `HAF-${skuMatch[1].toUpperCase()}`;
|
||
const slug = (url.split('/').filter(Boolean).pop() || 'unknown').replace(/[^\dA-Za-z]/g, '').slice(0, 30);
|
||
return `HAF-${slug.toUpperCase()}`;
|
||
}
|
||
|
||
function parseProductPage(html, url) {
|
||
const $ = cheerio.load(html);
|
||
|
||
const name = normalizeSpaces($('h1.productHeadline[itemprop="name"]').first().text() || $('h1.productHeadline').first().text());
|
||
const description = normalizeSpaces($('h2.productSubline[itemprop="description"]').first().text() || $('h2.productSubline').first().text());
|
||
const articleNumber = extractArticleNumber($);
|
||
|
||
const priceText =
|
||
normalizeSpaces($('#productPriceInformation .CUSTOMER_PRICE_Display .price span.price').first().text()) ||
|
||
normalizeSpaces($('#productPriceInformation .pricedisplay:not(.hidden) span.price').first().text()) ||
|
||
normalizeSpaces($('#productPriceInformation span.price').first().text()) ||
|
||
normalizeSpaces($('[id^="selectedArticlePriceInfo"] span.price').first().text()) ||
|
||
normalizeSpaces($('.priceContainer .price').first().text());
|
||
const priceCHF = parsePriceCHF(priceText);
|
||
if (priceText) {
|
||
currentEmit({ type: 'log', message: ` Prix trouvé: "${priceText}" → ${priceCHF} CHF` });
|
||
} else {
|
||
currentEmit({ type: 'log', message: ' Aucun prix trouvé dans le HTML.' });
|
||
}
|
||
const priceUnit = normalizeSpaces($('#productPriceInformation span.perUnit').first().text()) ||
|
||
normalizeSpaces($('[id^="selectedArticlePriceInfo"] span.perUnit').first().text()) || null;
|
||
|
||
const availabilityText = normalizeSpaces($('#productAvailabilityInformation .availability-flag').first().text());
|
||
const inStock = availabilityText
|
||
? (availabilityText.toLowerCase().includes('disponible') || availabilityText.toLowerCase().includes('stock')) &&
|
||
!availabilityText.toLowerCase().includes('indisponible')
|
||
: null;
|
||
|
||
// Deterministic image pick: prefer "primary" image, not the last diapo slide.
|
||
const pickImg = (sel) => {
|
||
const el = $(sel).first();
|
||
if (!el?.length) return null;
|
||
const src = el.attr('data-fullsrc') || el.attr('data-src') || el.attr('src') || '';
|
||
return absoluteUrl(String(src).trim());
|
||
};
|
||
|
||
// 1) Category/tile view markup (your snippet) — prefer when present.
|
||
const tileImg = pickImg('div.product_picture a.shopItemImage img, a.shopItemImage img');
|
||
|
||
// 2) OpenGraph is often the canonical hero image.
|
||
const ogImg = (
|
||
$('meta[property="og:image"]').attr('content') ||
|
||
$('meta[name="og:image"]').attr('content') ||
|
||
''
|
||
).trim();
|
||
const ogFull = absoluteUrl(ogImg);
|
||
|
||
// 3) Product detail hero image (if present).
|
||
const heroImg =
|
||
pickImg('.productImagePanelInner img[itemprop="image"]') ||
|
||
pickImg('.productImagePanelInner img');
|
||
|
||
// 4) First thumbnail (diapo) image.
|
||
const thumbImg = pickImg('.thumbsList img');
|
||
|
||
const imageUrl = tileImg || ogFull || heroImg || thumbImg || null;
|
||
|
||
const specs = extractKeyValueSpecs($);
|
||
|
||
if (Object.keys(specs).length > 0) {
|
||
const specsSummary = Object.entries(specs).map(([k, v]) => `${k}: ${v}`).join(' | ');
|
||
currentEmit({ type: 'log', message: ` Specs: ${specsSummary}` });
|
||
}
|
||
|
||
const category = normalizeSpaces($('.breadcrumb li:last-child').text()) || 'Hafele';
|
||
const thickness = parseFirstMm(specs['pour épaisseur du bois'] || specs['epaisseur'] || specs['épaisseur'] || '');
|
||
|
||
const allText = `${description} ${Object.values(specs).join(' ')}`;
|
||
const boreDiameter = parseBoreDiameter(allText);
|
||
const { depthBase, depthPiece } = parseBoreDepths(specs);
|
||
|
||
// Parse physical dimensions: diameter + length for cylindrical parts
|
||
let diameter = null;
|
||
let length = null;
|
||
const dimKeys = Object.keys(specs).map(k => k.toLowerCase());
|
||
for (const [key, value] of Object.entries(specs)) {
|
||
const k = key.toLowerCase();
|
||
if (!diameter && (k.includes('diamètre') || k.includes('diametre') || k.includes('durchmesser') || k.includes('diameter'))) {
|
||
diameter = parseFirstMm(value);
|
||
}
|
||
if (!length && (k.includes('longueur') || k.includes('länge') || k.includes('length') || k.includes('lang'))) {
|
||
length = parseFirstMm(value);
|
||
}
|
||
}
|
||
// Fallback: parse "Ø 8/60" or "8/60" pattern from the product name (e.g. SC 8/60 = Ø8mm, 60mm long)
|
||
if (!diameter) {
|
||
const nameMatch = (name || '').match(/(\d+(?:[.,]\d+)?)\s*\/\s*(\d+(?:[.,]\d+)?)/);
|
||
if (nameMatch) {
|
||
const d = parseFloat(nameMatch[1].replace(',', '.'));
|
||
const l = parseFloat(nameMatch[2].replace(',', '.'));
|
||
if (d > 0 && d <= 30) diameter = d;
|
||
if (l > 0 && l <= 200 && !length) length = l;
|
||
}
|
||
}
|
||
|
||
return {
|
||
materialCode: buildMaterialCode(url, articleNumber),
|
||
articleNumber,
|
||
name: name || articleNumber || url,
|
||
description,
|
||
collection: 'Hafele',
|
||
category,
|
||
width_mm: null,
|
||
height_mm: null,
|
||
thickness_mm: thickness,
|
||
diameter_mm: diameter,
|
||
length_mm: length,
|
||
bore_diameter_mm: boreDiameter,
|
||
bore_depth_base_mm: depthBase,
|
||
bore_depth_piece_mm: depthPiece,
|
||
price_chf: priceCHF,
|
||
price_unit: priceUnit,
|
||
image_url: imageUrl,
|
||
formex_url: url,
|
||
in_stock: inStock,
|
||
};
|
||
}
|
||
|
||
async function ensureSupplier() {
|
||
const existing = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['haefele']);
|
||
if (existing.rows.length > 0) return existing.rows[0].id;
|
||
|
||
await run(
|
||
'INSERT INTO suppliers (code, name, email, type, active) VALUES (?, ?, ?, ?, ?)',
|
||
['haefele', 'Hafele', '[email protected]', 'materials', 1]
|
||
);
|
||
const created = await pool.query('SELECT id FROM suppliers WHERE code = ?', ['haefele']);
|
||
return created.rows[0]?.id || null;
|
||
}
|
||
|
||
async function upsertMaterial(material, supplierId) {
|
||
const existing = await pool.query(
|
||
'SELECT id FROM formex_materials WHERE formex_code = ?',
|
||
[material.materialCode]
|
||
);
|
||
|
||
if (existing.rows.length > 0) {
|
||
await run(
|
||
`UPDATE formex_materials
|
||
SET name = ?, collection = ?, category = ?, width_mm = ?, height_mm = ?, thickness_mm = ?,
|
||
diameter_mm = COALESCE(?, diameter_mm), length_mm = COALESCE(?, length_mm),
|
||
bore_diameter_mm = COALESCE(?, bore_diameter_mm),
|
||
bore_depth_base_mm = COALESCE(?, bore_depth_base_mm),
|
||
bore_depth_piece_mm = COALESCE(?, bore_depth_piece_mm),
|
||
image_url = ?, formex_url = ?, price_chf = COALESCE(?, price_chf), price_unit = COALESCE(?, price_unit),
|
||
supplier_id = COALESCE(?, supplier_id), last_synced_at = datetime('now')
|
||
WHERE formex_code = ?`,
|
||
[
|
||
material.name,
|
||
material.collection,
|
||
material.category,
|
||
material.width_mm,
|
||
material.height_mm,
|
||
material.thickness_mm,
|
||
material.diameter_mm ?? null,
|
||
material.length_mm ?? null,
|
||
material.bore_diameter_mm ?? null,
|
||
material.bore_depth_base_mm ?? null,
|
||
material.bore_depth_piece_mm ?? null,
|
||
material.image_url,
|
||
material.formex_url,
|
||
material.price_chf ?? null,
|
||
material.price_unit ?? null,
|
||
supplierId,
|
||
material.materialCode,
|
||
]
|
||
);
|
||
return existing.rows[0].id;
|
||
}
|
||
|
||
await run(
|
||
`INSERT INTO formex_materials
|
||
(formex_code, name, collection, category, width_mm, height_mm, thickness_mm,
|
||
diameter_mm, length_mm, bore_diameter_mm, bore_depth_base_mm, bore_depth_piece_mm,
|
||
image_url, formex_url, price_chf, price_unit, supplier_id, last_synced_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
|
||
[
|
||
material.materialCode,
|
||
material.name,
|
||
material.collection,
|
||
material.category,
|
||
material.width_mm,
|
||
material.height_mm,
|
||
material.thickness_mm,
|
||
material.diameter_mm ?? null,
|
||
material.length_mm ?? null,
|
||
material.bore_diameter_mm ?? null,
|
||
material.bore_depth_base_mm ?? null,
|
||
material.bore_depth_piece_mm ?? null,
|
||
material.image_url,
|
||
material.formex_url,
|
||
material.price_chf ?? null,
|
||
material.price_unit ?? null,
|
||
supplierId,
|
||
]
|
||
);
|
||
|
||
const created = await pool.query('SELECT id FROM formex_materials WHERE formex_code = ?', [material.materialCode]);
|
||
return created.rows[0]?.id || null;
|
||
}
|
||
|
||
async function logSync(status, itemsCount, error = null) {
|
||
await run(
|
||
'INSERT INTO sync_logs (source, status, items_count, error) VALUES (?, ?, ?, ?)',
|
||
['haefele', status, itemsCount, error]
|
||
);
|
||
}
|
||
|
||
export async function syncHaefeleByUrls(urls = [], emit = () => {}, options = {}) {
|
||
const credentials = options.credentials || {};
|
||
currentCredentials = credentials;
|
||
currentEmit = emit;
|
||
const validUrls = [...new Set((urls || [])
|
||
.map((v) => (typeof v === 'string' ? v.trim() : ''))
|
||
.filter(Boolean)
|
||
.filter((v) => v.includes('haefele.ch')))];
|
||
|
||
if (validUrls.length === 0) {
|
||
throw new Error('Aucune URL Häfele valide fournie.');
|
||
}
|
||
|
||
session = new HaefeleSession();
|
||
browserLoggedIn = false;
|
||
const supplierId = await ensureSupplier();
|
||
await session.login(emit, credentials);
|
||
|
||
let synced = 0;
|
||
let failed = 0;
|
||
const start = Date.now();
|
||
|
||
try {
|
||
emit({ type: 'log', message: `${validUrls.length} URL(s) Häfele à traiter.` });
|
||
emit({ type: 'progress', current: 0, total: validUrls.length, collection: 'Hafele' });
|
||
|
||
for (let i = 0; i < validUrls.length; i++) {
|
||
const url = validUrls[i];
|
||
emit({ type: 'log', message: `Traitement [${i + 1}/${validUrls.length}] ${url}` });
|
||
emit({ type: 'progress', current: i, total: validUrls.length, collection: 'Hafele' });
|
||
|
||
if (i > 0) await sleep(CRAWL_DELAY);
|
||
|
||
try {
|
||
const html = await fetchPage(url);
|
||
const parsed = parseProductPage(html, url);
|
||
await upsertMaterial(parsed, supplierId);
|
||
synced++;
|
||
const pricePart = parsed.price_chf != null ? ` - ${parsed.price_chf} CHF` : '';
|
||
const boreParts = [
|
||
parsed.bore_diameter_mm != null ? `Ø${parsed.bore_diameter_mm}` : null,
|
||
parsed.bore_depth_base_mm != null ? `fond:${parsed.bore_depth_base_mm}` : null,
|
||
parsed.bore_depth_piece_mm != null ? `pièce:${parsed.bore_depth_piece_mm}` : null,
|
||
].filter(Boolean);
|
||
const boreStr = boreParts.length ? ` [${boreParts.join(', ')}mm]` : '';
|
||
emit({ type: 'log', message: ` ✓ ${parsed.name}${pricePart}${boreStr}` });
|
||
} catch (err) {
|
||
failed++;
|
||
emit({ type: 'log', message: ` ✗ Erreur: ${err.message}` });
|
||
}
|
||
}
|
||
|
||
const duration = ((Date.now() - start) / 1000).toFixed(1);
|
||
const status = failed > 0 && synced === 0 ? 'error' : 'success';
|
||
await logSync(status, synced, failed > 0 ? `${failed} url(s) failed` : null);
|
||
const result = {
|
||
success: status === 'success',
|
||
itemsCount: synced,
|
||
failedCount: failed,
|
||
duration,
|
||
error: status === 'error' ? 'Aucune URL Häfele n\'a pu être synchronisée.' : null
|
||
};
|
||
emit({ type: 'progress', current: validUrls.length, total: validUrls.length, collection: 'Hafele' });
|
||
if (result.success) emit({ type: 'complete', result });
|
||
else emit({ type: 'error', error: result.error });
|
||
return result;
|
||
} catch (err) {
|
||
await logSync('error', synced, err.message);
|
||
emit({ type: 'error', error: err.message });
|
||
return { success: false, itemsCount: synced, error: err.message };
|
||
} finally {
|
||
await closeBrowser();
|
||
}
|
||
}
|