monwebsite/apex
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

v1.0

Browse Source
This commit is contained in:
lejay
2026-02-13 00:21:53 +01:00
parent 81d2412835
commit 6923c319a1
85 changed files with 17836 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/services/formexCrawler.js
+638
View File
@@ -0,0 +1,638 @@
/**
* Formex.ch web crawler service
* Scrapes panel/material data from the Formex catalog
*/
import * as cheerio from 'cheerio';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import pool, { run } from '../db/index.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const BASE_URL = 'https://www.formex.ch';
const ASSORTIMENT_URL = `${BASE_URL}/fr/assortiment`;
const CRAWL_DELAY = parseInt(process.env.FORMEX_CRAWL_DELAY_MS || '500', 10);
// Directory for storing downloaded material images
const IMAGES_DIR = path.join(__dirname, '..', '..', 'client', 'public', 'materials');
// Ensure images directory exists
if (!fs.existsSync(IMAGES_DIR)) {
fs.mkdirSync(IMAGES_DIR, { recursive: true });
}
/**
* Sleep helper for rate limiting
*/
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Fetch a URL with error handling
*/
async function fetchPage(url) {
const response = await fetch(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; ConfigurateurArmoires/1.0)',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'fr-CH,fr;q=0.9'
}
});
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status}`);
}
return response.text();
}
/**
* Download an image and save it locally
* @param {string} url - Image URL to download
* @param {string} filename - Local filename (e.g., "12345.jpg")
* @returns {string|null} - Local path or null if failed
*/
async function downloadImage(url, filename) {
try {
if (!url) return null;
// Ensure absolute URL
const fullUrl = url.startsWith('http') ? url : `${BASE_URL}${url}`;
const response = await fetch(fullUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; ConfigurateurArmoires/1.0)',
'Accept': 'image/*'
}
});
if (!response.ok) {
console.warn(`Failed to download image ${fullUrl}: ${response.status}`);
return null;
}
const contentType = response.headers.get('content-type') || '';
let ext = '.jpg';
if (contentType.includes('png')) ext = '.png';
else if (contentType.includes('webp')) ext = '.webp';
// Ensure filename has correct extension
const finalFilename = filename.replace(/\.[^.]+$/, '') + ext;
const filepath = path.join(IMAGES_DIR, finalFilename);
const buffer = Buffer.from(await response.arrayBuffer());
fs.writeFileSync(filepath, buffer);
// Return the public path (relative to client/public)
return `/materials/${finalFilename}`;
} catch (err) {
console.warn(`Error downloading image ${url}:`, err.message);
return null;
}
}
/**
* Extract collection URLs from the assortiment page
*/
async function getCollections() {
const html = await fetchPage(ASSORTIMENT_URL);
const $ = cheerio.load(html);
const collections = [];
// Find collection links in the "Collections" section
$('a[href*="/fr/shop/"]').each((_, el) => {
const href = $(el).attr('href');
const name = $(el).text().trim();
// Only include category pages (contain -c followed by numbers)
if (href && href.match(/-c\d+$/) && name) {
const fullUrl = href.startsWith('http') ? href : `${BASE_URL}${href}`;
// Avoid duplicates
if (!collections.find(c => c.url === fullUrl)) {
collections.push({ name, url: fullUrl });
}
}
});
return collections;
}
/**
* Parse dimensions from a string like "2800mm x 2070mm x 19mm"
*/
function parseDimensions(dimStr) {
if (!dimStr) return { width: null, height: null, thickness: null };
const matches = dimStr.match(/(\d+)\s*mm\s*x\s*(\d+)\s*mm\s*x\s*(\d+)\s*mm/i);
if (matches) {
return {
width: parseInt(matches[1], 10),
height: parseInt(matches[2], 10),
thickness: parseInt(matches[3], 10)
};
}
return { width: null, height: null, thickness: null };
}
/**
* Extract decor code from product name or page content
* e.g., "Panneaux mélaminés Duplex Plus 10000 Tilia" -> "10000"
*/
function extractDecorCode(text) {
// Look for a 4-5 digit code
const match = text.match(/\b(\d{4,5})\b/);
return match ? match[1] : null;
}
/**
* Map of common color keywords found in Formex product names to representative hex colors
*/
const COLOR_KEYWORDS = {
// Whites and off-whites
'blanc': '#f5f5f5', 'white': '#f5f5f5', 'weiss': '#f5f5f5', 'bianco': '#f5f5f5',
'crème': '#fffdd0', 'cream': '#fffdd0', 'ivoire': '#fffff0', 'ivory': '#fffff0',
'magnolia': '#f8f4f0', 'perle': '#eae0c8',
// Greys
'gris': '#909090', 'grey': '#909090', 'gray': '#909090', 'grau': '#909090',
'anthracite': '#383838', 'anthrazit': '#383838', 'graphite': '#4a4a4a',
'platine': '#e5e4e2', 'argent': '#c0c0c0', 'silver': '#c0c0c0',
'titane': '#878681',
// Blacks
'noir': '#1a1a1a', 'black': '#1a1a1a', 'schwarz': '#1a1a1a',
// Browns and woods
'chêne': '#b5895b', 'oak': '#b5895b', 'eiche': '#b5895b',
'noyer': '#6f4e37', 'walnut': '#6f4e37', 'nussbaum': '#6f4e37',
'hêtre': '#c9a46c', 'beech': '#c9a46c', 'buche': '#c9a46c',
'érable': '#d4b572', 'maple': '#d4b572', 'ahorn': '#d4b572',
'cerisier': '#9c3b17', 'cherry': '#9c3b17', 'kirsch': '#9c3b17',
'teck': '#b08444', 'teak': '#b08444',
'pin': '#dcc8a0', 'pine': '#dcc8a0',
'orme': '#a07850', 'elm': '#a07850',
'bouleau': '#e0c99c', 'birch': '#e0c99c',
'frêne': '#c8b88a', 'ash': '#c8b88a',
'acacia': '#b89244',
'wenge': '#4a3728', 'wengé': '#4a3728',
'mélèze': '#c4a06c', 'larch': '#c4a06c',
'aulne': '#b07c4c', 'alder': '#b07c4c',
'brun': '#8b4513', 'brown': '#8b4513', 'braun': '#8b4513',
'marron': '#6d3e1c', 'chocolat': '#4a2c14',
'cognac': '#9f381d', 'caramel': '#a67040',
'noisette': '#ab7746',
// Beiges and sandstones
'beige': '#d9c8a9', 'sable': '#c2b280', 'sand': '#c2b280',
'sahara': '#c4a661', 'champagne': '#f7e7ce',
'vanille': '#f3e5ab', 'vanilla': '#f3e5ab',
'cappuccino': '#a0785a', 'moka': '#6f4e37',
// Reds
'rouge': '#cc3333', 'red': '#cc3333', 'rot': '#cc3333',
'bordeaux': '#722f37', 'burgundy': '#722f37',
'terracotta': '#c65d3e', 'rouille': '#b7410e',
// Blues
'bleu': '#4169e1', 'blue': '#4169e1', 'blau': '#4169e1',
'marine': '#000080', 'navy': '#000080',
'ciel': '#87ceeb', 'azur': '#007fff',
'cobalt': '#0047ab', 'indigo': '#4b0082',
// Greens
'vert': '#4caf50', 'green': '#4caf50', 'grün': '#4caf50',
'olive': '#808000', 'sauge': '#9caf88', 'sage': '#9caf88',
'emeraude': '#50c878', 'mousse': '#738276',
// Yellows and golds
'jaune': '#f0c040', 'yellow': '#f0c040',
'or': '#d4af37', 'gold': '#d4af37',
'miel': '#cd8032', 'honey': '#cd8032',
// Stone and concrete
'béton': '#a0a0a0', 'beton': '#a0a0a0', 'concrete': '#a0a0a0',
'pierre': '#b0a898', 'stone': '#b0a898', 'stein': '#b0a898',
'ardoise': '#5a5a5a', 'slate': '#5a5a5a',
'marbre': '#e0d8d0', 'marble': '#e0d8d0',
'granit': '#808080', 'granite': '#808080',
'travertin': '#d1c6b0',
// Others
'rouille': '#b7410e', 'rust': '#b7410e',
'cuivre': '#b87333', 'copper': '#b87333',
'linen': '#e0d5c0', 'lin': '#e0d5c0',
'taupe': '#8b7d6b',
'mauve': '#e0b0ff', 'lilas': '#c8a2c8',
'rose': '#ffc0cb', 'pink': '#ffc0cb', 'rosa': '#ffc0cb'
};
/**
* Extract dominant color from product name by matching color keywords
* @param {string} name - Product name
* @returns {string|null} - Hex color code or null
*/
function extractDominantColor(name) {
if (!name) return null;
const lower = name.toLowerCase().normalize('NFD').replace(/[\u0300-\u036f]/g, '');
const nameLower = name.toLowerCase();
// Try exact keyword matching (longest match first for compound words)
const sortedKeywords = Object.entries(COLOR_KEYWORDS)
.sort((a, b) => b[0].length - a[0].length);
for (const [keyword, hex] of sortedKeywords) {
// Check the raw name (preserving accents) and the normalized name
if (nameLower.includes(keyword) || lower.includes(keyword.normalize('NFD').replace(/[\u0300-\u036f]/g, ''))) {
return hex;
}
}
return null;
}
/**
* Get product details from a product detail page
* Enhanced to capture category, images, dimension variants, and color variants
*/
async function getProductDetails(productUrl) {
try {
const html = await fetchPage(productUrl);
const $ = cheerio.load(html);
// Get product name from h1
const name = $('h1').first().text().trim();
// Get decor code - look for "N° du décor:" or extract from name
let decorCode = null;
$('*').each((_, el) => {
const text = $(el).text();
const match = text.match(/N°\s*du\s*décor\s*:\s*(\d+)/i);
if (match) {
decorCode = match[1];
return false; // break
}
});
if (!decorCode) {
decorCode = extractDecorCode(name);
}
// Get category from breadcrumb (second-to-last item usually)
let category = null;
const breadcrumbLinks = $('a[href*="/fr/shop/"]').filter((_, el) => {
const href = $(el).attr('href') || '';
return href.match(/-c\d+$/); // Collection links
});
if (breadcrumbLinks.length > 0) {
category = $(breadcrumbLinks.last()).text().trim();
}
// Get main product image
let imageUrl = null;
// Look for product image in various selectors
$('img').each((_, el) => {
const src = $(el).attr('src') || '';
if (src.includes('/dekorbilder/') || src.includes('/sortiment/')) {
imageUrl = src.startsWith('http') ? src : `${BASE_URL}${src}`;
return false;
}
});
// Try data-src for lazy-loaded images
if (!imageUrl) {
$('img[data-src]').each((_, el) => {
const src = $(el).attr('data-src') || '';
if (src.includes('/dekorbilder/') || src.includes('/sortiment/')) {
imageUrl = src.startsWith('http') ? src : `${BASE_URL}${src}`;
return false;
}
});
}
// Get ALL dimension variants (multiple formats like "3650mm x 600mm x 29mm")
const dimensionVariants = [];
const seenDimensions = new Set();
$('a, button, span, div').each((_, el) => {
const text = $(el).text().trim();
const match = text.match(/^(\d+)\s*mm\s*x\s*(\d+)\s*mm\s*x\s*(\d+)\s*mm$/i);
if (match) {
const key = `${match[1]}-${match[2]}-${match[3]}`;
if (!seenDimensions.has(key)) {
seenDimensions.add(key);
dimensionVariants.push({
width: parseInt(match[1], 10),
height: parseInt(match[2], 10),
thickness: parseInt(match[3], 10)
});
}
}
});
// Default dimensions (first variant or from legacy parsing)
let dimensions = dimensionVariants[0] || { width: null, height: null, thickness: null };
if (!dimensions.width) {
$('a, span, div').each((_, el) => {
const text = $(el).text();
if (text.match(/\d+\s*mm\s*x\s*\d+\s*mm\s*x\s*\d+\s*mm/i)) {
dimensions = parseDimensions(text);
return false;
}
});
}
// Get color/decor variants (from "Choisir la couleur" section)
const colorVariants = [];
$('a[href*="/fr/shop/"]').each((_, el) => {
const href = $(el).attr('href') || '';
// Look for product links that are variants (not the current product)
if (href.match(/-p\d+$/) && !href.includes(productUrl.split('/').pop())) {
const variantUrl = href.startsWith('http') ? href : `${BASE_URL}${href}`;
const variantName = $(el).attr('title') || $(el).text().trim();
const variantCode = extractDecorCode(variantName) || extractDecorCode(href);
// Look for variant thumbnail image
let variantImage = null;
const img = $(el).find('img').first();
if (img.length) {
variantImage = img.attr('src') || img.attr('data-src');
if (variantImage && !variantImage.startsWith('http')) {
variantImage = `${BASE_URL}${variantImage}`;
}
}
if (variantCode && variantName && !colorVariants.find(v => v.code === variantCode)) {
colorVariants.push({
code: variantCode,
name: variantName,
url: variantUrl,
imageUrl: variantImage
});
}
}
});
// Download main product image locally
let localImageUrl = null;
if (imageUrl && decorCode) {
localImageUrl = await downloadImage(imageUrl, `${decorCode}.jpg`);
}
// Download variant thumbnails
for (const variant of colorVariants) {
if (variant.imageUrl && variant.code) {
const localPath = await downloadImage(variant.imageUrl, `${variant.code}_thumb.jpg`);
if (localPath) {
variant.imageUrl = localPath;
}
}
}
// Extract a representative dominant color from the product name
const dominantColor = extractDominantColor(name);
return {
name,
decorCode,
dimensions,
dimensionVariants,
colorVariants,
category,
imageUrl: localImageUrl || imageUrl, // Prefer local path, fallback to URL
dominantColor
};
} catch (err) {
console.error(`Error fetching product details from ${productUrl}:`, err.message);
return null;
}
}
/**
* Get products from a collection page
* @param {string} collectionUrl
* @param {string} collectionName
* @param {Function} emit - Optional emitter for progress events
*/
async function getProductsFromCollection(collectionUrl, collectionName, emit = () => {}) {
const products = [];
const html = await fetchPage(collectionUrl);
const $ = cheerio.load(html);
// Find product links
const productLinks = new Set();
$('a[href*="/fr/shop/"]').each((_, el) => {
const href = $(el).attr('href');
// Product pages contain -p followed by numbers
if (href && href.match(/-p\d+$/)) {
const fullUrl = href.startsWith('http') ? href : `${BASE_URL}${href}`;
productLinks.add(fullUrl);
}
});
const productCount = productLinks.size;
console.log(` Found ${productCount} products in ${collectionName}`);
emit({ type: 'log', message: ` ${productCount} produits trouvés dans ${collectionName}` });
// Fetch each product's details
let productIndex = 0;
for (const productUrl of productLinks) {
await sleep(CRAWL_DELAY);
const details = await getProductDetails(productUrl);
productIndex++;
if (details && details.decorCode) {
products.push({
formex_code: details.decorCode,
name: details.name,
collection: collectionName,
category: details.category || collectionName,
width_mm: details.dimensions.width,
height_mm: details.dimensions.height,
thickness_mm: details.dimensions.thickness,
image_url: details.imageUrl,
formex_url: productUrl,
dominant_color: details.dominantColor,
dimensionVariants: details.dimensionVariants || [],
colorVariants: details.colorVariants || []
});
const dimCount = details.dimensionVariants?.length || 0;
const varCount = details.colorVariants?.length || 0;
emit({ type: 'log', message: ` [${productIndex}/${productCount}] ${details.name} (${dimCount} dims, ${varCount} variants)` });
}
}
return products;
}
/**
* Upsert a material into the database with enhanced data
*/
async function upsertMaterial(material) {
// Try to update first
const { rows } = await pool.query(
'SELECT id FROM formex_materials WHERE formex_code = ?',
[material.formex_code]
);
let materialId;
if (rows.length > 0) {
materialId = rows[0].id;
// Update existing
await run(
`UPDATE formex_materials
SET name = ?, collection = ?, category = ?, width_mm = ?, height_mm = ?, thickness_mm = ?,
image_url = ?, formex_url = ?, dominant_color = COALESCE(?, dominant_color), last_synced_at = datetime('now')
WHERE formex_code = ?`,
[material.name, material.collection, material.category || null,
material.width_mm, material.height_mm, material.thickness_mm,
material.image_url || null, material.formex_url, material.dominant_color || null, material.formex_code]
);
} else {
// Insert new
await run(
`INSERT INTO formex_materials
(formex_code, name, collection, category, width_mm, height_mm, thickness_mm, image_url, formex_url, dominant_color)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
[material.formex_code, material.name, material.collection, material.category || null,
material.width_mm, material.height_mm, material.thickness_mm,
material.image_url || null, material.formex_url, material.dominant_color || null]
);
// Get the inserted ID
const inserted = await pool.query(
'SELECT id FROM formex_materials WHERE formex_code = ?',
[material.formex_code]
);
materialId = inserted.rows[0]?.id;
}
// Update dimension variants if we have a materialId
if (materialId && material.dimensionVariants && material.dimensionVariants.length > 0) {
// Delete existing dimensions
await run('DELETE FROM formex_dimensions WHERE material_id = ?', [materialId]);
// Insert new dimensions
for (let i = 0; i < material.dimensionVariants.length; i++) {
const dim = material.dimensionVariants[i];
await run(
`INSERT INTO formex_dimensions (material_id, width_mm, height_mm, thickness_mm, is_default)
VALUES (?, ?, ?, ?, ?)`,
[materialId, dim.width, dim.height, dim.thickness, i === 0 ? 1 : 0]
);
}
}
// Update color variants if we have a materialId
if (materialId && material.colorVariants && material.colorVariants.length > 0) {
// Delete existing variants
await run('DELETE FROM formex_variants WHERE material_id = ?', [materialId]);
// Insert new variants
for (const variant of material.colorVariants) {
await run(
`INSERT INTO formex_variants (material_id, variant_code, variant_name, variant_url, image_url)
VALUES (?, ?, ?, ?, ?)`,
[materialId, variant.code, variant.name, variant.url || null, variant.imageUrl || null]
);
}
}
return materialId;
}
/**
* Log sync result to database
*/
async function logSync(source, status, itemsCount, error = null) {
await run(
`INSERT INTO sync_logs (source, status, items_count, error) VALUES (?, ?, ?, ?)`,
[source, status, itemsCount, error]
);
}
/**
* Main sync function - crawls Formex and updates database
* @param {Function} emit - Optional emitter callback for SSE progress events
* Called with { type: 'log'|'progress'|'complete'|'error', ... }
*/
export async function sync(emit = () => {}) {
console.log('Starting Formex sync...');
emit({ type: 'log', message: 'Démarrage de la synchronisation Formex...' });
const startTime = Date.now();
let totalProducts = 0;
try {
// Get all collections
console.log('Fetching collections...');
emit({ type: 'log', message: 'Récupération des collections...' });
const collections = await getCollections();
const totalCollections = collections.length;
console.log(`Found ${totalCollections} collections`);
emit({ type: 'log', message: `${totalCollections} collections trouvées` });
emit({ type: 'progress', current: 0, total: totalCollections, collection: '' });
// Process each collection
for (let i = 0; i < collections.length; i++) {
const collection = collections[i];
console.log(`Processing: ${collection.name}`);
emit({ type: 'log', message: `Traitement: ${collection.name}` });
emit({ type: 'progress', current: i, total: totalCollections, collection: collection.name });
await sleep(CRAWL_DELAY);
try {
const products = await getProductsFromCollection(collection.url, collection.name, emit);
// Upsert each product
for (const product of products) {
await upsertMaterial(product);
totalProducts++;
}
emit({ type: 'log', message: ` ${products.length} produits enregistrés pour ${collection.name}` });
} catch (err) {
console.error(` Error processing ${collection.name}:`, err.message);
emit({ type: 'log', message: ` Erreur: ${err.message}` });
}
}
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
console.log(`Sync complete: ${totalProducts} products in ${elapsed}s`);
await logSync('formex', 'success', totalProducts);
const result = {
success: true,
itemsCount: totalProducts,
duration: elapsed
};
emit({ type: 'progress', current: totalCollections, total: totalCollections, collection: '' });
emit({ type: 'complete', result });
return result;
} catch (err) {
console.error('Sync failed:', err.message);
await logSync('formex', 'error', totalProducts, err.message);
const result = {
success: false,
error: err.message,
itemsCount: totalProducts
};
emit({ type: 'error', error: err.message });
return result;
}
}
/**
* Get sync logs from database
*/
export async function getSyncLogs(limit = 20) {
const { rows } = await pool.query(
'SELECT * FROM sync_logs ORDER BY created_at DESC LIMIT ?',
[limit]
);
return rows;
}
/**
* Get all scraped materials
*/
export async function getMaterials() {
const { rows } = await pool.query(
'SELECT * FROM formex_materials ORDER BY collection, name'
);
return rows;
}