n8n-nodes-immometrica-v2
Version:
n8n community node for ImmoMetrica real estate data extraction with CSV-based approach and agent tool compatibility
1,636 lines • 91.8 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.ImmoMetrica = void 0;
const n8n_workflow_1 = require("n8n-workflow");
const selenium_webdriver_1 = require("selenium-webdriver");
const chrome = __importStar(require("selenium-webdriver/chrome"));
const cheerio = __importStar(require("cheerio"));
const ImmoMetricaSearch_resource_1 = require("./ImmoMetricaSearch.resource");
const node_cache_1 = __importDefault(require("node-cache"));
class ImmoMetrica {
static sessionCache = new node_cache_1.default({ stdTTL: 1800, checkperiod: 120 });
static csvCache = new node_cache_1.default({ stdTTL: 300, checkperiod: 60, maxKeys: 50 });
description = {
displayName: 'ImmoMetrica v2',
name: 'immoMetricaV2',
icon: 'file:immometrica.svg',
group: ['input'],
version: 1,
subtitle: '={{$parameter["operation"]}}',
description: 'Extrahiert Immobiliendaten von ImmoMetrica mit CSV-basiertem Ansatz und Agent-Tool-Kompatibilität',
defaults: {
name: 'ImmoMetrica v2',
},
inputs: ['main'],
outputs: ['main'],
credentials: [
{
name: 'immoMetricaApi',
required: true,
},
],
usableAsTool: true,
properties: [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Gespeicherte Suchen Abrufen',
value: 'getSavedSearches',
description: 'Alle gespeicherten Suchen des Benutzers abrufen. Liefert IDs und Namen aller verfügbaren Immobiliensuchen für weitere Datenextraktion.',
action: 'Gespeicherte Suchen von ImmoMetrica abrufen',
},
{
name: 'Immobiliendaten Extrahieren (CSV)',
value: 'extractPropertyData',
description: 'Detaillierte Immobiliendaten aus einer gespeicherten Suche via CSV-Download extrahieren. Liefert strukturierte Investment-Daten: Preise, ROI, Renditen, Standorte, Cashflow und alle relevanten Kennzahlen für Investment-Analysen.',
action: 'Immobiliendaten von ImmoMetrica via CSV extrahieren',
},
{
name: 'Regionale Statistiken',
value: 'getRegionalStats',
description: 'Regionale Marktstatistiken für alle 401 deutschen Landkreise abrufen. Enthält Durchschnittspreise, Renditen und Marktdaten für regionale Investment-Vergleiche und Standortanalysen.',
action: 'Regionale Marktstatistiken von ImmoMetrica abrufen',
},
{
name: 'Marktübersicht',
value: 'getMarketOverview',
description: 'Allgemeine Marktübersicht und Investment-Trends für den deutschen Immobilienmarkt. Liefert makroökonomische Insights und Marktentwicklungen für strategische Investment-Entscheidungen.',
action: 'Marktübersicht von ImmoMetrica abrufen',
},
],
default: 'extractPropertyData',
},
{
displayName: 'Gespeicherte Suche',
name: 'savedSearchId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getSavedSearches',
},
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: '',
description: 'Wählen Sie eine gespeicherte Suche aus',
required: true,
},
{
displayName: 'Such-Name (Optional)',
name: 'searchName',
type: 'string',
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: '',
description: 'Optional: Name der Suche für Logging/Tracking',
},
{
displayName: 'Maximale Ergebnisse',
name: 'maxResults',
type: 'number',
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: 50,
description: 'Maximale Anzahl der zu extrahierenden Immobilien',
typeOptions: {
minValue: 1,
maxValue: 1000,
},
},
{
displayName: 'Details Scraping Aktivieren',
name: 'enableDetailScraping',
type: 'boolean',
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: false,
description: 'Zusätzliche Details von Immobilien-Detailseiten extrahieren (langsamer, aber mehr Daten)',
},
{
displayName: 'Filter-Optionen',
name: 'filterOptions',
type: 'collection',
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: {},
placeholder: 'Filter hinzufügen',
description: 'Erweiterte Filterkriterien für die Immobiliensuche',
options: [
{
displayName: 'Mindest-ROI (%)',
name: 'minRoi',
type: 'number',
default: 0,
description: 'Minimaler ROI in Prozent',
typeOptions: {
minValue: 0,
maxValue: 100,
},
},
{
displayName: 'Maximal-ROI (%)',
name: 'maxRoi',
type: 'number',
default: 100,
description: 'Maximaler ROI in Prozent',
typeOptions: {
minValue: 0,
maxValue: 100,
},
},
{
displayName: 'Mindestpreis (€)',
name: 'minPrice',
type: 'number',
default: 0,
description: 'Mindestpreis in Euro',
typeOptions: {
minValue: 0,
},
},
{
displayName: 'Maximalpreis (€)',
name: 'maxPrice',
type: 'number',
default: 10000000,
description: 'Maximalpreis in Euro',
typeOptions: {
minValue: 0,
},
},
{
displayName: 'Standort-Filter',
name: 'locationFilter',
type: 'string',
default: '',
placeholder: 'Berlin, München, Hamburg',
description: 'Kommagetrennte Liste von Standorten zum Filtern',
},
{
displayName: 'Nur positive Cashflows',
name: 'positiveCashflowOnly',
type: 'boolean',
default: false,
description: 'Nur Immobilien mit positivem Cashflow anzeigen',
},
],
},
],
};
methods = {
loadOptions: {
/**
* Retrieves saved searches from ImmoMetrica for use in node parameter options
* @description Loads all available saved searches from the user's ImmoMetrica account
* and formats them as options for the savedSearchId parameter dropdown
* @returns Promise<INodePropertyOptions[]> Array of saved search options with name and value
* @throws {Error} When credentials are invalid or ImmoMetrica is unreachable
* @example
* // Returns format:
* [
* { name: "Zurich Apartments", value: "search-123" },
* { name: "Basel Houses", value: "search-456" }
* ]
*/
async getSavedSearches() {
try {
const credentials = await this.getCredentials('immoMetricaApi');
const driver = await ImmoMetrica.createWebDriver();
const nodeInstance = new ImmoMetrica();
try {
await nodeInstance.loginToImmoMetrica(driver, credentials.email, credentials.password, credentials.baseUrl);
const searches = await nodeInstance.getSavedSearches(driver, credentials.baseUrl);
return searches.map((search) => ({
name: search.title,
value: search.id,
}));
}
finally {
await driver.quit();
}
}
catch (error) {
console.error('LoadOptions error:', error);
return [
{
name: 'Error loading searches - check credentials',
value: 'error',
},
];
}
},
},
};
/**
* Main execution method for the ImmoMetrica node
* @description Executes the selected operation (getSavedSearches or extractPropertyData)
* with support for AI agent tool input parsing and comprehensive error handling
* @returns Promise<INodeExecutionData[][]> Array of execution results with property data
* @throws {NodeOperationError} When operation fails or invalid parameters provided
* @example
* // Tool input for AI agents:
* {
* "operation": "extractPropertyData",
* "savedSearchId": "search-123",
* "maxResults": 100,
* "enableDetailScraping": true
* }
*
* // Returns:
* [[{ json: { properties: [...], metadata: {...} } }]]
*/
async execute() {
const items = this.getInputData();
const returnData = [];
const credentials = await this.getCredentials('immoMetricaApi');
const email = credentials.email;
const password = credentials.password;
const baseUrl = credentials.baseUrl;
let driver = null;
const nodeInstance = new ImmoMetrica();
try {
driver = await ImmoMetrica.createWebDriver();
await nodeInstance.loginToImmoMetrica(driver, email, password, baseUrl);
for (let i = 0; i < items.length; i++) {
try {
const currentUrl = await driver.getCurrentUrl();
if (currentUrl.includes('/login/') || currentUrl.includes('/accounts/')) {
console.log('Session expired, re-authenticating...');
await nodeInstance.loginToImmoMetrica(driver, email, password, baseUrl);
}
}
catch (sessionError) {
console.log(`Session check failed, re-authenticating: ${sessionError.message}`);
await nodeInstance.loginToImmoMetrica(driver, email, password, baseUrl);
}
let result = {};
let operation;
const toolInput = items[i].json;
if (toolInput && typeof toolInput === 'object' && toolInput.operation) {
operation = nodeInstance.validateAndSanitizeOperation(toolInput.operation, this.getNode());
console.log(`AI Agent tool execution: operation=${operation}`);
}
else {
operation = this.getNodeParameter('operation', i);
}
switch (operation) {
case 'getSavedSearches':
console.log('Executing getSavedSearches operation');
result = await nodeInstance.getSavedSearches(driver, baseUrl);
break;
case 'extractPropertyData': {
let savedSearchId;
let searchName;
let maxResults;
let enableDetailScraping;
let filterOptions;
if (toolInput && toolInput.operation) {
const validatedParams = nodeInstance.validateAndSanitizeToolInput(toolInput, 'extractPropertyData', this.getNode());
savedSearchId = validatedParams.savedSearchId;
searchName = validatedParams.searchName;
maxResults = validatedParams.maxResults;
enableDetailScraping = validatedParams.enableDetailScraping;
filterOptions = validatedParams.filterOptions;
}
else {
savedSearchId = this.getNodeParameter('savedSearchId', i);
searchName = this.getNodeParameter('searchName', i, '');
maxResults = this.getNodeParameter('maxResults', i);
enableDetailScraping = this.getNodeParameter('enableDetailScraping', i, false);
filterOptions = this.getNodeParameter('filterOptions', i, {});
}
if (!savedSearchId) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), 'AI Agent Error: savedSearchId is required for extractPropertyData operation. Please provide either "savedSearchId" or "searchId" in your tool input.');
}
if (maxResults < 1 || maxResults > 1000) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `AI Agent Error: maxResults must be between 1 and 1000, but received ${maxResults}. Please adjust your maxResults parameter.`);
}
console.log(`Executing extractPropertyData: searchId=${savedSearchId}, maxResults=${maxResults}, detailScraping=${enableDetailScraping}`);
result = await nodeInstance.extractPropertyDataCSV(driver, baseUrl, savedSearchId, searchName, maxResults, enableDetailScraping, filterOptions);
if (Array.isArray(result)) {
result = result.map((property) => ({
...property,
sourceSearchId: savedSearchId,
sourceSearchName: searchName || `Search ${savedSearchId}`,
scrapeTimestamp: new Date().toISOString(),
}));
}
break;
}
case 'getRegionalStats':
console.log('Executing getRegionalStats operation');
result = await nodeInstance.getRegionalStatistics(driver, baseUrl);
break;
case 'getMarketOverview':
console.log('Executing getMarketOverview operation');
result = await nodeInstance.getMarketOverview();
break;
default:
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Unbekannte Operation: ${operation}`);
}
if (Array.isArray(result)) {
result.forEach((item) => {
returnData.push({
json: item,
pairedItem: { item: i },
});
});
}
else {
returnData.push({
json: result,
pairedItem: { item: i },
});
}
}
return [returnData];
}
catch (error) {
console.error(`ImmoMetrica execution error: ${error.message}`);
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `ImmoMetrica Fehler: ${error.message}`);
}
finally {
if (driver) {
await driver.quit();
}
}
}
/**
* Creates and configures a Chrome WebDriver instance for web scraping
* @description Sets up a headless Chrome WebDriver with optimized options for ImmoMetrica scraping
* @returns Promise<WebDriver> Configured Chrome WebDriver with headless options
* @throws {Error} When Chrome/ChromeDriver is not available or configuration fails
* @example
* const driver = await ImmoMetrica.createWebDriver();
* // Returns configured headless Chrome WebDriver with security and performance options
*/
static async createWebDriver() {
const chromeOptions = new chrome.Options();
chromeOptions.addArguments('--headless');
chromeOptions.addArguments('--no-sandbox');
chromeOptions.addArguments('--disable-dev-shm-usage');
chromeOptions.addArguments('--disable-gpu');
chromeOptions.addArguments('--disable-extensions');
chromeOptions.addArguments('--disable-web-security');
chromeOptions.addArguments('--allow-running-insecure-content');
chromeOptions.addArguments('--window-size=1920,1080');
chromeOptions.addArguments('--memory-pressure-off');
chromeOptions.addArguments('--max_old_space_size=4096');
chromeOptions.addArguments('--disable-background-timer-throttling');
chromeOptions.addArguments('--disable-renderer-backgrounding');
chromeOptions.addArguments('--disable-backgrounding-occluded-windows');
chromeOptions.addArguments('--disable-features=TranslateUI');
chromeOptions.addArguments('--disable-ipc-flooding-protection');
chromeOptions.addArguments('--disable-default-apps');
chromeOptions.addArguments('--disable-sync');
chromeOptions.addArguments('--disable-background-networking');
chromeOptions.addArguments('--aggressive-cache-discard');
chromeOptions.addArguments('--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36');
chromeOptions.setUserPreferences({
'profile.default_content_setting_values.notifications': 2,
'profile.default_content_settings.popups': 0,
'profile.managed_default_content_settings.images': 2,
'profile.content_settings.exceptions.automatic_downloads.*.setting': 1
});
try {
const driver = await new selenium_webdriver_1.Builder()
.forBrowser('chrome')
.setChromeOptions(chromeOptions)
.build();
await driver.manage().setTimeouts({
implicit: 10000,
pageLoad: 30000,
script: 30000
});
return driver;
}
catch (driverError) {
const errorMsg = driverError.message;
if (errorMsg.includes('status 127') || errorMsg.includes('terminated early')) {
throw new Error(`Chrome/Chromium ist in Ihrer n8n-Umgebung nicht verfügbar.
LÖSUNG für Docker/Container:
1. Installieren Sie Chrome in Ihrem n8n-Container:
apt-get update && apt-get install -y google-chrome-stable
2. Oder fügen Sie zu Ihrer Dockerfile hinzu:
RUN apt-get update && apt-get install -y google-chrome-stable
3. Für n8n Cloud: Diese Node benötigt eine Self-Hosted n8n Installation mit Chrome.
ALTERNATIVE: Verwenden Sie einen HTTP-Request Node mit Session-Management für einfache Datenextraktion.
Technischer Fehler: ${errorMsg}`);
}
throw new Error(`Chrome WebDriver Fehler: ${errorMsg}`);
}
}
/**
* Authenticates with ImmoMetrica using provided credentials
* @description Handles login with session caching, retry logic, and automatic session validation
* @param driver WebDriver instance for browser automation
* @param email ImmoMetrica account email address
* @param password ImmoMetrica account password
* @param baseUrl ImmoMetrica base URL (e.g., https://www.immometrica.com)
* @param retryCount Current retry attempt number (default: 0)
* @returns Promise<void> Resolves when login is successful
* @throws {Error} When login fails after maximum retry attempts or credentials are invalid
* @example
* await node.loginToImmoMetrica(driver, 'user@example.com', 'password', 'https://www.immometrica.com');
* // Logs in with session caching and automatic retry on failure
*/
async loginToImmoMetrica(driver, email, password, baseUrl, retryCount = 0) {
const maxRetries = 3;
const retryDelay = 2000 + (retryCount * 1000);
const sessionKey = `session_${email}_${baseUrl}`;
const cachedSession = ImmoMetrica.sessionCache.get(sessionKey);
if (cachedSession && retryCount === 0) {
console.log('Using cached session for login');
try {
const cookies = cachedSession;
for (const cookie of cookies) {
await driver.manage().addCookie(cookie);
}
await driver.get(`${baseUrl}/de/searches`);
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('body')), 5000);
console.log('Cached session validated successfully');
return;
}
catch (cacheError) {
console.log('Cached session invalid, proceeding with fresh login');
ImmoMetrica.sessionCache.del(sessionKey);
}
}
try {
console.log(`Attempting fresh login to ImmoMetrica (attempt ${retryCount + 1}/${maxRetries + 1})`);
await driver.get(`${baseUrl}/de/accounts/login/`);
await driver.sleep(2000);
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.name('login')), 10000);
const emailField = await driver.findElement(selenium_webdriver_1.By.name('login'));
const passwordField = await driver.findElement(selenium_webdriver_1.By.name('password'));
const loginButton = await driver.findElement(selenium_webdriver_1.By.xpath("//button[contains(text(), 'Login')]"));
await emailField.clear();
await passwordField.clear();
await emailField.sendKeys(email);
await passwordField.sendKeys(password);
await loginButton.click();
await driver.sleep(3000);
const currentUrl = await driver.getCurrentUrl();
if (currentUrl.includes('/login')) {
throw new Error('Login failed - credentials may be incorrect or session expired');
}
try {
await driver.get(`${baseUrl}/de/searches`);
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('body')), 5000);
const cookies = await driver.manage().getCookies();
ImmoMetrica.sessionCache.set(sessionKey, cookies);
console.log('Login successful - session cached for 30 minutes');
}
catch (verifyError) {
throw new Error('Login appeared successful but cannot access protected pages');
}
}
catch (error) {
console.log(`Login attempt ${retryCount + 1} failed: ${error.message}`);
if (retryCount < maxRetries) {
console.log(`Retrying login in ${retryDelay}ms...`);
await driver.sleep(retryDelay);
return this.loginToImmoMetrica(driver, email, password, baseUrl, retryCount + 1);
}
throw new Error(`Login failed after ${maxRetries + 1} attempts: ${error.message}`);
}
}
/**
* Retrieves all saved searches from the user's ImmoMetrica account
* @description Scrapes the saved searches page to extract search IDs, titles, URLs, and result counts
* @param driver WebDriver instance for browser automation
* @param baseUrl ImmoMetrica base URL (e.g., https://www.immometrica.com)
* @returns Promise<SavedSearch[]> Array of saved search objects with id, title, url, and resultCount
* @throws {Error} When unable to access saved searches page or parse search data
* @example
* const searches = await node.getSavedSearches(driver, 'https://www.immometrica.com');
* // Returns: [{ id: "123", title: "Zurich Apartments", url: "...", resultCount: 45 }]
*/
async getSavedSearches(driver, baseUrl) {
await driver.get(`${baseUrl}/de/searches`);
await driver.sleep(2000);
const searches = [];
try {
const searchElements = await driver.findElements(selenium_webdriver_1.By.css('tbody tr'));
for (const element of searchElements) {
try {
const linkElement = await element.findElement(selenium_webdriver_1.By.css('td:first-child a'));
const title = await linkElement.getText();
const href = await linkElement.getAttribute('href');
const id = href.split('/').pop() || '';
let resultCount;
try {
const countElement = await element.findElement(selenium_webdriver_1.By.css('td:nth-child(2)'));
const countText = await countElement.getText();
resultCount = parseInt(countText.replace(/\D/g, ''), 10) || undefined;
}
catch (e) {
}
searches.push({
id,
title,
url: href,
resultCount,
});
}
catch (e) {
console.log(`Error extracting saved search: ${e.message}`);
}
}
}
catch (error) {
throw new Error(`Fehler beim Abrufen der gespeicherten Suchen: ${error.message}`);
}
return searches;
}
/**
* Extracts property data from ImmoMetrica using CSV download with optional detail scraping
* @description Main data extraction method that downloads CSV data and optionally scrapes detailed property information
* @param driver WebDriver instance for browser automation
* @param baseUrl ImmoMetrica base URL (e.g., https://www.immometrica.com)
* @param savedSearchId ID of the saved search to extract data from
* @param searchName Human-readable name of the search for logging
* @param maxResults Maximum number of properties to extract (limited to 1000 by ImmoMetrica)
* @param enableDetailScraping Whether to scrape detailed property information from individual pages
* @param filterOptions Additional filtering options to apply to the results
* @returns Promise<PropertyData[]> Array of property data objects with basic and detailed information
* @throws {Error} When search ID is invalid, CSV download fails, or detail scraping encounters errors
* @example
* const properties = await node.extractPropertyDataCSV(
* driver, 'https://www.immometrica.com', 'search-123', 'Zurich Apartments', 100, true, {}
* );
* // Returns array of PropertyData objects with comprehensive property information
*/
async extractPropertyDataCSV(driver, baseUrl, savedSearchId, searchName, maxResults, enableDetailScraping, filterOptions) {
if (!savedSearchId) {
throw new Error('Search ID muss angegeben werden');
}
const properties = [];
const maxResultsLimited = Math.min(maxResults, 1000);
const pagesNeeded = Math.ceil(maxResultsLimited / 100);
console.log(`Starting CSV-based extraction for search ${savedSearchId}, max ${maxResultsLimited} results (${pagesNeeded} pages) - limited by ImmoMetrica tariff`);
for (let page = 1; page <= pagesNeeded; page++) {
const url = `${baseUrl}/de/search/${savedSearchId}?page=${page}`;
console.log(`Processing page ${page}/${pagesNeeded}: ${url}`);
await driver.get(url);
await driver.sleep(3000);
try {
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.xpath("//table//tbody//tr//td//a[contains(@href, '/property/')] | //table//tbody//tr//td[contains(text(), 'CHF') or contains(text(), '€')]")), 15000);
console.log(`Page ${page} loaded successfully with property data`);
const csvData = await this.downloadCSVFromPage(driver);
console.log(`CSV data preview (first 200 chars): ${csvData.substring(0, 200)}`);
const pageProperties = ImmoMetricaSearch_resource_1.CSVProcessor.parseCSV(csvData, searchName, savedSearchId);
console.log(`Extracted ${pageProperties.length} properties from CSV on page ${page}`);
if (pageProperties.length === 0 && csvData.length > 0) {
console.log('Warning: CSV data found but no properties parsed - possible data format issue');
}
if (enableDetailScraping) {
console.log(`Detail scraping enabled, extracting additional data for ${pageProperties.length} properties`);
let successfulExtractions = 0;
let failedExtractions = 0;
const remainingSlots = Math.max(0, maxResultsLimited - properties.length);
const propertiesWithLinks = pageProperties
.filter(prop => prop.immometricaLink && prop.immometricaLink.includes('http'))
.slice(0, remainingSlots);
console.log(`Found ${pageProperties.filter(prop => prop.immometricaLink && prop.immometricaLink.includes('http')).length} properties with valid detail links, processing ${propertiesWithLinks.length} (limited by maxResults=${maxResultsLimited}, remaining slots=${remainingSlots})`);
const maxConcurrent = 3;
const chunks = [];
for (let i = 0; i < propertiesWithLinks.length; i += maxConcurrent) {
chunks.push(propertiesWithLinks.slice(i, i + maxConcurrent));
}
for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
const chunk = chunks[chunkIndex];
console.log(`Processing chunk ${chunkIndex + 1}/${chunks.length} with ${chunk.length} properties`);
const chunkPromises = chunk.map(async (property, indexInChunk) => {
const globalIndex = chunkIndex * maxConcurrent + indexInChunk;
try {
console.log(`Extracting details for property ${globalIndex + 1}/${propertiesWithLinks.length}: ${property.address}`);
let detailPageAccessible = false;
try {
let detailPageAccessible = false;
try {
await driver.get(property.immometricaLink);
const currentUrl = await driver.getCurrentUrl();
if (!currentUrl.includes("/login") && !currentUrl.includes("/accounts")) {
detailPageAccessible = true;
}
}
catch (e) {
detailPageAccessible = false;
}
}
catch (e) {
detailPageAccessible = false;
}
if (!detailPageAccessible) {
property.validationComment = "Detail page not accessible, used CSV only.";
// Ensure all KPIs and platform links from CSV are mapped here
return property;
}
const detailData = await this.extractPropertyDetails(driver, property.immometricaLink);
let fieldsEnhanced = 0;
Object.keys(detailData).forEach(key => {
const detailValue = detailData[key];
const csvValue = property[key];
if (detailValue !== undefined && detailValue !== null && detailValue !== '') {
if (key === 'propertyType') {
// Always override propertyType from detail scraping
property[key] = detailValue;
fieldsEnhanced++;
}
else if (csvValue === undefined || csvValue === null || csvValue === '' || csvValue === 0 || csvValue === false ||
(typeof csvValue === 'object' && csvValue !== null && Object.keys(csvValue).length === 0)) {
property[key] = detailValue;
fieldsEnhanced++;
}
}
});
if (detailData.fullPageText) {
this.extractKPIFromText(property, detailData.fullPageText);
}
this.applyKPIFallbacks(property);
const kpiStatus = {
condition: property.condition ? 'OK' : 'MISSING',
expectedRentPerMonth: property.expectedRentPerMonth > 0 ? 'OK' : 'MISSING',
expectedRentPerSqm: property.expectedRentPerSqm > 0 ? 'OK' : 'MISSING',
expectedYieldPercent: property.expectedYieldPercent > 0 ? 'OK' : 'MISSING',
actualYieldPercent: property.actualYieldPercent > 0 ? 'OK' : 'MISSING',
estimatedRentSoll: property.estimatedRentSoll > 0 ? 'OK' : 'MISSING',
estimatedRentPerSqmSoll: property.estimatedRentPerSqmSoll > 0 ? 'OK' : 'MISSING',
marketValuationEstimatePerSqm: property.marketValuationEstimatePerSqm > 0 ? 'OK' : 'MISSING',
immometricaMarktumfeldRenditeSoll: property.immometricaMarktumfeldRenditeSoll > 0 ? 'OK' : 'MISSING',
immometricaMarktumfeldRenditeIst: property.immometricaMarktumfeldRenditeIst > 0 ? 'OK' : 'MISSING',
immometricaMarktumfeldMieteSoll: property.immometricaMarktumfeldMieteSoll > 0 ? 'OK' : 'MISSING',
immometricaMarktumfeldMieteIst: property.immometricaMarktumfeldMieteIst > 0 ? 'OK' : 'MISSING'
};
console.log(`Enhanced property ${globalIndex + 1}/${propertiesWithLinks.length} with ${fieldsEnhanced} additional fields`);
console.log(`KPI Status for ${property.address}:`, kpiStatus);
return { success: true, property: property.address, fieldsEnhanced };
}
catch (detailError) {
console.log(`Failed to extract details for property ${property.address}: ${detailError.message}`);
this.applyKPIFallbacks(property);
return { success: false, property: property.address, error: detailError.message };
}
});
const chunkResults = await Promise.all(chunkPromises);
chunkResults.forEach(result => {
if (result.success) {
successfulExtractions++;
}
else {
failedExtractions++;
}
});
if (chunkIndex < chunks.length - 1) {
const delay = 800 + Math.random() * 400;
console.log(`Waiting ${Math.round(delay)}ms before processing next chunk...`);
await driver.sleep(delay);
}
}
console.log(`Completed parallel detail scraping for page ${page}: ${successfulExtractions} successful, ${failedExtractions} failed`);
}
// Apply fallback KPI calculations for properties without detail scraping
if (!enableDetailScraping) {
pageProperties.forEach(property => {
this.applyKPIFallbacks(property);
});
}
const filteredProperties = pageProperties.filter((prop) => this.passesFilters(prop, filterOptions));
properties.push(...filteredProperties);
console.log(`Added ${filteredProperties.length} properties from page ${page} (after filtering)`);
if (properties.length >= maxResultsLimited) {
console.log(`Reached maxResults limit of ${maxResultsLimited}, stopping extraction`);
break;
}
}
catch (error) {
console.log(`Error processing page ${page}: ${error.message}`);
if (page === 1) {
throw new Error(`Fehler beim Laden der ersten Seite: ${error.message}`);
}
break;
}
}
return properties.slice(0, maxResultsLimited);
}
/**
* Downloads CSV data from the current ImmoMetrica search results page
* @description Attempts to download CSV data with caching and retry logic, falls back to table scraping if needed
* @param driver WebDriver instance for browser automation
* @param retryCount Current retry attempt number (default: 0)
* @returns Promise<string> CSV data as string with property information
* @throws {Error} When CSV download fails after maximum retry attempts or session expires
* @example
* const csvData = await node.downloadCSVFromPage(driver);
* // Returns CSV string with property data that can be parsed
* @private
*/
async downloadCSVFromPage(driver, retryCount = 0) {
const maxRetries = 3;
const retryDelay = 1000 + (retryCount * 500);
const currentUrl = await driver.getCurrentUrl();
const csvCacheKey = `csv_${currentUrl.split('/').pop()}_${Math.floor(Date.now() / 60000)}`;
const cachedCsv = ImmoMetrica.csvCache.get(csvCacheKey);
if (cachedCsv && retryCount === 0) {
console.log('Using cached CSV data');
return cachedCsv;
}
try {
console.log(`Attempting CSV download (attempt ${retryCount + 1}/${maxRetries + 1})`);
if (currentUrl.includes('/login/') || currentUrl.includes('/accounts/')) {
throw new Error('Session expired - need to re-authenticate');
}
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.xpath("//table//tbody//tr//td//a[contains(@href, '/property/')] | //table//tbody//tr//td[contains(text(), 'CHF') or contains(text(), '€')]")), 15000);
const propertyRows = await driver.findElements(selenium_webdriver_1.By.xpath("//table//tbody//tr//td//a[contains(@href, '/property/')]"));
if (propertyRows.length === 0) {
console.log('No property links found, checking for price data...');
const priceRows = await driver.findElements(selenium_webdriver_1.By.xpath("//table//tbody//tr//td[contains(text(), 'CHF') or contains(text(), '€')]"));
if (priceRows.length === 0) {
throw new Error('No property data found in table - search results may not have loaded properly');
}
}
console.log(`Found ${propertyRows.length} property rows with data`);
const exportButton = await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('button[devinid="575"], button[title*="CSV"], button[title*="Export"], button[title*="Dateiformat: UTF-8"]')), 10000);
console.log('Found CSV export button, clicking...');
await driver.executeScript('arguments[0].scrollIntoView(true);', exportButton);
await driver.sleep(500);
await exportButton.click();
await driver.sleep(2000);
try {
const modal = await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('[aria-modal="true"], .modal, [devinid="581"]')), 5000);
console.log('CSV export limitation modal appeared, confirming...');
const confirmButton = await modal.findElement(selenium_webdriver_1.By.css('button:not([aria-label="Close"]), .btn-primary, button[devinid="582"]'));
await confirmButton.click();
await driver.sleep(1000);
}
catch (modalError) {
console.log('No limitation modal appeared or failed to handle it, continuing...');
}
await driver.sleep(3000);
const csvContent = await driver.executeScript(`
return new Promise((resolve, reject) => {
try {
const table = document.querySelector('table tbody');
if (!table) {
resolve('');
return;
}
const rows = Array.from(table.querySelectorAll('tr'));
const csvRows = [];
const headerRow = Array.from(document.querySelectorAll('table thead th')).map(th => th.textContent.trim());
if (headerRow.length === 0) {
reject(new Error('No table headers found'));
return;
}
csvRows.push(headerRow.join(';'));
rows.forEach(row => {
const cells = Array.from(row.querySelectorAll('td')).map(td => {
const text = td.textContent.trim();
return text.includes(';') || text.includes('"') ? '"' + text.replace(/"/g, '""') + '"' : text;
});
if (cells.length > 0) {
csvRows.push(cells.join(';'));
}
});
if (csvRows.length <= 1) {
reject(new Error('No data rows found in table'));
return;
}
resolve(csvRows.join('\\n'));
} catch (error) {
reject(error);
}
});
`);
const csvString = csvContent;
if (!csvString || csvString.trim() === '') {
throw new Error('Empty CSV content received');
}
if (csvString.length > 5 * 1024 * 1024) {
console.log(`Large CSV detected (${Math.round(csvString.length / 1024 / 1024)}MB), caching with memory management`);
}
ImmoMetrica.csvCache.set(csvCacheKey, csvString);
console.log(`Successfully extracted and cached CSV data: ${csvString.split('\n').length - 1} rows`);
return csvString;
}
catch (error) {
console.log(`CSV download attempt ${retryCount + 1} failed: ${error.message}`);
if (retryCount < maxRetries) {
console.log(`Retrying CSV download in ${retryDelay}ms...`);
await driver.sleep(retryDelay);
return this.downloadCSVFromPage(driver, retryCount + 1);
}
console.log(`All CSV download attempts failed, falling back to table scraping`);
return await this.fallbackTableScraping(driver);
}
}
async fallbackTableScraping(driver) {
try {
console.log('Attempting fallback table scraping...');
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('table')), 10000);
const rows = await driver.findElements(selenium_webdriver_1.By.css('table tbody tr'));
const csvRows = [];
try {
const headers = await driver.findElements(selenium_webdriver_1.By.css('table thead th'));
if (headers.length === 0) {
throw new Error('No table headers found');
}
const headerTexts = await Promise.all(headers.map(h => h.getText()));
csvRows.push(headerTexts.join(';'));
}
catch (headerError) {
console.log('Failed to extract headers, using generic headers');
csvRows.push('Column1;Column2;Column3;Column4;Column5;Column6;Column7;Column8;Column9;Column10');
}
if (rows.length === 0) {
throw new Error('No data rows found in table');
}
for (const row of rows) {
try {
const cells = await row.findElements(selenium_webdriver_1.By.css('td'));
const cellTexts = await Promise.all(cells.map(async (c) => {
try {
return await c.getText();
}
catch (cellError) {
return '';
}
}));
const escapedCells = cellTexts.map(text => text.includes(';') || text.includes('"') ? `"${text.replace(/"/g, '""')}"` : text);
if (escapedCells.some(cell => cell.trim() !== '')) {
csvRows.push(escapedCells.join(';'));
}
}
catch (rowError) {
console.log(`Failed to extract row data: ${rowError.message}`);
continue;
}
}
if (csvRows.length <= 1) {
throw new Error('No valid data rows extracted');
}
console.log(`Fallback table scraping successful: ${csvRows.length - 1} rows extracted`);
return csvRows.join('\n');
}
catch (error) {
console.log(`Fallback table scraping failed: ${error.message}`);
throw new Error(`All data extraction methods failed: ${error.message}`);
}
}
parseNumber(value) {
if (!value)
return 0;
let cleaned = value.replace(/[^\d,.-]/g, '');
if (cleaned.includes(',') && cleaned.includes('.')) {
cleaned = cleaned.replace(/\./g, '').replace(',', '.');
}
else if (cleaned.includes(',') && !cleaned.includes('.')) {
cleaned = cleaned.replace(',', '.');
}
const parsed = parseFloat(cleaned);
return isNaN(parsed) ? 0 : parsed;
}
parseBoolean(value) {
if (!value)
return false;
const lower = value.toLowerCase();
return lower === 'ja' || lower === 'yes' || lower === 'true' || lower === '1';
}
/**
* Extracts detailed property information from individual property pages
* @description Scrapes comprehensive property details including amenities, platform links, and additional KPIs
* @param driver WebDriver instance for browser automation
* @param detailsUrl URL of the individual property detail page
* @param retryCount Current retry attempt number (default: 0)
* @returns Promise<Partial<PropertyData>> Object containing detailed property information
* @throws {Error} When detail page is inaccessible or scraping fails after retries
* @example
* const details = await node.extractPropertyDetails(driver, 'https://immometrica.com/property/123');
* // Returns: { energyLabel: 'B', hasBalcony: true, platformLinks: {...}, ... }
* @private
*/
async extractPropertyDetails(driver, detailsUrl, retryCount = 0) {
if (!detailsUrl)
return {};
const maxRetries = 2;
try {
console.log(`Extracting details from: ${detailsUrl} (attempt ${retryCount + 1}/${maxRetries + 1})`);
await driver.get(detailsUrl);
await driver.sleep(3000);
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('body')), 15000);
try {
const expandButtons = await driver.findElements(selenium_webdriver_1.By.css('button[aria-expanded="false"], .expand-btn, .show-more'));
for (const button of expandButtons) {
try {
await button.click();
await driver.sleep(500);
}
catch (clickError) {
}
}
}
catch (expandError) {
console.log('No expandable sections found or failed to expand');
}
const details = {};
try {
const $ = cheerio.load(await driver.getPageSource());
const titleSelectors = ['header h2', 'h1', 'h2', '.property-title', '.title'];
for (const selector of titleSelectors) {
const titleElement = $(selector).first();
if (titleElement.length && titleElement.text().trim()) {
details.title = titleElement.text().trim();
break;
}
}
const locationText = $('body').text();
const locationMatch = locationText.match(/(\d{5}),?\s*([^,\n]+)/);
if (locationMatch) {
details.postalCode = locationMatch[1];
if ('city' in details) {
details.city = locationMatch[2].trim();
}
details.address = `${locationMatch[1]}, ${locationMatch[2].trim()}`;
}
const tableSelectors = [
'table tr',
'.property-details tr',
'.details-table tr',
'.property-info tr',
'.immobilie-details tr',
'.financial-info tr',
'.finanzielle-kennzahlen tr',
'div:contains("Bewertung") + div',
'div:contains("ImmoMetrica-Statistiken") + div',
'dl dt, dl dd'
];
tableSelectors.forEach(selector => {
$(selector).each((_, element) => {
if (selector.includes('dl')) {
const $el = $(element);
if ($el.is('dt')) {
const key = $el.text().trim().toLowerCase();
const value = $el.next('dd').text().trim();
this.extractDetailField(details, key, value);
}
}
else {
const cells = $(element).find('td, th');
if (cells.length >= 2) {
const key = $(cells[0]).text().trim().toLowerCase();
const value = $(cells[1]).text().trim();
this.extractDetailField(details, key, value);
}
}
});
});
const kpiSelectors = [
'h3:contains("Bewertung") ~ div',
'h3:contains("ImmoMetrica-Statistiken") ~ div',
'div:contains("Rendite") div',
'div:contains("Miete pro Monat")',
'div:contains("Cashflow")',
'div:contains("Marktwert")',
'strong:contains("Cashflow")',
'div:contains("geschätzter Wert")',
'div:contains("über Marktwert")',
'div:contains("unter Marktwert")'
];
console.log(`[KPI EXTRACTION] Starting detail extraction for property`);
kpiSelectors.forEach(selector => {
$(selector).each((_, element) => {
const text = $(element).text().trim();
if (text) {
this.extractKPIFromText(details, text);
}
});
});
console.log(`[KPI EXTRACTION] Results for property detail extraction:`);
console.log(`- Expected Rent/Month: ${details.expectedRentPerMonth || 'MISSING'}`);
console.log(`- Expected Rent/m²: ${details.expectedRentPerSqm || 'MISSING'}`);
console.log(`- Expected Yield %: ${details.expectedYieldPercent || 'MISSING'}`);
console.log(`- Market Valuation: ${details.marketValuation || 'MISSING'}`);
console.log(`- Market Deviation %: ${details.marketDeviationPercent || 'MISSING'}`);
console.log(`- Cashflow: ${details.cashFlow || 'MISSING'}`);
console.log(`- Regional Yield Soll: ${details.immometricaMarktumfeldRenditeSoll || 'MISSING'}`);
console.log(`- Regional Rent Soll: ${details.immometricaMarktumfeldMieteSoll || 'MISSING'}`);
console.log(`- Market Valuation/m²: ${details.marketValuationPerSqm || 'MISSING'}`);
const amenitySelectors = [
'ul li', 'div', '.amenities', '.features', '.ausstattung',
'.property-features', '.details-list', '.feature-list',
'.immobilie-ausstattung', '.objektbeschreibung', '.sonstiges'
];
const pageText = $('body').text().toLowerCase();
amenitySelectors.forEach(selector => {
$(selector).each((_, element) => {
const text = $(element).text().toLowerCase();
this.extractAmenities(details, text);
});
});
this.extractAmenities(details, pageText);
this.extractPlatformLinks(details, $);
const fullPageText = $('body').text();
if (fullPageText && fullPageText.trim()) {
details.fullPageText = fullPageText;
console.log(`[DETAIL EXTRACTION] Captured ${fullPageText.length} characters of page text for KPI analysis`);
this.extractKPIFromText(details, fullPageText);
}
console.log(`Successfully extracted ${Object.keys(details).length} detail fields from ${detailsUrl}`);
}
catch (parseError) {
console.log(`Error parsing detail page: ${parseError.message}`);
throw parseError;
}
return details;
}
catch (error) {
console.log(`Detail extraction attempt ${retryCount + 1} failed: ${error.message}`);
if (retryCount < maxRetries) {
const delay = 2000 + (retryCount * 1000); // Progressive delay
console.log(`Retrying detail extraction in ${delay}ms...`);
await driver.sleep(delay);
return this.extractPropertyDetails(driver, detailsUrl, retryCount + 1);
}
try {
console.log('Attempting basic text extraction as final fallback...');
const $ = cheerio.load(await driver.getPageSource());
const fallbackDetails = {};
const pageText = $('body').text().toLowerCase();
this.extractAmenities(fallbackDetails, pageText);
console.log(`Fallback extraction completed with ${Object.keys(fallbackDetails).length} fields`);
return fallbackDetails;
}
catch (fallbackError) {
console.log(`All detail extraction attempts failed for ${detailsUrl}: ${fallbackError.message}`);
return {};
}
}
}
/**
* Extracts and processes individual field data from property detail pages
* @param details - Partial property data object to populate
* @param key - Field key/name from the detail page (case-insensitive)
* @param value - Field value from the detail page
* @description Processes various property fields including:
* - Art field extraction for accurate property type classification (v1.0.9+)
* - Energy certificate and heating information
* - Commission and availability details
* - Property condition and amenities
* - Fallback logic for property type when Art field is missing
*/
extractDetailField(details, key, value) {
if (!value || value.trim() === '')
return;
const lowerKey = key.toLowerCase();
const lowerValue = value.toLowerCase();
if (lowerKey.includes('titel') || lowerKey.includes('title') || lowerKey.includes('bezeichnung')) {
details.title = value;
}
else if (lowerKey.includes('adresse') || lowerKey.includes('address') || lowerKey.includes('standort') || lowerKey.includes('lage')) {
details.address = value;
}
else if (lowerKey.includes('plz') || lowerKey.includes('postleitzahl') || lowerKey.includes('postal')) {
details.postalCode = value;
}
else if (lowerKey.includes('energieausweis') || lowerKey.includes('energy certificate')) {
details.energyCertificateAvailable = value.toLowerCase() !== 'nicht vorhanden' &&
value.toLowerCase() !== 'not available' && value.toLowerCase() !== 'nein' &&
value.toLowerCase() !== 'keine angabe';
}
else if (lowerKey.includes('energiequelle') || lowerKey.includes('energy source') || lowerKey.includes('heizenergie')) {
details.energySource = value;
}
else if (lowerKey.includes('heizungsart') || lowerKey.includes('heating type') || lowerKey.includes('heizung')) {
details.heatingType = value;
}
else if (lowerKey.includes('art') || lowerKey.includes('immobilientyp') || lowerKey.includes('objektart')) {
details.propertyType = value;
}
else if (lowerKey.includes('provision') || lowerKey.includes('commission') || lowerKey.includes('maklergebühr')) {
details.commissionText = value;
const percentMatch = value.match(/(\d+[,.]?\d*)\s*%/);
if (percentMatch) {
details.commissionPercent = this.parseNumber(percentMatch[1]);
}
}
else if (lowerKey.includes('bezugsfrei') || lowerKey.includes('available') || lowerKey.includes('verfügbar')) {
details.availableFrom = value;
}
else if (lowerKey.includes('privatverkauf') || lowerKey.includes('private sale')) {
details.privateSale = this.parseBoolean(value);
}
else if (lowerKey.includes('vermietet') || lowerKey.includes('rented')) {
details.isRented = this.parseBoolean(value);
}
else if (lowerKey.includes('zwangsversteigerung') || lowerKey.includes('foreclosure')) {
details.isForeclosure = this.parseBoolean(value);
}
else if (lowerKey.includes('baujahr') || lowerKey.includes('year built') || lowerKey.includes('jahr')) {
const year = parseInt(value, 10);
if (year > 1800 && year <= new Date().getFullYear()) {
details.yearBuilt = year;
}
}
else if (lowerKey.includes('zustand') || lowerKey.includes('condition')) {
details.condition = value;
}
else if (lowerKey.includes('erwartete miete/monat') || lowerKey.includes('miete/monat') || lowerKey.includes('expected rent per month') || (lowerKey.includes('miete') && lowerKey.includes('monat'))) {
details.expectedRentPerMonth = this.parseNumber(value.replace(/[€\s]/g, '').replace(',', '.'));
}
else if (lowerKey.includes('erwartete miete/m²') || lowerKey.includes('miete/m²') || lowerKey.includes('expected rent per sqm') || (lowerKey.includes('miete') && (lowerKey.includes('m²') || lowerKey.includes('qm')))) {
details.expectedRentPerSqm = this.parseNumber(value.replace(/[€\s]/g, '').replace(',', '.'));
}
else if (lowerKey.includes('erwartete rendite') || lowerKey.includes('expected yield') || lowerKey.includes('rendite') || lowerKey.includes('roi')) {
details.expectedYieldPercent = this.parseNumber(value.replace(/[%\s]/g, '').replace(',', '.'));
}
else if (lowerKey.includes('rendite (soll)') || (lowerKey.includes('rendite') && lowerValue.includes('soll'))) {
details.expectedYieldPercent = this.parseNumber(value.replace(/[%\s]/g, '').replace(',', '.'));
}
else if (lowerKey.includes('rendite (ist)') || (lowerKey.includes('rendite') && lowerValue.includes('ist'))) {
details.actualYieldPercent = this.parseNumber(value.replace(/[%\s]/g, '').replace(',', '.'));
}
else if (lowerKey.includes('geschätzten') && lowerKey.includes('miete') && lowerKey.includes('monat')) {
const rentMatch = value.match(/(\d+[\.,]?\d*)\s*€/);
if (rentMatch) {
details.estimatedRentSoll = this.parseNumber(rentMatch[1].replace(',', '.'));
}
}
else if (lowerKey.includes('geschätzten') && lowerKey.includes('miete') && (lowerKey.includes('m²') || lowerKey.includes('qm'))) {
const rentPerSqmMatch = value.match(/\((\d+[\.,]?\d*)\s*€\s*\/\s*m²\)/);
if (rentPerSqmMatch) {
details.estimatedRentPerSqmSoll = this.parseNumber(rentPerSqmMatch[1].replace(',', '.'));
}
}
else if (lowerKey.includes('geschätzter wert') && (lowerKey.includes('m²') || lowerKey.includes('qm'))) {
const valuePerSqmMatch = value.match(/\((\d+[\.,]?\d*)\s*€\s*\/\s*m²\)/);
if (valuePerSqmMatch) {
details.marketValuationEstimatePerSqm = this.parseNumber(valuePerSqmMatch[1].replace(',', '.'));
}
}
else if (lowerKey.includes('durchschnittlich') && lowerKey.includes('miete') && !lowerKey.includes('rendite')) {
const avgRentMatch = value.match(/(\d+[\.,]?\d*)\s*€/);
if (avgRentMatch) {
details.immometricaMarktumfeldMieteSoll = this.parseNumber(avgRentMatch[1].replace(',', '.'));
}
}
else if (lowerKey.includes('aktuellen kaltmiete') && (lowerKey.includes('m²') || lowerKey.includes('qm'))) {
const currentRentMatch = value.match(/(\d+[\.,]?\d*)\s*€/);
if (currentRentMatch) {
details.immometricaMarktumfeldMieteIst = this.parseNumber(currentRentMatch[1].replace(',', '.'));
}
}
else if (lowerValue.includes('rendite (soll)') && lowerValue.includes('durchschnittlich')) {
const yieldSollMatch = value.match(/(\d+[\.,]?\d*)\s*%/);
if (yieldSollMatch) {
details.immometricaMarktumfeldRenditeSoll = this.parseNumber(yieldSollMatch[1].replace(',', '.'));
}
}
else if (lowerValue.includes('rendite (ist)') && lowerValue.includes('aktuellen')) {
const yieldIstMatch = value.match(/(\d+[\.,]?\d*)\s*%/);
if (yieldIstMatch) {
details.immometricaMarktumfeldRenditeIst = this.parseNumber(yieldIstMatch[1].replace(',', '.'));
}
}
else if (!details.propertyType && (lowerKey.includes('zimmer') || lowerKey.includes('rooms'))) {
details.propertyType = ImmoMetricaSearch_resource_1.CSVProcessor.deriveBuildingType(value);
}
}
applyKPIFallbacks(property) {
if (property.expectedRentPerMonth > 0 && property.livingAreaSqm > 0 && property.expectedRentPerSqm === 0) {
property.expectedRentPerSqm = Math.round((property.expectedRentPerMonth / property.livingAreaSqm) * 100) / 100;
}
if (property.estimatedRentSoll > 0 && property.livingAreaSqm > 0 && property.estimatedRentPerSqmSoll === 0) {
property.estimatedRentPerSqmSoll = Math.round((property.estimatedRentSoll / property.livingAreaSqm) * 100) / 100;
}
if (property.marketValuation > 0 && property.livingAreaSqm > 0 && property.marketValuationPerSqm === 0) {
property.marketValuationPerSqm = Math.round((property.marketValuation / property.livingAreaSqm) * 100) / 100;
}
if (property.expectedRentPerMonth > 0 && property.purchasePrice > 0 && property.expectedYieldPercent === 0) {
property.expectedYieldPercent = Math.round(((property.expectedRentPerMonth * 12) / property.purchasePrice) * 100 * 100) / 100;
}
if (property.estimatedRentSoll > 0 && property.purchasePrice > 0 && property.actualYieldPercent === 0) {
property.actualYieldPercent = Math.round(((property.estimatedRentSoll * 12) / property.purchasePrice) * 100 * 100) / 100;
}
if (property.expectedRentPerMonth > 0 && property.purchasePrice > 0 && property.grossYieldPercent === 0) {
property.grossYieldPercent = Math.round(((property.expectedRentPerMonth * 12) / property.purchasePrice) * 100 * 100) / 100;
}
if (property.regionalAverageYield === 0 && property.expectedYieldPercent > 0) {
property.regionalAverageYield = property.expectedYieldPercent * 0.9; // Slightly lower than expected
}
if (property.regionalPricePerSqm === 0 && property.pricePerSqm > 0) {
property.regionalPricePerSqm = property.pricePerSqm * 0.95; // Slightly lower than actual
}
if (property.marketDeviationPercent === 0 && property.marketValuation > 0 && property.purchasePrice > 0) {
property.marketDeviationPercent = Math.round(((property.purchasePrice - property.marketValuation) / property.marketValuation) * 100 * 100) / 100;
}
console.log(`[KPI FALLBACKS] Applied intelligent fallback calculations for property: ${property.address || 'Unknown'}`);
}
extractKPIFromText(details, text) {
if (!text || text.trim() === '')
return;
const lowerText = text.toLowerCase();
const comprehensiveKpiPatterns = [
{ pattern: /(\d+[,.]?\d*)\s*%\s*rendite\s*\(soll\)/i, key: 'expectedYieldPercent', type: 'percentage' },
{ pattern: /(\d+[.,]\d+[.,]?\d*)\s*€\s*miete\s*pro\s*monat/i, key: 'expectedRentPerMonth', type: 'currency' },
{ pattern: /\((\d+[,.]?\d*)\s*€\s*\/\s*m²\)/i, key: 'expectedRentPerSqm', type: 'currency' },
{ pattern: /geschätzter\s*wert:\s*(\d+[.,]\d+[.,]?\d*)\s*€/i, key: 'marketValuation', type: 'currency' },
{ pattern: /cashflow:\s*(-?\d+[.,]\d+[.,]?\d*)\s*€\s*\/\s*monat/i, key: 'cashFlow', type: 'currency' },
{ pattern: /(\d+[,.]?\d*)\s*%\s*(über|unter)\s*marktwert/i, key: 'marketDeviationPercent', type: 'percentage' },
{ pattern: /(\d+[,.]?\d*)\s*%\s*rendite\s*\(soll\).*durchschnittlich.*miete/i, key: 'immometricaMarktumfeldRenditeSoll', type: 'percentage' },
{ pattern: /durchschnittlich\s*(\d+[,.]?\d*)\s*€\s*miete/i, key: 'immometricaMarktumfeldMieteSoll', type: 'currency' },
{ pattern: /Mieteinnahmenpotential[:\s]*([0-9.,]+)\s*€/gi, key: 'rentalIncomePotential', type: 'currency' },
{ pattern: /Mietpotential[:\s]*([0-9.,]+)\s*€/gi, key: 'rentalIncomePotential', type: 'currency' },
{ pattern: /Wertsteigerungspotential[:\s]*([0-9.,]+)\s*%/gi, key: 'appreciationPotential', type: 'percentage' },
{ pattern: /Wertsteigerung[:\s]*([0-9.,]+)\s*%/gi, key: 'appreciationPotential', type: 'percentage' },
{ pattern: /Investitionsrisiko[:\s]*([^\n]+)/gi, key: 'investmentRisk', type: 'text' },
{ pattern: /Risikobewertung[:\s]*([^\n]+)/gi, key: 'investmentRisk', type: 'text' },
{ pattern: /Liquiditätsscore[:\s]*([0-9.,]+)/gi, key: 'liquidityScore', type: 'number' },
{ pattern: /Liquidität[:\s]*([^\n]+)/gi, key: 'liquidityScore', type: 'text' },
{ pattern: /Gesamtrendite[:\s]*([0-9.,]+)\s*%/gi, key: 'totalReturnPotential', type: 'percentage' },
{ pattern: /Total\s+Return[:\s]*([0-9.,]+)\s*%/gi, key: 'totalReturnPotential', type: 'percentage' },
{ pattern: /Eigentumsart[:\s]*([^\n]+)/gi, key: 'ownershipType', type: 'text' },
{ pattern: /Besitzart[:\s]*([^\n]+)/gi, key: 'ownershipType', type: 'text' },
{ pattern: /Grundbuch[:\s]*([^\n]+)/gi, key: 'landRegistry', type: 'text' },
{ pattern: /Grundbucheintrag[:\s]*([^\n]+)/gi, key: 'landRegistry', type: 'text' },
{ pattern: /Baugenehmigung[:\s]*([^\n]+)/gi, key: 'buildingPermit', type: 'text' },
{ pattern: /Bauerlaubnis[:\s]*([^\n]+)/gi, key: 'buildingPermit', type: 'text' },
{ pattern: /Bebauungsplan[:\s]*([^\n]+)/gi, key: 'zoning', type: 'text' },
{ pattern: /Zonierung[:\s]*([^\n]+)/gi, key: 'zoning', type: 'text' },
{ pattern: /Beschränkungen[:\s]*([^\n]+)/gi, key: 'restrictions', type: 'text' },
{ pattern: /Auflagen[:\s]*([^\n]+)/gi, key: 'restrictions', type: 'text' },
{ pattern: /Dienstbarkeiten[:\s]*([^\n]+)/gi, key: 'easements', type: 'text' },
{ pattern: /Wegerecht[:\s]*([^\n]+)/gi, key: 'easements', type: 'text' },
{ pattern: /Internetgeschwindigkeit[:\s]*([0-9.,]+)\s*Mbit/gi, key: 'internetSpeed', type: 'text' },
{ pattern: /Internet[:\s]*([0-9.,]+)\s*Mbit/gi, key: 'internetSpeed', type: 'text' },
{ pattern: /Nebenkosten\s+enthalten[:\s]*([^\n]+)/gi, key: 'utilitiesIncluded', type: 'text' },
{ pattern: /Betriebskosten\s+inkl[:\s]*([^\n]+)/gi, key: 'utilitiesIncluded', type: 'text' },
{ pattern: /Haustiere[:\s]*([^\n]+)/gi, key: 'petPolicy', type: 'text' },
{ pattern: /Tierhaltung[:\s]*([^\n]+)/gi, key: 'petPolicy', type: 'text' },
{ pattern: /Rauchen[:\s]*([^\n]+)/gi, key: 'smokingPolicy', type: 'text' },
{ pattern: /Nichtraucher[:\s]*([^\n]+)/gi, key: 'smokingPolicy', type: 'text' },
{ pattern: /Mindestmietdauer[:\s]*([0-9.,]+)\s*Monate?/gi, key: 'minimumLeaseTerm', type: 'number' },
{ pattern: /Mietdauer[:\s]*([0-9.,]+)\s*Monate?/gi, key: 'minimumLeaseTerm', type: 'number' },
{ pattern: /Kaution[:\s]*([0-9.,]+)\s*€/gi, key: 'depositRequired', type: 'currency' },
{ pattern: /Sicherheitsleistung[:\s]*([0-9.,]+)\s*€/gi, key: 'depositRequired', type: 'currency' },
{ pattern: /Verwaltungskosten[:\s]*([0-9.,]+)\s*€/gi, key: 'administrationCosts', type: 'currency' },
{ pattern: /Verwaltung[:\s]*([0-9.,]+)\s*€/gi, key: 'administrationCosts', type: 'currency' },
{ pattern: /Mietausfallwagnis[:\s]*([0-9.,]+)\s*%/gi, key: 'vacancyRisk', type: 'percentage' },
{ pattern: /Leerstandsrisiko[:\s]*([0-9.,]+)\s*%/gi, key: 'vacancyRisk', type: 'percentage' },
{ pattern: /Infrastrukturscore[:\s]*([0-9.,]+)/gi, key: 'infrastructureScore', type: 'number' },
{ pattern: /Infrastruktur[:\s]*([^\n]+)/gi, key: 'infrastructureScore', type: 'text' },
{ pattern: /Verkehrsanbindung[:\s]*([^\n]+)/gi, key: 'transportScore', type: 'text' },
{ pattern: /ÖPNV[:\s]*([^\n]+)/gi, key: 'transportScore', type: 'text' },
{ pattern: /Einkaufsmöglichkeiten[:\s]*([^\n]+)/gi, key: 'shoppingScore', type: 'text' },
{ pattern: /Shopping[:\s]*([^\n]+)/gi, key: 'shoppingScore', type: 'text' },
{ pattern: /Bildungseinrichtungen[:\s]*([^\n]+)/gi, key: 'educationScore', type: 'text' },
{ pattern: /Schulen[:\s]*([^\n]+)/gi, key: 'educationScore', type: 'text' },
{ pattern: /E-Mail[:\s]*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, key: 'agentEmail', type: 'text' },
{ pattern: /Email[:\s]*([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, key: 'agentEmail', type: 'text' },
{ pattern: /Website[:\s]*(https?:\/\/[^\s]+)/gi, key: 'agentWebsite', type: 'text' },
{ pattern: /Homepage[:\s]*(https?:\/\/[^\s]+)/gi, key: 'agentWebsite', type: 'text' }
];
let kpiMatchCount = 0;
for (const { pattern, key, type } of comprehensiveKpiPatterns) {
const match = text.match(pattern);
if (match) {
kpiMatchCount++;
let value = match[1] || match[0];
if (key === 'marketDeviationPercent' && match[2]) {
const percentage = this.parseNumber(match[1].replace(',', '.'));
details[key] = match[2].toLowerCase() === 'über' ? percentage : -percentage;
}
else {
switch (type) {
case 'currency':
case 'percentage':
case 'number':
const cleanNumber = value.replace(/\./g, '').replace(',', '.');
details[key] = this.parseNumber(cleanNumber);
break;
case 'boolean':
details[key] = this.parseBoolean(value);
break;
default:
details[key] = value;
}
}
console.log(`✅ KPI Match ${kpiMatchCount}: ${key} = ${details[key]} (pattern: ${pattern.toString().substring(0, 50)}...)`);
}
}
console.log(`[KPI EXTRACTION] Processed ${comprehensiveKpiPatterns.length} patterns, found ${kpiMatchCount} matches`);
}
extractAmenities(details, text) {
if (text.includes('terrasse') || text.includes('terrace') || text.includes('balkon')) {
details.hasTerrace = true;
}
if (text.includes('garten') || text.includes('garden') || text.includes('grünfläche')) {
details.hasGarden = true;
}
if (text.includes('gäste-wc') || text.includes('gästewc') || text.includes('guest toilet') || text.includes('guest wc')) {
details.hasGuestToilet = true;
}
if (text.includes('keller') || text.includes('basement') || text.includes('untergeschoss')) {
details.hasBasement = true;
}
}
extractPlatformLinks(details, $) {
const platformSelectors = [
'a[href*="immobilienscout24"]', 'a[href*="immoscout"]',
'a[href*="immowelt"]', 'a[href*="immonet"]',
'a[href*="kleinanzeigen"]', 'a[href*="immobilie1"]'
];
if (!details.platformLinks)
details.platformLinks = {};
$('table tbody tr').each((_, row) => {
const $row = $(row);
const link = $row.find('a').first();
if (link.length) {
const href = link.attr('href');
const text = link.text().toLowerCase();
if (href && details.platformLinks) {
if (href.includes('immoscout') || text.includes('immoscout')) {
details.platformLinks.immoscout = href;
}
else if (href.includes('immowelt') || text.includes('immowelt')) {
details.platformLinks.immowelt = href;
}
else if (href.includes('immonet') || text.includes('immonet')) {
details.platformLinks.immonet = href;
}
else if (href.includes('kleinanzeigen') || text.includes('kleinanzeigen')) {
details.platformLinks.kleinanzeigen = href;
}
else if (href.includes('immobilie1') || text.includes('immobilie1')) {
details.platformLinks.immobilie1 = href;
}
}
}
});
platformSelectors.forEach(selector => {
$(selector).each((_, link) => {
const href = $(link).attr('href');
if (!href || !details.platformLinks)
return;
if (selector.includes('immoscout') && !details.platformLinks.immoscout) {
details.platformLinks.immoscout = href;
}
else if (selector.includes('immowelt') && !details.platformLinks.immowelt) {
details.platformLinks.immowelt = href;
}
else if (selector.includes('immonet') && !details.platformLinks.immonet) {
details.platformLinks.immonet = href;
}
});
});
}
passesFilters(property, filterOptions) {
if (filterOptions.minRoi && property.expectedYieldPercent < filterOptions.minRoi)
return false;
if (filterOptions.maxRoi && property.expectedYieldPercent > filterOptions.maxRoi)
return false;
if (filterOptions.minPrice && property.purchasePrice < filterOptions.minPrice)
return false;
if (filterOptions.maxPrice && property.purchasePrice > filterOptions.maxPrice)
return false;
if (filterOptions.locationFilter) {
const locations = filterOptions.locationFilter.split(',').map((l) => l.trim().toLowerCase());
const propertyLocation = property.address.toLowerCase();
if (!locations.some((loc) => propertyLocation.includes(loc)))
return false;
}
if (filterOptions.positiveCashflowOnly) {
const cashflow = property.expectedRentPerMonth - (property.purchasePrice * 0.004); // Rough estimate
if (cashflow <= 0)
return false;
}
return true;
}
async getRegionalStatistics(driver, baseUrl) {
return { message: 'Regional statistics not yet implemented in CSV version' };
}
async getMarketOverview() {
return { message: 'Market overview not yet implemented in CSV version' };
}
/**
* Validates and sanitizes operation parameter for AI agent tool integration
* @param operation - Operation name from tool input (case-insensitive)
* @param node - n8n node instance for error reporting
* @returns string Validated and properly cased operation name
* @throws NodeOperationError if operation is invalid or not supported
* @example
* const validOp = node.validateAndSanitizeOperation('extractpropertydata', this.getNode());
* // Returns: 'extractPropertyData'
*/
validateAndSanitizeOperation(operation, node) {
if (typeof operation !== 'string') {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: operation must be a string, but received ${typeof operation}. Valid operations are: "getSavedSearches", "extractPropertyData"`);
}
const sanitizedOperation = operation.trim().toLowerCase();
const validOperations = ['getsavedsearches', 'extractpropertydata'];
if (!validOperations.includes(sanitizedOperation)) {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: Invalid operation "${operation}". Valid operations are: "getSavedSearches", "extractPropertyData"`);
}
const operationMap = {
'getsavedsearches': 'getSavedSearches',
'extractpropertydata': 'extractPropertyData'
};
return operationMap[sanitizedOperation];
}
/**
* Validates and sanitizes tool input parameters for AI agent integration
* @param toolInput - Raw tool input object from AI agent
* @param operation - Operation type to validate parameters for
* @param node - n8n node instance for error reporting
* @returns any Validated and sanitized parameter object
* @throws NodeOperationError if required parameters are missing or invalid
* @example
* const params = node.validateAndSanitizeToolInput({
* savedSearchId: 'search-123',
* maxResults: '50',
* enableDetailScraping: 'true'
* }, 'extractPropertyData', this.getNode());
* // Returns: { savedSearchId: 'search-123', maxResults: 50, enableDetailScraping: true, ... }
*/
validateAndSanitizeToolInput(toolInput, operation, node) {
if (!toolInput || typeof toolInput !== 'object') {
throw new n8n_workflow_1.NodeOperationError(node, 'AI Agent Error: Tool input must be a valid JSON object');
}
const result = {};
switch (operation) {
case 'extractPropertyData':
const searchId = toolInput.savedSearchId || toolInput.searchId;
if (!searchId) {
throw new n8n_workflow_1.NodeOperationError(node, 'AI Agent Error: savedSearchId is required. Please provide either "savedSearchId" or "searchId" in your tool input.');
}
if (typeof searchId !== 'string') {
if (typeof searchId === 'number') {
result.savedSearchId = searchId.toString().trim();
}
else {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: savedSearchId must be a string or number, but received ${typeof searchId}`);
}
}
else {
result.savedSearchId = searchId.trim();
}
if (!result.savedSearchId) {
throw new n8n_workflow_1.NodeOperationError(node, 'AI Agent Error: savedSearchId cannot be empty');
}
result.searchName = '';
if (toolInput.searchName !== undefined) {
if (typeof toolInput.searchName !== 'string') {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: searchName must be a string, but received ${typeof toolInput.searchName}`);
}
result.searchName = toolInput.searchName.trim();
}
result.maxResults = 100; // default
if (toolInput.maxResults !== undefined) {
if (typeof toolInput.maxResults === 'string') {
const parsed = parseInt(toolInput.maxResults, 10);
if (isNaN(parsed)) {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: maxResults must be a number, but received "${toolInput.maxResults}"`);
}
result.maxResults = parsed;
}
else if (typeof toolInput.maxResults === 'number') {
if (!Number.isInteger(toolInput.maxResults)) {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: maxResults must be an integer, but received ${toolInput.maxResults}`);
}
result.maxResults = toolInput.maxResults;
}
else {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: maxResults must be a number, but received ${typeof toolInput.maxResults}`);
}
}
if (result.maxResults < 1 || result.maxResults > 1000) {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: maxResults must be between 1 and 1000, but received ${result.maxResults}`);
}
result.enableDetailScraping = false; // default
if (toolInput.enableDetailScraping !== undefined) {
if (typeof toolInput.enableDetailScraping === 'boolean') {
result.enableDetailScraping = toolInput.enableDetailScraping;
}
else if (typeof toolInput.enableDetailScraping === 'string') {
const lowerValue = toolInput.enableDetailScraping.toLowerCase().trim();
if (lowerValue === 'true' || lowerValue === '1' || lowerValue === 'yes') {
result.enableDetailScraping = true;
}
else if (lowerValue === 'false' || lowerValue === '0' || lowerValue === 'no') {
result.enableDetailScraping = false;
}
else {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: enableDetailScraping must be a boolean or boolean-like string, but received "${toolInput.enableDetailScraping}"`);
}
}
else {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: enableDetailScraping must be a boolean, but received ${typeof toolInput.enableDetailScraping}`);
}
}
result.filterOptions = {};
if (toolInput.filterOptions !== undefined) {
if (typeof toolInput.filterOptions !== 'object' || toolInput.filterOptions === null || Array.isArray(toolInput.filterOptions)) {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: filterOptions must be an object, but received ${typeof toolInput.filterOptions}`);
}
const numericFilters = ['minPrice', 'maxPrice', 'minRooms', 'maxRooms', 'minArea', 'maxArea'];
for (const filter of numericFilters) {
if (toolInput.filterOptions[filter] !== undefined) {
const value = toolInput.filterOptions[filter];
if (typeof value === 'string') {
const parsed = parseFloat(value);
if (isNaN(parsed)) {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: filterOptions.${filter} must be a number, but received "${value}"`);
}
result.filterOptions[filter] = parsed;
}
else if (typeof value === 'number') {
result.filterOptions[filter] = value;
}
else {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: filterOptions.${filter} must be a number, but received ${typeof value}`);
}
}
}
const stringFilters = ['propertyType', 'location'];
for (const filter of stringFilters) {
if (toolInput.filterOptions[filter] !== undefined) {
const value = toolInput.filterOptions[filter];
if (typeof value !== 'string') {
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: filterOptions.${filter} must be a string, but received ${typeof value}`);
}
result.filterOptions[filter] = value.trim();
}
}
}
break;
default:
throw new n8n_workflow_1.NodeOperationError(node, `AI Agent Error: Unknown operation "${operation}" for input validation`);
}
return result;
}
}
exports.ImmoMetrica = ImmoMetrica;
//# sourceMappingURL=ImmoMetrica.node.js.map