n8n-nodes-immometrica
Version:
n8n community node for ImmoMetrica real estate data extraction
952 lines (901 loc) • 170 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.ImmoMetrica = void 0;
const n8n_workflow_1 = require("n8n-workflow");
const selenium_webdriver_1 = require("selenium-webdriver");
const chrome = __importStar(require("selenium-webdriver/chrome"));
class ImmoMetrica {
constructor() {
this.description = {
displayName: 'ImmoMetrica',
name: 'immoMetrica',
icon: 'file:immometrica.svg',
group: ['input'],
version: 1,
subtitle: '={{$parameter["operation"]}}',
description: 'Extrahiert Immobiliendaten von ImmoMetrica',
defaults: {
name: 'ImmoMetrica',
},
inputs: ["main"],
outputs: ["main"],
credentials: [
{
name: 'immoMetricaApi',
required: true,
},
],
usableAsTool: true,
properties: [
{
displayName: 'Operation',
name: 'operation',
type: 'options',
noDataExpression: true,
options: [
{
name: 'Gespeicherte Suchen Abrufen',
value: 'getSavedSearches',
description: 'Alle gespeicherten Suchen des Benutzers abrufen. Liefert IDs und Namen aller verfügbaren Immobiliensuchen für weitere Datenextraktion.',
action: 'Gespeicherte Suchen von ImmoMetrica abrufen',
},
{
name: 'Immobiliendaten Extrahieren',
value: 'extractPropertyData',
description: 'Detaillierte Immobiliendaten aus einer gespeicherten Suche extrahieren. Liefert strukturierte Investment-Daten: Preise, ROI, Renditen, Standorte, Cashflow und alle relevanten Kennzahlen für Investment-Analysen.',
action: 'Immobiliendaten von ImmoMetrica extrahieren',
},
{
name: 'Regionale Statistiken',
value: 'getRegionalStats',
description: 'Regionale Marktstatistiken für alle 401 deutschen Landkreise abrufen. Enthält Durchschnittspreise, Renditen und Marktdaten für regionale Investment-Vergleiche und Standortanalysen.',
action: 'Regionale Marktstatistiken von ImmoMetrica abrufen',
},
{
name: 'Marktübersicht',
value: 'getMarketOverview',
description: 'Allgemeine Marktübersicht und Investment-Trends für den deutschen Immobilienmarkt. Liefert makroökonomische Insights und Marktentwicklungen für strategische Investment-Entscheidungen.',
action: 'Marktübersicht von ImmoMetrica abrufen',
},
],
default: 'extractPropertyData',
},
{
displayName: 'Gespeicherte Suche',
name: 'savedSearchId',
type: 'options',
typeOptions: {
loadOptionsMethod: 'getSavedSearches',
},
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: '',
description: 'Wählen Sie eine gespeicherte Suche aus',
required: true,
},
{
displayName: 'Maximale Ergebnisse',
name: 'maxResults',
type: 'number',
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: 50,
description: 'Maximale Anzahl der zu extrahierenden Immobilien',
typeOptions: {
minValue: 1,
maxValue: 1000,
},
},
{
displayName: 'Filter-Optionen',
name: 'filterOptions',
type: 'collection',
displayOptions: {
show: {
operation: ['extractPropertyData'],
},
},
default: {},
placeholder: 'Filter hinzufügen',
description: 'Erweiterte Filterkriterien für die Immobiliensuche',
options: [
{
displayName: 'Mindest-ROI (%)',
name: 'minRoi',
type: 'number',
default: 0,
description: 'Minimaler ROI in Prozent',
typeOptions: {
minValue: 0,
maxValue: 100,
},
},
{
displayName: 'Maximal-ROI (%)',
name: 'maxRoi',
type: 'number',
default: 100,
description: 'Maximaler ROI in Prozent',
typeOptions: {
minValue: 0,
maxValue: 100,
},
},
{
displayName: 'Mindestpreis (€)',
name: 'minPrice',
type: 'number',
default: 0,
description: 'Mindestpreis in Euro',
typeOptions: {
minValue: 0,
},
},
{
displayName: 'Maximalpreis (€)',
name: 'maxPrice',
type: 'number',
default: 10000000,
description: 'Maximalpreis in Euro',
typeOptions: {
minValue: 0,
},
},
{
displayName: 'Standort-Filter',
name: 'locationFilter',
type: 'string',
default: '',
placeholder: 'Berlin, München, Hamburg',
description: 'Kommagetrennte Liste von Standorten zum Filtern',
},
{
displayName: 'Nur positive Cashflows',
name: 'positiveCashflowOnly',
type: 'boolean',
default: false,
description: 'Nur Immobilien mit positivem Cashflow anzeigen',
},
],
},
],
};
this.methods = {
loadOptions: {
async getSavedSearches() {
try {
const credentials = await this.getCredentials('immoMetricaApi');
const { Builder } = require('selenium-webdriver');
const chrome = require('selenium-webdriver/chrome');
const options = new chrome.Options();
options.addArguments('--headless', '--no-sandbox', '--disable-dev-shm-usage');
const driver = await new Builder().forBrowser('chrome').setChromeOptions(options).build();
const nodeInstance = new ImmoMetrica();
try {
await nodeInstance.loginToImmoMetrica(driver, credentials.email, credentials.password, 'https://www.immometrica.com');
const searches = await nodeInstance.getSavedSearches(driver, 'https://www.immometrica.com');
return searches.map((search) => ({
name: search.title,
value: search.id,
}));
}
finally {
await driver.quit();
}
}
catch (error) {
console.error('LoadOptions error:', error);
return [
{
name: 'Error loading searches - check credentials',
value: 'error',
}
];
}
}
}
};
}
async execute() {
const items = this.getInputData();
const returnData = [];
const operation = this.getNodeParameter('operation', 0);
const credentials = await this.getCredentials('immoMetricaApi');
const email = credentials.email;
const password = credentials.password;
const baseUrl = credentials.baseUrl;
let driver = null;
const nodeInstance = new ImmoMetrica();
try {
const chromeOptions = new chrome.Options();
chromeOptions.addArguments('--headless');
chromeOptions.addArguments('--no-sandbox');
chromeOptions.addArguments('--disable-dev-shm-usage');
chromeOptions.addArguments('--disable-gpu');
chromeOptions.addArguments('--disable-extensions');
chromeOptions.addArguments('--disable-web-security');
chromeOptions.addArguments('--allow-running-insecure-content');
chromeOptions.addArguments('--window-size=1920,1080');
chromeOptions.addArguments('--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36');
try {
driver = await new selenium_webdriver_1.Builder()
.forBrowser('chrome')
.setChromeOptions(chromeOptions)
.build();
}
catch (driverError) {
const errorMsg = driverError.message;
if (errorMsg.includes('status 127') || errorMsg.includes('terminated early')) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Chrome/Chromium ist in Ihrer n8n-Umgebung nicht verfügbar.
LÖSUNG für Docker/Container:
1. Installieren Sie Chrome in Ihrem n8n-Container:
apt-get update && apt-get install -y google-chrome-stable
2. Oder fügen Sie zu Ihrer Dockerfile hinzu:
RUN apt-get update && apt-get install -y google-chrome-stable
3. Für n8n Cloud: Diese Node benötigt eine Self-Hosted n8n Installation mit Chrome.
ALTERNATIVE: Verwenden Sie einen HTTP-Request Node mit Session-Management für einfache Datenextraktion.
Technischer Fehler: ${errorMsg}`);
}
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Chrome WebDriver Fehler: ${errorMsg}`);
}
await nodeInstance.loginToImmoMetrica(driver, email, password, baseUrl);
for (let i = 0; i < items.length; i++) {
let result = {};
switch (operation) {
case 'getSavedSearches':
result = await nodeInstance.getSavedSearches(driver, baseUrl);
break;
case 'extractPropertyData': {
const savedSearchId = this.getNodeParameter('savedSearchId', i);
const maxResults = this.getNodeParameter('maxResults', i);
const filterOptions = this.getNodeParameter('filterOptions', i);
result = await nodeInstance.extractPropertyData(driver, baseUrl, savedSearchId, maxResults, filterOptions);
break;
}
case 'getRegionalStats':
result = await nodeInstance.getRegionalStatistics(driver, baseUrl);
break;
case 'getMarketOverview':
result = await nodeInstance.getMarketOverview();
break;
default:
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Unbekannte Operation: ${operation}`);
}
if (Array.isArray(result)) {
result.forEach((item) => {
returnData.push({
json: item,
pairedItem: { item: i },
});
});
}
else {
returnData.push({
json: result,
pairedItem: { item: i },
});
}
}
return [returnData];
}
catch (error) {
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `ImmoMetrica Fehler: ${error.message}`);
}
finally {
if (driver) {
await driver.quit();
}
}
}
async extractPropertyDetails(driver, detailsUrl) {
try {
console.log(`Starting property details extraction for URL: ${detailsUrl}`);
await new Promise(resolve => setTimeout(resolve, 3000 + Math.random() * 4000));
let navigationAttempts = 0;
const maxNavigationAttempts = 3;
while (navigationAttempts < maxNavigationAttempts) {
try {
await driver.get(detailsUrl);
await driver.wait(selenium_webdriver_1.until.elementLocated(selenium_webdriver_1.By.css('main')), 15000);
const currentUrl = await driver.getCurrentUrl();
console.log(`Navigation attempt ${navigationAttempts + 1}: Current URL is ${currentUrl}`);
if (!currentUrl.includes('/de/offer/') && !currentUrl.includes('/offer/')) {
throw new Error(`Navigation failed: Expected detail page but got ${currentUrl}`);
}
if (currentUrl.includes('/de/search') || currentUrl.includes('/search')) {
throw new Error(`Redirected to search page instead of detail page: ${currentUrl}`);
}
console.log(`Successfully navigated to property detail page: ${currentUrl}`);
break;
} catch (navError) {
navigationAttempts++;
console.log(`Navigation attempt ${navigationAttempts} failed: ${navError.message}`);
if (navigationAttempts >= maxNavigationAttempts) {
throw new Error(`Failed to navigate to property detail page after ${maxNavigationAttempts} attempts. Last error: ${navError.message}`);
}
console.log(`Retrying navigation in 2 seconds...`);
await new Promise(resolve => setTimeout(resolve, 2000));
}
}
const details = {};
console.log('Starting comprehensive section expansion and data extraction');
try {
console.log('First expansion attempt - expanding all collapsible sections');
await this.expandAllSections(driver);
await driver.sleep(1500);
console.log('Second expansion attempt - ensuring all sections are fully expanded');
await this.expandAllSections(driver);
await driver.sleep(1000);
console.log('Verifying section expansion status');
const collapsedElements = await driver.findElements(selenium_webdriver_1.By.css('[aria-expanded="false"]'));
console.log(`Found ${collapsedElements.length} still collapsed elements after expansion attempts`);
} catch (expansionError) {
console.log(`Error during section expansion: ${expansionError.message}`);
console.log('Continuing with extraction despite expansion issues');
}
try {
console.log('Extracting from Eigenschaften section with stale element prevention');
await this.extractFromEigenschaftenSection(driver, details);
} catch (eigenschaftenError) {
console.log(`Error extracting Eigenschaften: ${eigenschaftenError.message}`);
console.log('Attempting retry with fresh element lookup');
try {
await driver.sleep(1000);
await this.extractFromEigenschaftenSection(driver, details);
} catch (retryError) {
console.log(`Eigenschaften retry failed: ${retryError.message}`);
}
}
try {
console.log('Extracting from Energiedaten section with stale element prevention');
await this.extractFromEnergiedatenSection(driver, details);
} catch (energiedatenError) {
console.log(`Error extracting Energiedaten: ${energiedatenError.message}`);
console.log('Attempting retry with fresh element lookup');
try {
await driver.sleep(1000);
await this.extractFromEnergiedatenSection(driver, details);
} catch (retryError) {
console.log(`Energiedaten retry failed: ${retryError.message}`);
}
}
try {
console.log('Extracting from specific sections with stale element prevention');
await this.extractFromSpecificSections(driver, details);
} catch (specificSectionsError) {
console.log(`Error extracting specific sections: ${specificSectionsError.message}`);
console.log('Attempting retry with fresh element lookup');
try {
await driver.sleep(1000);
await this.extractFromSpecificSections(driver, details);
} catch (retryError) {
console.log(`Specific sections retry failed: ${retryError.message}`);
}
}
try {
const hiddenContentSelectors = [
'.hidden-content',
'.collapsed-content',
'.expandable-content',
'[style*="display: none"]',
'[style*="visibility: hidden"]'
];
for (const selector of hiddenContentSelectors) {
try {
const hiddenElements = await driver.findElements(selenium_webdriver_1.By.css(selector));
for (const element of hiddenElements) {
try {
await driver.executeScript("arguments[0].style.display = 'block'; arguments[0].style.visibility = 'visible';", element);
console.log(`Made hidden content visible: ${selector}`);
}
catch (e) {
console.log('Error making content visible:', e.message);
}
}
}
catch (e) {
console.log('Error in section expansion:', e.message);
}
}
}
catch (e) {
console.log('Error in extraction process:', e.message);
}
try {
const expandButtons = await driver.findElements(selenium_webdriver_1.By.css('div[aria-expanded="false"]'));
for (const button of expandButtons) {
try {
await button.click();
await driver.sleep(500);
}
catch (e) {
console.log('Error in operation:', e.message);
}
}
let energyTable;
try {
energyTable = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[text()='Energiedaten']/../following-sibling::table"));
}
catch {
energyTable = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[contains(text(),'Energiedaten')]/following-sibling::table | //h3[contains(text(),'Energiedaten')]/../following-sibling::table"));
}
const energyRows = await energyTable.findElements(selenium_webdriver_1.By.css('tr'));
for (const row of energyRows) {
const cells = await row.findElements(selenium_webdriver_1.By.css('th, td'));
if (cells.length >= 2) {
const key = await cells[0].getText();
const value = await cells[1].getText();
switch (key.trim()) {
case 'Energieausweis':
details.energy_certificate = value;
break;
case 'Energieeffizienzklasse':
details.energy_efficiency_class = value;
break;
case 'energy pass type':
details.energy_pass_type = value;
break;
case 'Primärenergieträger':
details.primary_energy_source = value;
break;
}
}
}
}
catch (error) {
console.log(`Could not extract Energiedaten: ${error.message}`);
console.log(`Page URL: ${await driver.getCurrentUrl()}`);
console.log(`Page title: ${await driver.getTitle()}`);
}
try {
const evaluationSection = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[text()='Bewertung']/../.."));
const evaluationText = await evaluationSection.getText();
const yieldMatch = evaluationText.match(/(\d+,?\d*)\s*%\s*Rendite/);
if (yieldMatch)
details.estimated_yield = yieldMatch[1];
const rentMatch = evaluationText.match(/(\d+,?\d*)\s*€\s*Miete pro Monat/);
if (rentMatch)
details.estimated_monthly_rent = rentMatch[1];
const rentPerSqmMatch = evaluationText.match(/\((\d+)\s*€\s*\/\s*m²\)/);
if (rentPerSqmMatch)
details.estimated_rent_per_sqm = rentPerSqmMatch[1];
const marketValueMatch = evaluationText.match(/(\d+)\s*%\s*über Marktwert/);
if (marketValueMatch)
details.market_value_deviation = marketValueMatch[1];
const coldRentMatch = evaluationText.match(/aktuelle\s+Kaltmiete[:\s]*(\d+,?\d*)\s*€/i);
if (coldRentMatch)
details.cold_rent = coldRentMatch[1];
const coldRentPerSqmMatch = evaluationText.match(/Kaltmiete\s*\/\s*m²[:\s]*(\d+,?\d*)\s*€/i);
if (coldRentPerSqmMatch)
details.cold_rent_per_sqm = coldRentPerSqmMatch[1];
const commissionMatch = evaluationText.match(/Maklerprovision[:\s]*(\d+,?\d*)\s*%/i);
if (commissionMatch)
details.commission_percentage = commissionMatch[1];
if (marketValueMatch)
details.market_value_comparison = marketValueMatch[1];
const estimatedValueMatch = evaluationText.match(/geschätzter Wert:\s*(\d+\.?\d*)\s*€/);
if (estimatedValueMatch)
details.estimated_value = estimatedValueMatch[1];
}
catch (error) {
console.log('Could not extract Bewertung:', error.message);
}
try {
const expandButtons = await driver.findElements(selenium_webdriver_1.By.css('div[aria-expanded="false"]'));
for (const button of expandButtons) {
try {
await button.click();
await driver.sleep(500);
}
catch (e) {
console.log('Error in operation:', e.message);
}
}
let statsSection;
try {
statsSection = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[text()='ImmoMetrica-Statistiken (Marktumfeld)']/../../following-sibling::div"));
}
catch {
statsSection = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[contains(text(),'ImmoMetrica-Statistiken')]/following-sibling::div | //h3[contains(text(),'ImmoMetrica-Statistiken')]/../following-sibling::div"));
}
const statsText = await statsSection.getText();
const marketYieldMatch = statsText.match(/(\d+,?\d*)\s*%\s*Rendite.*bei durchschnittlich\s*(\d+,?\d*)\s*€\s*Miete/);
if (marketYieldMatch) {
details.market_average_yield = marketYieldMatch[1];
details.market_average_rent = marketYieldMatch[2];
}
const marketPriceMatch = statsText.match(/(\d+)\s*%\s*über Marktmittel.*(\d+\.?\d*)\s*€\s*pro m²/);
if (marketPriceMatch) {
details.market_price_comparison = marketPriceMatch[1];
details.market_average_price_per_sqm = marketPriceMatch[2];
}
}
catch (error) {
console.log(`Could not extract ImmoMetrica-Statistiken: ${error.message}`);
console.log(`Page URL: ${await driver.getCurrentUrl()}`);
console.log(`Page title: ${await driver.getTitle()}`);
}
try {
const cashflowElement = await driver.findElement(selenium_webdriver_1.By.xpath("//strong[contains(text(), 'Cashflow:')]"));
const cashflowText = await cashflowElement.getText();
const cashflowMatch = cashflowText.match(/Cashflow:\s*([-+]?\d+,?\d*)\s*€/);
if (cashflowMatch)
details.monthly_cashflow = cashflowMatch[1];
}
catch (error) {
console.log('Could not extract Cashflow:', error.message);
}
try {
const infoSection = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[text()='Infos']/../.."));
const infoText = await infoSection.getText();
const addressMatch = infoText.match(/([A-Za-zäöüß\-.s]+\d+[a-z]?,\s*\d{5}\s+[A-Za-zäöüß\s()]+)/);
if (addressMatch)
details.full_address = addressMatch[1].trim();
const daysOnlineMatch = infoText.match(/(\d+)\s*Tage online/);
if (daysOnlineMatch)
details.days_online = parseInt(daysOnlineMatch[1]);
const onlineSinceMatch = infoText.match(/seit\s+(\d{1,2}\.\d{1,2}\.\d{4})/);
if (onlineSinceMatch)
details.online_since = onlineSinceMatch[1];
const lastActivityMatch = infoText.match(/zuletzt\s+aktiv\s+am\s+(\d{1,2}\.\d{1,2}\.\d{4})/);
if (lastActivityMatch)
details.last_activity = lastActivityMatch[1];
const objectIdMatch = infoText.match(/Objekt-ID:\s*([A-Za-z0-9\-_]+)/);
if (objectIdMatch)
details.object_id = objectIdMatch[1];
const referenceMatch = infoText.match(/Referenz:\s*([A-Za-z0-9\-_/]+)/);
if (referenceMatch)
details.reference_number = referenceMatch[1];
const contactMatch = infoText.match(/Kontakt:\s*([^\n]+)/);
if (contactMatch)
details.contact_info = contactMatch[1].trim();
const phoneMatch = infoText.match(/Tel.?\s*:?\s*([+\d\s\-()/]+)/);
if (phoneMatch)
details.phone_number = phoneMatch[1].trim();
const emailMatch = infoText.match(/([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+.[a-zA-Z]{2,})/);
if (emailMatch)
details.email = emailMatch[1];
const portalLinks = await driver.findElements(selenium_webdriver_1.By.css('table a[href*="immowelt"], table a[href*="immonet"], table a[href*="immobilienscout24"], table a[href*="ebay-kleinanzeigen"], table a[href*="immoweb"], table a[href*="athome"]'));
details.portal_links = [];
for (const link of portalLinks) {
const href = await link.getAttribute('href');
const text = await link.getText();
if (href && text) {
details.portal_links.push({ portal: text.trim(), url: href });
}
}
let infoTable;
try {
infoTable = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[text()='Infos']/following-sibling::div//table"));
}
catch {
infoTable = await driver.findElement(selenium_webdriver_1.By.xpath("//h3[contains(text(),'Infos')]/following-sibling::div//table | //h3[contains(text(),'Infos')]/../following-sibling::div//table"));
}
const infoRows = await infoTable.findElements(selenium_webdriver_1.By.css('tr'));
for (const row of infoRows) {
const cells = await row.findElements(selenium_webdriver_1.By.css('th, td'));
if (cells.length >= 2) {
const key = await cells[0].getText();
const value = await cells[1].getText();
switch (key.trim()) {
case 'Verfügbar ab':
details.available_from = value;
break;
case 'Besichtigung':
details.viewing_info = value;
break;
case 'Provision':
details.commission_info = value;
break;
case 'Kaution':
details.deposit_info = value;
break;
case 'Nebenkosten':
details.additional_costs_info = value;
break;
case 'Heizkosten':
details.heating_costs = value;
break;
case 'Warmmiete':
details.warm_rent = value;
break;
case 'Kaltmiete':
details.cold_rent = value;
break;
case 'Stellplatz':
details.parking_info = value;
break;
case 'Etage':
details.floor = value;
break;
case 'Anzahl Etagen':
details.total_floors = value;
break;
case 'Wohnungstyp':
details.apartment_type = value;
break;
case 'Ausstattung':
details.equipment = value;
break;
case 'Haustiere':
details.pets_info = value;
break;
case 'WG geeignet':
details.shared_flat_suitable = value;
break;
case 'Raucher':
details.smoking_allowed = value;
break;
}
}
}
const statusElements = await driver.findElements(selenium_webdriver_1.By.css('.status, .listing-status, [class*="status"]'));
for (const statusEl of statusElements) {
const statusText = await statusEl.getText();
if (statusText && statusText.trim()) {
details.listing_status = statusText.trim();
break;
}
}
}
catch (error) {
console.log(`Could not extract additional info: ${error.message}`);
console.log(`Page URL: ${await driver.getCurrentUrl()}`);
console.log(`Page title: ${await driver.getTitle()}`);
}
await this.extractMarketStatistics(driver, details);
await this.extractContactInformation(driver, details);
await this.extractAdditionalKPIs(driver, details);
this.extractCriticalFieldsFallback(details);
this.normalizeExtractedKeys(details);
console.log('🔍 DEBUG: Raw extracted data before structuring:', JSON.stringify(details, null, 2));
console.log('🔍 DEBUG: Critical field values:', {
purchasePrice: details?.purchasePrice,
livingArea: details?.livingArea,
rooms: details?.rooms,
yearBuilt: details?.yearBuilt,
preis: details?.preis,
kaufpreis: details?.kaufpreis,
wohnflaeche: details?.wohnflaeche,
zimmer: details?.zimmer,
baujahr: details?.baujahr
});
console.log('🔍 DEBUG: Total extracted fields count:', Object.keys(details || {}).length);
for (const key in details) {
if (typeof details[key] === "undefined") {
console.warn(`⚠️ Undefined value for key: ${key}`);
}
}
const structuredData = this.structurePropertyOutput(details);
console.log('🔍 DEBUG: Final structured data:', JSON.stringify(structuredData, null, 2));
console.log('🔍 DEBUG: Critical structured fields:', {
purchasePrice: structuredData?.base_data?.purchasePrice,
livingArea: structuredData?.base_data?.livingArea,
rooms: structuredData?.base_data?.rooms,
yearBuilt: structuredData?.base_data?.yearBuilt,
location: structuredData?.base_data?.location,
estimatedYield: structuredData?.base_data?.estimatedYield
});
return structuredData;
}
catch (error) {
console.error('Error extracting property details:', error);
return {};
}
}
async expandAllSections(driver) {
try {
console.log('Starting enhanced section expansion with comprehensive selectors and stale element handling');
const maxRetries = 5;
let totalExpandedCount = 0;
const expandSelectors = [
'div[aria-expanded="false"]',
'div[aria-expanded="false"] h3',
'[aria-expanded="false"]',
'button[aria-expanded="false"]',
'details:not([open])',
'.accordion-header',
'.expandable-header',
'[data-toggle="collapse"]',
'.collapse-toggle',
'.section-toggle'
];
try {
console.log('🔍 Performing dynamic section title scanning...');
const allH3Elements = await driver.executeScript(`
const h3Elements = document.querySelectorAll('h3');
return Array.from(h3Elements).map(h3 => ({
text: h3.textContent.trim(),
hasExpandableParent: h3.closest('[aria-expanded]') !== null,
parentExpanded: h3.closest('[aria-expanded="true"]') !== null
}));
`);
console.log(`Found ${allH3Elements.length} h3 elements on page:`);
allH3Elements.forEach((h3, index) => {
console.log(` ${index + 1}. "${h3.text}" (expandable: ${h3.hasExpandableParent}, expanded: ${h3.parentExpanded})`);
});
} catch (scanError) {
console.log(`⚠️ Dynamic section scanning failed: ${scanError.message}`);
}
for (let mainRetry = 0; mainRetry < maxRetries; mainRetry++) {
console.log(`🔄 Main expansion attempt ${mainRetry + 1}/${maxRetries}`);
let expandedInThisRound = 0;
for (const selector of expandSelectors) {
try {
const elements = await driver.findElements(selenium_webdriver_1.By.css(selector));
console.log(`Found ${elements.length} elements with selector: ${selector}`);
for (let i = 0; i < elements.length; i++) {
try {
const currentElements = await driver.findElements(selenium_webdriver_1.By.css(selector));
if (i >= currentElements.length) continue;
const element = currentElements[i];
const isVisible = await element.isDisplayed();
const isEnabled = await element.isEnabled();
if (isVisible && isEnabled) {
const ariaExpanded = await element.getAttribute('aria-expanded');
const isOpen = await element.getAttribute('open');
if (ariaExpanded === 'false' || (selector.includes('details') && !isOpen)) {
await driver.executeScript("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});", element);
await driver.sleep(200);
try {
await element.click();
} catch (clickError) {
console.log(`Regular click failed, trying JavaScript click: ${clickError.message}`);
await driver.executeScript("arguments[0].click();", element);
}
await driver.sleep(500);
expandedInThisRound++;
totalExpandedCount++;
console.log(`✅ Expanded section using selector: ${selector}`);
}
}
}
catch (error) {
if (error.name === 'StaleElementReferenceError') {
console.log('⚠️ Stale element reference, retrying...');
break; // Break inner loop to re-find elements
}
console.log(`❌ Error expanding element: ${error.message}`);
}
}
}
catch (error) {
console.log(`❌ Error finding elements with selector ${selector}: ${error.message}`);
}
}
const xpathExpandSelectors = [
"//div[@aria-expanded='false'][.//h3[contains(text(),'Eigenschaften')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'Energiedaten')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'Notizen')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'ImmoMetrica-Statistiken')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'Finanzierung')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'Bewertung')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'Rendite')]]",
"//div[@aria-expanded='false'][.//h3[contains(text(),'Ausstattung')]]",
"//div[@aria-expanded='false']",
"//button[@aria-expanded='false']",
"//details[not(@open)]",
"//h3[contains(@class, 'accordion')]",
"//h3[contains(@class, 'collapsible')]",
"//div[contains(@class, 'section-header')]//button",
"//div[contains(@class, 'toggle')]//button"
];
for (const xpath of xpathExpandSelectors) {
try {
const elements = await driver.findElements(selenium_webdriver_1.By.xpath(xpath));
console.log(`Found ${elements.length} elements with XPath: ${xpath}`);
for (let i = 0; i < elements.length; i++) {
try {
const currentElements = await driver.findElements(selenium_webdriver_1.By.xpath(xpath));
if (i >= currentElements.length) continue;
const element = currentElements[i];
const isVisible = await element.isDisplayed();
const isEnabled = await element.isEnabled();
if (isVisible && isEnabled) {
const ariaExpanded = await element.getAttribute('aria-expanded');
const isOpen = await element.getAttribute('open');
if (ariaExpanded === 'false' || (xpath.includes('details') && !isOpen)) {
await driver.executeScript("arguments[0].scrollIntoView({behavior: 'smooth', block: 'center'});", element);
await driver.sleep(200);
try {
await element.click();
} catch (clickError) {
console.log(`Regular click failed, trying JavaScript click: ${clickError.message}`);
await driver.executeScript("arguments[0].click();", element);
}
await driver.sleep(500);
expandedInThisRound++;
totalExpandedCount++;
console.log(`✅ Expanded section using XPath: ${xpath}`);
}
}
}
catch (error) {
if (error.name === 'StaleElementReferenceError') {
console.log('⚠️ Stale element reference, retrying...');
break; // Break inner loop to re-find elements
}
console.log(`❌ Error expanding element with XPath: ${error.message}`);
}
}
}
catch (error) {
console.log(`❌ Error finding elements with XPath ${xpath}: ${error.message}`);
}
}
console.log(`🎯 Expanded ${expandedInThisRound} sections in round ${mainRetry + 1}`);
try {
const stillCollapsed = await driver.findElements(selenium_webdriver_1.By.css('[aria-expanded="false"]'));
console.log(`📊 DOM verification: ${stillCollapsed.length} sections still collapsed after round ${mainRetry + 1}`);
if (stillCollapsed.length > 0) {
console.log('🔍 Logging remaining collapsed sections for debugging:');
for (let i = 0; i < Math.min(stillCollapsed.length, 5); i++) {
try {
const outerHTML = await stillCollapsed[i].getAttribute('outerHTML');
const truncatedHTML = outerHTML.substring(0, 200) + (outerHTML.length > 200 ? '...' : '');
console.log(` ${i + 1}. ${truncatedHTML}`);
} catch (e) {
console.log(` ${i + 1}. Could not get HTML: ${e.message}`);
}
}
}
} catch (verificationError) {
console.log(`⚠️ DOM verification failed: ${verificationError.message}`);
}
if (expandedInThisRound === 0) {
console.log('✅ No more sections to expand, stopping retries');
break;
}
if (mainRetry < maxRetries - 1) {
const backoffDelay = 1500 + (mainRetry * 500);
console.log(`⏳ Waiting ${backoffDelay}ms before next retry (exponential backoff)...`);
await driver.sleep(backoffDelay);
}
}
for (let finalRetry = 0; finalRetry < 2; finalRetry++) {
try {
const retryElements = await driver.findElements(selenium_webdriver_1.By.css('[aria-expanded="false"]'));
if (retryElements.length > 0) {
console.log(`🔄 Final retry ${finalRetry + 1}: Found ${retryElements.length} still collapsed elements`);
for (let i = 0; i < retryElements.length; i++) {
try {
const currentRet