cdi-mcp-server
Version:
CDI (Chemical Distribution Institute) MCP Server for retrieving and processing maritime inspection data with full CDI authentication support
306 lines • 13.8 kB
JavaScript
import winston from 'winston';
import { ensureDirectoryExists, generatePdfFilename } from './utils.js';
import axios from 'axios';
import * as cheerio from 'cheerio';
import * as dotenv from 'dotenv';
import { promises as fs } from 'fs';
import path from 'path';
import { URL } from 'url';
dotenv.config();
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(winston.format.simple(), winston.format.printf(({ message }) => `[tools] ${message}`)),
transports: [new winston.transports.Console()],
});
class CDIScraper {
httpClient;
constructor() {
this.httpClient = axios;
}
buildLoginUrl(docName, passedCredentials) {
const credentials = this.getCredentials(docName, passedCredentials);
return `https://www.cdim.org/psp/cdim.wp_postlogin?p_session_id=&p_mode=1&p_user=%7Enot+used%7E&p_URL=https%3A%2F%2Fwww.cdim.org%2Fpsp%2Fcdim.wp_home&p_userid=${credentials.username}&p_password=${credentials.password}`;
}
getCredentials(docName, passedCredentials) {
// Use passed credentials if available, otherwise fall back to environment variables
const username = passedCredentials?.username || process.env[`CDI_${docName}_USERNAME`];
const password = passedCredentials?.password || process.env[`CDI_${docName}_PASSWORD`];
if (!username || !password) {
throw new Error(`Missing credentials: provide username/password arguments or set CDI_${docName}_USERNAME and CDI_${docName}_PASSWORD environment variables`);
}
return { username, password };
}
getSessionId(html) {
const patterns = [
/p_session_id=([^&"']+)/,
/session_id\s*=\s*["']([^"']+)["']/,
/name="p_session_id"\s+value="([^"]+)"/
];
for (const pattern of patterns) {
const match = html.match(pattern);
if (match) {
return match[1];
}
}
throw new Error('Session ID not found in login response');
}
parseVesselTable(html) {
const $ = cheerio.load(html);
const table = $('table.cell_table');
if (!table.length) {
return [];
}
const rows = [];
table.find('tr').each((_, row) => {
const cells = [];
$(row).find('th, td').each((_, cell) => {
cells.push($(cell).text().trim());
});
if (cells.length > 5) {
rows.push(cells);
}
});
if (rows.length < 2) {
return [];
}
const headers = rows[0];
const data = rows.slice(1).map(row => row.slice(0, headers.length));
return data.map(row => {
const vessel = { IMO: '' };
headers.forEach((header, index) => {
vessel[header] = row[index] || '';
});
return vessel;
});
}
async fetchVesselList(sessionId, maxPages) {
const allVessels = [];
for (let page = 1; page <= maxPages; page++) {
const url = `https://www.cdim.org/psp/cdim.wp_all_ships?p_session_id=${sessionId}&p_page_no=${page}`;
logger.info(`Fetching vessel list page ${page}`);
try {
const response = await this.httpClient.get(url);
const vessels = this.parseVesselTable(response.data);
if (vessels.length === 0) {
logger.info(`No data on vessel list page ${page}, stopping`);
break;
}
allVessels.push(...vessels);
}
catch (error) {
logger.error(`Error fetching vessel list page ${page}: ${error}`);
break;
}
}
return allVessels;
}
async fetchInspections(sessionId, vessels, downloadDir) {
const inspections = [];
const seen = new Set();
await ensureDirectoryExists(downloadDir);
for (const vessel of vessels) {
const imo = vessel.IMO;
const vesselName = vessel['Ship Name'] || vessel['Vessel Name'] || '';
const detailUrl = `https://www.cdim.org/psp/cdim.wp_view_ship?p_session_id=${sessionId}&p_lrn_number=${imo}`;
logger.debug(`Fetching details for IMO ${imo}`);
try {
const response = await this.httpClient.get(detailUrl);
const $ = cheerio.load(response.data);
const table = $('table.cell_table');
if (!table.length) {
continue;
}
const headers = [];
table.find('th').each((_, th) => {
headers.push($(th).text().trim());
});
const tableRows = table.find('tr').slice(1);
for (let i = 0; i < tableRows.length; i++) {
const row = tableRows.eq(i);
const cols = [];
row.find('td').each((_, td) => {
cols.push($(td).text().trim());
});
if (cols.length === 0 || !cols.some(col => col.length > 0)) {
continue;
}
const info = {
IMO: imo,
Vessel_Name: vesselName
};
// Map columns to headers
cols.slice(0, headers.length).forEach((col, index) => {
if (headers[index]) {
info[headers[index]] = col;
}
});
const key = `${imo}-${info['Insp. Date']}-${info[headers[0]] || ''}`;
if (seen.has(key)) {
continue;
}
seen.add(key);
// Look for PDF link
const linkTag = row.find('a[href*="view"]');
if (linkTag.length > 0) {
const href = linkTag.attr('href');
if (href) {
logger.info(`Found PDF link for IMO ${imo}: ${href}`);
try {
const pdfData = await this.fetchPdfData(href, sessionId);
if (pdfData) {
info.pdf_path = pdfData.pdf_path;
info.pdf_link = pdfData.pdf_link;
}
}
catch (error) {
logger.error(`Error fetching PDF for IMO ${imo}: ${error}`);
info.pdf_path = null;
info.pdf_link = href;
}
}
}
else {
logger.debug(`No PDF link found for IMO ${imo}`);
info.pdf_path = null;
info.pdf_link = null;
}
inspections.push(info);
}
}
catch (error) {
logger.error(`Error fetching details for IMO ${imo}: ${error}`);
}
}
return inspections;
}
async fetchPdfData(href, sessionId) {
try {
const url = new URL(href, 'https://www.cdim.org');
const params = new URLSearchParams(url.search);
const pdfPageUrl = new URL('https://www.cdim.org/pls/apex/cdim.wp_summary');
pdfPageUrl.searchParams.set('p_session_id', params.get('p_session_id') || sessionId);
pdfPageUrl.searchParams.set('p_inspection_number', params.get('p_inspection_number') || '');
pdfPageUrl.searchParams.set('p_lrn_number', params.get('p_lrn_number') || '');
pdfPageUrl.searchParams.set('p_qset', params.get('p_qset') || '');
pdfPageUrl.searchParams.set('p_report_number', params.get('p_report_number') || '');
pdfPageUrl.searchParams.set('p_language', '10');
pdfPageUrl.searchParams.set('p_print_type', 'pre_pdf');
pdfPageUrl.searchParams.set('p_is_loading', '0');
logger.info(`Fetching PDF page: ${pdfPageUrl.toString()}`);
const response = await this.httpClient.get(pdfPageUrl.toString());
const $ = cheerio.load(response.data);
const pdfTag = $('a[href]:contains("View PDF file")');
if (pdfTag.length > 0) {
let pdfUrl = pdfTag.attr('href');
if (pdfUrl && !pdfUrl.startsWith('http')) {
pdfUrl = 'https://www.cdim.org' + pdfUrl;
}
if (pdfUrl) {
logger.info(`Found PDF URL: ${pdfUrl}`);
// Download the PDF
const pdfResponse = await this.httpClient.get(pdfUrl, { responseType: 'arraybuffer' });
const content = Buffer.from(pdfResponse.data);
// Generate filename
const imo = params.get('p_lrn_number') || 'unknown';
const vesselName = 'vessel'; // This would need to be passed from the calling function
const inspectionDate = new Date().toISOString().split('T')[0];
const filename = generatePdfFilename(vesselName, imo, inspectionDate);
const filepath = path.join('downloads', filename);
await ensureDirectoryExists(path.dirname(filepath));
await fs.writeFile(filepath, content);
logger.info(`Successfully downloaded PDF: ${filepath} (size: ${content.length} bytes)`);
// Verify file was written
const stats = await fs.stat(filepath);
if (stats.size > 0) {
logger.info(`PDF file verified on disk: ${filepath}`);
return {
pdf_path: path.resolve(filepath),
pdf_link: pdfUrl
};
}
else {
logger.error(`PDF file not found on disk or empty: ${filepath}`);
return null;
}
}
}
else {
logger.warn('PDF tag not found');
return null;
}
}
catch (error) {
logger.error(`PDF fetch error: ${error}`);
return null;
}
return null;
}
async scrapeData(docName, options = {}, credentials) {
const { downloadDir = 'downloads', maxPages = 5 } = options;
logger.info(`Starting scrape for ${docName}`);
try {
// Login
const loginUrl = this.buildLoginUrl(docName, credentials);
const loginResponse = await this.httpClient.get(loginUrl);
const sessionId = this.getSessionId(loginResponse.data);
logger.info(`Logged in, session=${sessionId}`);
// Fetch vessel list
const vessels = await this.fetchVesselList(sessionId, maxPages);
logger.info(`Found ${vessels.length} vessels`);
// Fetch inspections and PDFs
const inspections = await this.fetchInspections(sessionId, vessels, downloadDir);
const pdfPaths = inspections
.filter(insp => insp.pdf_path)
.map(insp => insp.pdf_path);
logger.info(`Completed: ${vessels.length} vessels, ${inspections.length} inspections, ${pdfPaths.length} PDFs`);
if (pdfPaths.length > 0) {
logger.info('PDF paths found:');
pdfPaths.forEach(pdfPath => logger.info(` - ${pdfPath}`));
}
else {
logger.warn('No PDF paths found in inspections');
}
const pdfCountInInspections = inspections.filter(insp => insp.pdf_path).length;
logger.info(`Inspections with PDF paths: ${pdfCountInInspections} out of ${inspections.length}`);
return {
vessel_table: vessels,
inspection_reports: inspections,
pdf_paths: pdfPaths
};
}
catch (error) {
logger.error(`Scraping error: ${error}`);
throw error;
}
}
}
export async function cdiDataMain(docName, credentials) {
const scraper = new CDIScraper();
try {
const results = await scraper.scrapeData(docName, undefined, credentials);
logger.info(`Results obtained for ${docName}`);
const combinedResults = [];
// Process each inspection report and combine with vessel data
for (const inspection of results.inspection_reports) {
const vesselInfo = results.vessel_table.find(v => v.IMO === inspection.IMO) || {};
const combinedResult = {
...vesselInfo,
...inspection
};
combinedResults.push(combinedResult);
// Log if this inspection has a PDF
if (inspection.pdf_path) {
logger.info(`Inspection for IMO ${inspection.IMO} has PDF: ${inspection.pdf_path}`);
}
}
// Log summary
const inspectionsWithPdfs = combinedResults.filter(r => r.pdf_path).length;
logger.info(`Combined results: ${combinedResults.length} total, ${inspectionsWithPdfs} with PDFs`);
return combinedResults;
}
catch (error) {
logger.error(`Error during scraping: ${error}`);
throw error;
}
}
//# sourceMappingURL=tools.js.map