@tomisakae/syosetu-api
Version:
Enterprise-grade Fastify TypeScript API for Syosetu.com data extraction using official API and web scraping. Run instantly with 'npx @tomisakae/syosetu-api'
159 lines • 6.65 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.scraperService = exports.ScraperService = void 0;
const axios_1 = __importDefault(require("axios"));
const cheerio = __importStar(require("cheerio"));
const config_1 = require("@/config");
const utils_1 = require("@/utils");
const logger = (0, utils_1.createChildLogger)('ScraperService');
function extractCleanText($, selector) {
const element = $(selector).first();
if (element.length === 0)
return '';
element.find('script, style, .ad, .advertisement, .banner').remove();
element.find('br').replaceWith('\n');
let rawText = element.text() || '';
rawText = rawText
.replace(/\r\n/g, '\n')
.replace(/\r/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.replace(/[ \t]{2,}/g, ' ')
.replace(/\n[ \t]*\n/g, '\n\n')
.replace(/^[ \t]+|[ \t]+$/gm, '')
.replace(/^\n+|\n+$/g, '')
.trim();
return rawText;
}
class ScraperService {
ncodeBaseUrl;
constructor() {
this.ncodeBaseUrl = config_1.syosetuConfig.ncodeBaseUrl;
}
async getChapterContent(ncode, chapterNumber) {
try {
await utils_1.globalRateLimiter.waitForRateLimit();
logger.info(`Fetching content for ncode: ${ncode}, chapter: ${chapterNumber} using Cheerio`);
const chapterUrl = `${this.ncodeBaseUrl}/${ncode}/${chapterNumber}/`;
const response = await axios_1.default.get(chapterUrl, {
timeout: config_1.syosetuConfig.requestTimeout,
headers: {
'User-Agent': config_1.syosetuConfig.userAgent,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'ja,en-US;q=0.7,en;q=0.3',
'Accept-Encoding': 'gzip, deflate, br',
DNT: '1',
Connection: 'keep-alive',
'Upgrade-Insecure-Requests': '1',
},
});
const $ = cheerio.load(response.data);
const title = extractCleanText($, '.novel_subtitle, .p-novel__title');
const textContent = extractCleanText($, '#novel_honbun, .p-novel__body');
const date = extractCleanText($, '.novel_subdate, .p-novel__date');
const characterCount = textContent.length;
const estimatedReadingTime = Math.ceil(characterCount / 500);
if (!textContent || textContent.length < 10) {
return {
success: false,
error: 'Không thể lấy nội dung chapter hoặc nội dung quá ngắn',
timestamp: new Date().toISOString(),
};
}
const result = {
success: true,
data: {
ncode,
chapterNumber,
title: title || `Chapter ${chapterNumber}`,
htmlContent: '',
textContent,
date: date || new Date().toISOString().split('T')[0] || '',
characterCount,
estimatedReadingTime,
url: chapterUrl,
},
timestamp: new Date().toISOString(),
};
logger.info(`Content fetched successfully for ncode: ${ncode}, chapter: ${chapterNumber} using Cheerio`);
return result;
}
catch (error) {
logger.error(`Error fetching content for ncode ${ncode}, chapter ${chapterNumber}:`, error);
if (axios_1.default.isAxiosError(error)) {
if (error.response?.status === 404) {
return {
success: false,
error: 'Chapter không tồn tại',
timestamp: new Date().toISOString(),
};
}
if (error.response?.status === 403) {
return {
success: false,
error: 'Bị chặn truy cập, vui lòng thử lại sau',
timestamp: new Date().toISOString(),
};
}
}
return {
success: false,
error: `Lỗi khi lấy nội dung chapter: ${error instanceof Error ? error.message : 'Unknown error'}`,
timestamp: new Date().toISOString(),
};
}
}
async isAvailable() {
try {
const response = await axios_1.default.get(this.ncodeBaseUrl, {
timeout: 5000,
headers: {
'User-Agent': config_1.syosetuConfig.userAgent,
},
});
return response.status === 200;
}
catch (error) {
logger.warn('Syosetu website not available:', error);
return false;
}
}
}
exports.ScraperService = ScraperService;
exports.scraperService = new ScraperService();
//# sourceMappingURL=scraper.service.js.map