@animepaste/database
Version:
Local SQLite database used for Anime Paste CLI
651 lines (638 loc) • 17.6 kB
JavaScript
;
const simptrad = require('simptrad');
const fs = require('node:fs');
const path = require('node:path');
const node_url = require('node:url');
const Prisma = require('@prisma/client');
const createDebug = require('debug');
const dateFns = require('date-fns');
const HttpsProxyAgent = require('https-proxy-agent');
const node = require('ofetch/node');
const animegarden = require('animegarden');
function _interopDefaultCompat (e) { return e && typeof e === 'object' && 'default' in e ? e.default : e; }
function _interopNamespaceCompat(e) {
if (e && typeof e === 'object' && 'default' in e) return e;
const n = Object.create(null);
if (e) {
for (const k in e) {
n[k] = e[k];
}
}
n.default = e;
return n;
}
const fs__default = /*#__PURE__*/_interopDefaultCompat(fs);
const path__default = /*#__PURE__*/_interopDefaultCompat(path);
const Prisma__namespace = /*#__PURE__*/_interopNamespaceCompat(Prisma);
const createDebug__default = /*#__PURE__*/_interopDefaultCompat(createDebug);
const HttpsProxyAgent__default = /*#__PURE__*/_interopDefaultCompat(HttpsProxyAgent);
const _AbstractDatabase = class {
constructor(option = {}) {
if (option.url) {
this.filepath = option.url;
this.prisma = new Prisma__namespace.PrismaClient({
datasources: { db: { url: "file:" + option.url } }
});
} else {
this.filepath = _AbstractDatabase.DefaultFilepath;
this.prisma = new Prisma__namespace.PrismaClient();
}
}
async ensure() {
if (!fs__default.existsSync(this.filepath)) {
await fs__default.promises.copyFile(
_AbstractDatabase.DefaultFilepath,
this.filepath
);
}
}
};
let AbstractDatabase = _AbstractDatabase;
AbstractDatabase.DefaultFilepath = path__default.join(
node_url.fileURLToPath((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (document.currentScript && document.currentScript.src || new URL('index.cjs', document.baseURI).href))),
"../../prisma/anime.db"
);
class VideoStore extends AbstractDatabase {
constructor(option) {
super(option);
}
async createVideo(payload) {
return await this.prisma.video.create({
data: {
id: payload.videoId,
platform: payload.platform,
title: payload.title,
createdAt: payload.createdAt,
cover: payload.cover,
playUrls: JSON.stringify(payload.playUrl),
magnetId: payload.source.magnetId,
directory: payload.source.directory,
hash: payload.source.hash
}
});
}
async updateVideo(payload) {
return await this.prisma.video.update({
where: {
id_platform: {
id: payload.videoId,
platform: payload.platform
}
},
data: {
id: payload.videoId,
platform: payload.platform,
title: payload.title,
createdAt: payload.createdAt,
cover: payload.cover,
playUrls: JSON.stringify(payload.playUrl),
magnetId: payload.source.magnetId,
directory: payload.source.directory,
hash: payload.source.hash
}
});
}
async findVideo(platform, id) {
const resp = await this.prisma.video.findUnique({
where: {
id_platform: {
id,
platform
}
}
});
if (resp) {
return this.toVideoInfo(resp);
} else {
return void 0;
}
}
async deleteVideo(platform, id) {
await this.prisma.video.delete({
where: {
id_platform: {
id,
platform
}
}
});
}
async list() {
const data = await this.prisma.video.findMany();
return data.map(this.toVideoInfo);
}
toVideoInfo(resp) {
return {
videoId: resp.id,
platform: resp.platform,
title: resp.title,
createdAt: resp.createdAt.toISOString(),
cover: resp.cover ?? void 0,
playUrl: JSON.parse(resp.playUrls),
source: {
magnetId: resp.magnetId ?? void 0,
directory: resp.directory ?? void 0,
hash: resp.hash ?? void 0
}
};
}
}
function sleep(timeout = 1e3) {
return new Promise((res) => {
setTimeout(() => {
res();
}, timeout);
});
}
const P1080 = ["1080P", "1080p", "1920x1080", "1920X1080"];
const P720 = ["720P", "720p", "1280x720", "1280X720"];
const P2160 = ["3840x2160", "3840X2160"];
const HEVC = ["HEVC-10bit", "HEVC", "MKV"];
class MagnetParser {
constructor() {
this.TAGS = [
// Image Resolution
...P2160,
...P1080,
...P720,
"1920x816",
"x264",
"60fps",
// Web DL
"TVrip",
"WEB-DL",
"WEBrip",
"WebRip",
"WEB-RIP",
"BDRip",
"BD-RIP",
"DVDRIP",
"Baha",
"Bilibili",
"bilibili",
"ViuTV",
"B-Global",
// Video encode,
...HEVC,
"MP4",
"BIG5-MP4",
"BIG5_MP4",
"BIG5",
"GB-MP4",
"GB_MP4",
"GB-JP",
"GB_JP",
"GB",
"EAC3",
// Audio encode
"AVC 8bit",
"AVC",
"AAC",
"ASS",
"SRT",
// Language
"CHS",
"CHT",
"\u7B80\u7E41\u65E5\u5185\u5C01\u5B57\u5E55",
"\u7B80\u7E41\u65E5\u5185\u5C01",
"\u7B80\u7E41\u65E5\u53CC\u8BED",
"\u7B80\u65E5\u53CC\u8BED\u5B57\u5E55",
"\u7B80\u7E41\u5185\u5D4C\u5B57\u5E55",
"\u7B80\u7E41\u5185\u5D4C",
"\u7B80\u7E41\u5185\u5C01\u5B57\u5E55",
"\u7B80\u7E41\u5185\u5C01",
"\u7C21\u7E41\u5167\u5C01",
"\u7B80\u4F53\u5185\u5D4C",
"\u7E41\u4F53\u5185\u5D4C",
"\u7E41\u9AD4\u5167\u5D4C",
"\u7B80\u7E41\u5B57\u5E55",
"\u7B80\u7E41\u5916\u6302\u5B57\u5E55",
"\u7B80\u7E41\u5916\u6302",
"\u7B80\u7E41\u5185\u6302",
"\u7B80\u65E5\u53CC\u8BED",
"\u7B80\u65E5\u5B57\u5E55",
"\u7E41\u9AD4\u5916\u639B",
"\u7E41\u65E5\u96D9\u8A9E",
"\u7E41\u65E5\u5B57\u5E55",
"\u7C21\u7E41\u65E5\u5916\u639B",
"\u5185\u5C01\u5B57\u5E55",
"\u7B80\u4F53",
"\u7B80\u4E2D",
"\u7E41\u9AD4",
"\u7E41\u4E2D",
"\u82F1\u6587",
"\u5185\u5D4C",
"\u5185\u5C01"
];
this.REMOVE = [
/\[\d+\.\d+\.\d+\]/,
/\[[vV]\d+\]/,
// 招募
/\[[^\[]*(招募|急招|招人)[^\]]*\]/,
/（[^（]*(招募|急招|招人)[^）]*）/,
/【[^【]*(招募|急招|招人)[^】]*】/,
// Other
"Donghua",
/★0?1月新番★?/,
/★0?4月新番★?/,
/★0?7月新番★?/,
/★10月新番★?/,
/\[0?1月新番\]/,
/【0?1月新番】/,
/\[0?4月新番\]/,
/【0?4月新番】/,
/\[0?7月新番\]/,
/【0?7月新番】/,
/[10月新番]/,
/【10月新番】/,
"(\u5148\u884C\u7248\u672C)",
"(\u6B63\u5F0F\u7248\u672C)",
"\uFF08\u50C5\u9650\u6E2F\u6FB3\u53F0\u5730\u5340\uFF09",
/\((检索|檢索)[^\)]*\)/,
/（(检索|檢索)[^\)]*）/
];
}
parse(title) {
title = title.trim();
title = this.removePrefix(title);
const { title: newTitle1, ep } = this.extractEP(title);
const { title: newTitle2, tags } = this.extractTags(newTitle1);
title = this.removeBracket(newTitle2.trim());
const [newTitle3, ...alias] = this.extractAlias(title);
return { title: newTitle3, ep, tags, alias };
}
normalize(title) {
title = simptrad.tradToSimple(title);
title = title.replace(/[“”‘’【】（）《》\s]/g, "");
for (const [src, dst] of [
["\uFF01", "!"],
["\xA5", "$"],
["\uFF0C", ","],
["\u3002", "."],
["\uFF1B", ";"],
["\uFF1A", ":"],
["\uFF1F", "?"],
["\uFF5E", "~"]
]) {
title = title.replace(src, dst);
}
return title;
}
normalizeMagnetTitle(originTitle) {
const { title, alias } = this.parse(originTitle);
const titles = [title, ...alias];
return JSON.stringify(titles.map(this.normalize));
}
hevc(magnet) {
return magnet.tags.some((t) => HEVC.includes(t));
}
quality(magnet) {
for (const tag of magnet.tags) {
if (P1080.includes(tag)) {
return 1080;
} else if (P720.includes(tag)) {
return 720;
}
}
return 1080;
}
language(magnet) {
for (const tag of magnet.tags) {
if (tag.indexOf("\u7B80") !== -1) {
return "zh-Hans";
} else if (tag.indexOf("\u7E41") !== -1) {
return "zh-Hant";
}
}
return "zh-Hans";
}
removeBracket(title) {
for (const RE of [/^\[[^\]]+\]$/, /【[^】]+】/]) {
if (RE.test(title)) {
return title.substring(1, title.length - 1).trim();
}
}
return title;
}
removePrefix(title) {
const RE1 = /^\[[^\]]+\]/;
const RE2 = /^【[^】]+】/;
const RE3 = /^\([^\)]+\)/;
for (const RE of [RE1, RE2, RE3]) {
if (RE.test(title)) {
return title.replace(RE, "").trim();
}
}
return title;
}
extractTags(title) {
const tags = [];
for (const tag of this.TAGS) {
const RE1 = new RegExp(`\\[${tag}\\]`);
const RE2 = new RegExp(`\u3010${tag}\u3011`);
const RE3 = new RegExp(`\\(${tag}\\)`);
const RE4 = new RegExp(tag);
for (const RE of [RE1, RE2, RE3, RE4]) {
if (RE.test(title)) {
title = title.replace(RE, "");
tags.push(tag);
}
}
}
for (const tag of this.REMOVE) {
title = title.replace(tag, "");
}
title = title.replace(/\[[\s_\-+&@]+\]/g, "").replace(/【[\s_\-+&@]+】/g, "").replace(/\([\s_\-+&@]+\)/g, "").trim();
return { title, tags };
}
extractEP(title) {
for (const RE of [
/\[(\d+)(_?[vV]\d+)?\]/,
/【(\d+)(_?[vV]\d+)?】/,
/- (\d+) /,
/- (\d+)$/,
/第(\d+)話/,
/第(\d+)话/,
/第(\d+)集/
]) {
const match = RE.exec(title);
if (match) {
title = title.replace(RE, "");
return { title, ep: +match[1] };
}
}
return { title, ep: void 0 };
}
extractAlias(title) {
return title.split("/").map((t) => t.trim()).filter(Boolean);
}
}
createDebug__default("anime:search");
async function fetchResource(page) {
const { resources } = await animegarden.fetchResources(
(url) => node.ofetch.native(url, {
// @ts-ignore
agent: proxy() ? new HttpsProxyAgent__default(proxy()) : void 0
}),
{
page
}
);
return resources.map((r) => ({
type: r.type,
id: r.href.split("/").at(-1),
title: r.title,
fansub: r.fansub?.name ?? "",
magnet: r.magnet,
createdAt: new Date(r.createdAt)
}));
}
function proxy() {
const proxy2 = process.env.https_proxy ?? process.env.HTTPS_PROXY ?? process.env.http_proxy ?? process.env.HTTP_PROXY;
if (proxy2) {
const RE = /(\d+\.\d+\.\d+\.\d+):(\d+)/;
const match = RE.exec(proxy2);
if (match) {
return {
protocol: "http",
host: match[1],
port: +match[2]
};
} else {
return void 0;
}
} else {
return void 0;
}
}
const debug$1 = createDebug__default("anime:database");
class MagnetStore extends AbstractDatabase {
constructor(option = {}) {
super(option);
this.parser = new MagnetParser();
}
async index({
limit = dateFns.subMonths(/* @__PURE__ */ new Date(), 6),
startPage,
endPage,
earlyStop = true,
listener
} = {}) {
debug$1(`Index to date: ${limit}`);
debug$1(`Index page from ${startPage} to ${endPage}`);
debug$1(`Early Stop ${earlyStop ? "enabled" : "disabled"}`);
let timestamp = void 0;
for (let page = startPage ?? 1; !endPage || page <= endPage; page++) {
const url = `https://share.dmhy.org/topics/list/page/${page}`;
listener && listener({ page, url, timestamp });
const payloads = await fetchResource(page);
let stop = false;
let inserted = 0;
for (const p of payloads) {
const createdAt = new Date(p.createdAt);
timestamp = createdAt;
if (dateFns.isBefore(createdAt, limit)) {
stop = true;
break;
}
const ok = await this.createResource(p);
if (ok) {
inserted++;
}
}
listener && listener({ page, url, timestamp, ok: inserted });
if (stop || earlyStop && !inserted) {
break;
}
await sleep();
}
}
async search(keyword, option = {}) {
if (option.limit) {
const oldest = await this.timestamp();
if (dateFns.isBefore(option.limit, oldest)) {
option.earlyStop = false;
option.limit = dateFns.subDays(option.limit, 1);
await this.index(option);
}
}
const keywords = typeof keyword === "string" ? [keyword] : keyword;
const titleWhere = [
...keywords,
...keywords.map((k) => simptrad.simpleToTrad(k))
].map((w) => ({
title: { contains: w }
}));
const keywordsWhere = keywords.map(this.parser.normalize).map((w) => ({ keywords: { contains: w } }));
debug$1(keywords.map(this.parser.normalize).join("\n"));
const result = await this.prisma.resource.findMany({
where: {
OR: [...keywordsWhere, ...titleWhere],
type: "\u52D5\u756B",
createdAt: {
gt: option.limit
}
},
include: {
Video: option.Video,
Episode: option.Episode
}
});
return result;
}
async createResource(payload) {
if (payload.type === "\u52D5\u756B") {
payload.keywords = this.parser.normalizeMagnetTitle(payload.title);
}
try {
const resp = await this.prisma.resource.create({ data: payload });
await this.timestamp(new Date(payload.createdAt));
return resp;
} catch (error) {
if (error instanceof Prisma__namespace.Prisma.PrismaClientKnownRequestError) {
if (error.code === "P2002") {
debug$1(`Found title: ${payload.title}`);
if (payload.keywords) {
try {
await this.prisma.resource.update({
where: {
id: payload.id
},
data: {
keywords: payload.keywords
}
});
} catch (error2) {
debug$1(error2);
}
}
} else {
debug$1(error);
}
} else {
debug$1(error);
}
}
}
async createResources(payloads) {
return (await Promise.all(payloads.map((p) => this.createResource(p)))).filter(Boolean);
}
async findById(id) {
return await this.prisma.resource.findUnique({
where: {
id
}
}) ?? void 0;
}
async timestamp(newValue) {
if (!this._timestamp) {
const t = await this.prisma.resource.aggregate({
_min: {
createdAt: true
}
});
this._timestamp = t._min.createdAt ?? /* @__PURE__ */ new Date();
debug$1("Init oldest timestamp: " + this._timestamp.toLocaleDateString());
}
if (newValue) {
return dateFns.isBefore(newValue, this._timestamp) ? this._timestamp = newValue : this._timestamp;
} else {
return this._timestamp;
}
}
async list(option = {}) {
return await this.prisma.resource.findMany({
skip: option.skip,
take: option.take,
orderBy: {
createdAt: option.order ?? "desc"
}
});
}
async destroy() {
await this.prisma.$disconnect();
}
idToLink(magnetId) {
return `https://share.dmhy.org/topics/view/${magnetId}.html`;
}
}
const debug = createDebug__default("anime:database");
class EpisodeStore extends AbstractDatabase {
constructor() {
super(...arguments);
this.parser = new MagnetParser();
}
toEpisode(raw) {
const attrs = JSON.parse(raw.attrs);
return {
bgmId: String(raw.bgmId),
magnet: raw.magnet,
ep: raw.ep,
fansub: raw.fansub,
quality: this.parser.quality({ tags: attrs }),
language: this.parser.language({ tags: attrs })
};
}
async createEpisode(bgmId, magnet) {
const parsed = this.parser.parse(magnet.title);
debug(magnet.title + " => " + JSON.stringify(parsed, null, 2));
try {
return this.toEpisode(
await this.prisma.episode.create({
data: {
magnetId: magnet.id,
bgmId: +bgmId,
ep: parsed.ep,
fansub: magnet.fansub,
attrs: JSON.stringify(parsed.tags ?? [])
},
include: {
magnet: true
}
})
);
} catch (error) {
if (error instanceof Prisma__namespace.Prisma.PrismaClientKnownRequestError) {
if (error.code === "P2002") {
debug(`Found episode: ${magnet.title}`);
} else {
debug(error);
}
} else {
debug(error);
}
}
}
async findEpisode(magnetId) {
const raw = await this.prisma.episode.findUnique({
where: {
magnetId
},
include: {
magnet: true
}
});
if (raw) {
return this.toEpisode(raw);
} else {
return void 0;
}
}
async listEpisodes(bgmId) {
const eps = await this.prisma.episode.findMany({
where: {
bgmId: +bgmId
},
include: {
magnet: true
}
});
return eps.map((ep) => this.toEpisode(ep));
}
}
exports.EpisodeStore = EpisodeStore;
exports.MagnetParser = MagnetParser;
exports.MagnetStore = MagnetStore;
exports.VideoStore = VideoStore;
exports.fetchResource = fetchResource;
exports.proxy = proxy;