@animepaste/database
Version:
Local SQLite database used for Anime Paste CLI
624 lines (615 loc) • 16.4 kB
JavaScript
import { tradToSimple, simpleToTrad } from 'simptrad';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import * as Prisma from '@prisma/client';
import createDebug from 'debug';
import { isBefore, subMonths, subDays } from 'date-fns';
import HttpsProxyAgent from 'https-proxy-agent';
import { ofetch } from 'ofetch/node';
import { fetchResources } from 'animegarden';
const _AbstractDatabase = class {
constructor(option = {}) {
if (option.url) {
this.filepath = option.url;
this.prisma = new Prisma.PrismaClient({
datasources: { db: { url: "file:" + option.url } }
});
} else {
this.filepath = _AbstractDatabase.DefaultFilepath;
this.prisma = new Prisma.PrismaClient();
}
}
async ensure() {
if (!fs.existsSync(this.filepath)) {
await fs.promises.copyFile(
_AbstractDatabase.DefaultFilepath,
this.filepath
);
}
}
};
let AbstractDatabase = _AbstractDatabase;
AbstractDatabase.DefaultFilepath = path.join(
fileURLToPath(import.meta.url),
"../../prisma/anime.db"
);
class VideoStore extends AbstractDatabase {
constructor(option) {
super(option);
}
async createVideo(payload) {
return await this.prisma.video.create({
data: {
id: payload.videoId,
platform: payload.platform,
title: payload.title,
createdAt: payload.createdAt,
cover: payload.cover,
playUrls: JSON.stringify(payload.playUrl),
magnetId: payload.source.magnetId,
directory: payload.source.directory,
hash: payload.source.hash
}
});
}
async updateVideo(payload) {
return await this.prisma.video.update({
where: {
id_platform: {
id: payload.videoId,
platform: payload.platform
}
},
data: {
id: payload.videoId,
platform: payload.platform,
title: payload.title,
createdAt: payload.createdAt,
cover: payload.cover,
playUrls: JSON.stringify(payload.playUrl),
magnetId: payload.source.magnetId,
directory: payload.source.directory,
hash: payload.source.hash
}
});
}
async findVideo(platform, id) {
const resp = await this.prisma.video.findUnique({
where: {
id_platform: {
id,
platform
}
}
});
if (resp) {
return this.toVideoInfo(resp);
} else {
return void 0;
}
}
async deleteVideo(platform, id) {
await this.prisma.video.delete({
where: {
id_platform: {
id,
platform
}
}
});
}
async list() {
const data = await this.prisma.video.findMany();
return data.map(this.toVideoInfo);
}
toVideoInfo(resp) {
return {
videoId: resp.id,
platform: resp.platform,
title: resp.title,
createdAt: resp.createdAt.toISOString(),
cover: resp.cover ?? void 0,
playUrl: JSON.parse(resp.playUrls),
source: {
magnetId: resp.magnetId ?? void 0,
directory: resp.directory ?? void 0,
hash: resp.hash ?? void 0
}
};
}
}
function sleep(timeout = 1e3) {
return new Promise((res) => {
setTimeout(() => {
res();
}, timeout);
});
}
const P1080 = ["1080P", "1080p", "1920x1080", "1920X1080"];
const P720 = ["720P", "720p", "1280x720", "1280X720"];
const P2160 = ["3840x2160", "3840X2160"];
const HEVC = ["HEVC-10bit", "HEVC", "MKV"];
class MagnetParser {
constructor() {
this.TAGS = [
// Image Resolution
...P2160,
...P1080,
...P720,
"1920x816",
"x264",
"60fps",
// Web DL
"TVrip",
"WEB-DL",
"WEBrip",
"WebRip",
"WEB-RIP",
"BDRip",
"BD-RIP",
"DVDRIP",
"Baha",
"Bilibili",
"bilibili",
"ViuTV",
"B-Global",
// Video encode,
...HEVC,
"MP4",
"BIG5-MP4",
"BIG5_MP4",
"BIG5",
"GB-MP4",
"GB_MP4",
"GB-JP",
"GB_JP",
"GB",
"EAC3",
// Audio encode
"AVC 8bit",
"AVC",
"AAC",
"ASS",
"SRT",
// Language
"CHS",
"CHT",
"\u7B80\u7E41\u65E5\u5185\u5C01\u5B57\u5E55",
"\u7B80\u7E41\u65E5\u5185\u5C01",
"\u7B80\u7E41\u65E5\u53CC\u8BED",
"\u7B80\u65E5\u53CC\u8BED\u5B57\u5E55",
"\u7B80\u7E41\u5185\u5D4C\u5B57\u5E55",
"\u7B80\u7E41\u5185\u5D4C",
"\u7B80\u7E41\u5185\u5C01\u5B57\u5E55",
"\u7B80\u7E41\u5185\u5C01",
"\u7C21\u7E41\u5167\u5C01",
"\u7B80\u4F53\u5185\u5D4C",
"\u7E41\u4F53\u5185\u5D4C",
"\u7E41\u9AD4\u5167\u5D4C",
"\u7B80\u7E41\u5B57\u5E55",
"\u7B80\u7E41\u5916\u6302\u5B57\u5E55",
"\u7B80\u7E41\u5916\u6302",
"\u7B80\u7E41\u5185\u6302",
"\u7B80\u65E5\u53CC\u8BED",
"\u7B80\u65E5\u5B57\u5E55",
"\u7E41\u9AD4\u5916\u639B",
"\u7E41\u65E5\u96D9\u8A9E",
"\u7E41\u65E5\u5B57\u5E55",
"\u7C21\u7E41\u65E5\u5916\u639B",
"\u5185\u5C01\u5B57\u5E55",
"\u7B80\u4F53",
"\u7B80\u4E2D",
"\u7E41\u9AD4",
"\u7E41\u4E2D",
"\u82F1\u6587",
"\u5185\u5D4C",
"\u5185\u5C01"
];
this.REMOVE = [
/\[\d+\.\d+\.\d+\]/,
/\[[vV]\d+\]/,
// 招募
/\[[^\[]*(招募|急招|招人)[^\]]*\]/,
/（[^（]*(招募|急招|招人)[^）]*）/,
/【[^【]*(招募|急招|招人)[^】]*】/,
// Other
"Donghua",
/★0?1月新番★?/,
/★0?4月新番★?/,
/★0?7月新番★?/,
/★10月新番★?/,
/\[0?1月新番\]/,
/【0?1月新番】/,
/\[0?4月新番\]/,
/【0?4月新番】/,
/\[0?7月新番\]/,
/【0?7月新番】/,
/[10月新番]/,
/【10月新番】/,
"(\u5148\u884C\u7248\u672C)",
"(\u6B63\u5F0F\u7248\u672C)",
"\uFF08\u50C5\u9650\u6E2F\u6FB3\u53F0\u5730\u5340\uFF09",
/\((检索|檢索)[^\)]*\)/,
/（(检索|檢索)[^\)]*）/
];
}
parse(title) {
title = title.trim();
title = this.removePrefix(title);
const { title: newTitle1, ep } = this.extractEP(title);
const { title: newTitle2, tags } = this.extractTags(newTitle1);
title = this.removeBracket(newTitle2.trim());
const [newTitle3, ...alias] = this.extractAlias(title);
return { title: newTitle3, ep, tags, alias };
}
normalize(title) {
title = tradToSimple(title);
title = title.replace(/[“”‘’【】（）《》\s]/g, "");
for (const [src, dst] of [
["\uFF01", "!"],
["\xA5", "$"],
["\uFF0C", ","],
["\u3002", "."],
["\uFF1B", ";"],
["\uFF1A", ":"],
["\uFF1F", "?"],
["\uFF5E", "~"]
]) {
title = title.replace(src, dst);
}
return title;
}
normalizeMagnetTitle(originTitle) {
const { title, alias } = this.parse(originTitle);
const titles = [title, ...alias];
return JSON.stringify(titles.map(this.normalize));
}
hevc(magnet) {
return magnet.tags.some((t) => HEVC.includes(t));
}
quality(magnet) {
for (const tag of magnet.tags) {
if (P1080.includes(tag)) {
return 1080;
} else if (P720.includes(tag)) {
return 720;
}
}
return 1080;
}
language(magnet) {
for (const tag of magnet.tags) {
if (tag.indexOf("\u7B80") !== -1) {
return "zh-Hans";
} else if (tag.indexOf("\u7E41") !== -1) {
return "zh-Hant";
}
}
return "zh-Hans";
}
removeBracket(title) {
for (const RE of [/^\[[^\]]+\]$/, /【[^】]+】/]) {
if (RE.test(title)) {
return title.substring(1, title.length - 1).trim();
}
}
return title;
}
removePrefix(title) {
const RE1 = /^\[[^\]]+\]/;
const RE2 = /^【[^】]+】/;
const RE3 = /^\([^\)]+\)/;
for (const RE of [RE1, RE2, RE3]) {
if (RE.test(title)) {
return title.replace(RE, "").trim();
}
}
return title;
}
extractTags(title) {
const tags = [];
for (const tag of this.TAGS) {
const RE1 = new RegExp(`\\[${tag}\\]`);
const RE2 = new RegExp(`\u3010${tag}\u3011`);
const RE3 = new RegExp(`\\(${tag}\\)`);
const RE4 = new RegExp(tag);
for (const RE of [RE1, RE2, RE3, RE4]) {
if (RE.test(title)) {
title = title.replace(RE, "");
tags.push(tag);
}
}
}
for (const tag of this.REMOVE) {
title = title.replace(tag, "");
}
title = title.replace(/\[[\s_\-+&@]+\]/g, "").replace(/【[\s_\-+&@]+】/g, "").replace(/\([\s_\-+&@]+\)/g, "").trim();
return { title, tags };
}
extractEP(title) {
for (const RE of [
/\[(\d+)(_?[vV]\d+)?\]/,
/【(\d+)(_?[vV]\d+)?】/,
/- (\d+) /,
/- (\d+)$/,
/第(\d+)話/,
/第(\d+)话/,
/第(\d+)集/
]) {
const match = RE.exec(title);
if (match) {
title = title.replace(RE, "");
return { title, ep: +match[1] };
}
}
return { title, ep: void 0 };
}
extractAlias(title) {
return title.split("/").map((t) => t.trim()).filter(Boolean);
}
}
createDebug("anime:search");
async function fetchResource(page) {
const { resources } = await fetchResources(
(url) => ofetch.native(url, {
// @ts-ignore
agent: proxy() ? new HttpsProxyAgent(proxy()) : void 0
}),
{
page
}
);
return resources.map((r) => ({
type: r.type,
id: r.href.split("/").at(-1),
title: r.title,
fansub: r.fansub?.name ?? "",
magnet: r.magnet,
createdAt: new Date(r.createdAt)
}));
}
function proxy() {
const proxy2 = process.env.https_proxy ?? process.env.HTTPS_PROXY ?? process.env.http_proxy ?? process.env.HTTP_PROXY;
if (proxy2) {
const RE = /(\d+\.\d+\.\d+\.\d+):(\d+)/;
const match = RE.exec(proxy2);
if (match) {
return {
protocol: "http",
host: match[1],
port: +match[2]
};
} else {
return void 0;
}
} else {
return void 0;
}
}
const debug$1 = createDebug("anime:database");
class MagnetStore extends AbstractDatabase {
constructor(option = {}) {
super(option);
this.parser = new MagnetParser();
}
async index({
limit = subMonths(/* @__PURE__ */ new Date(), 6),
startPage,
endPage,
earlyStop = true,
listener
} = {}) {
debug$1(`Index to date: ${limit}`);
debug$1(`Index page from ${startPage} to ${endPage}`);
debug$1(`Early Stop ${earlyStop ? "enabled" : "disabled"}`);
let timestamp = void 0;
for (let page = startPage ?? 1; !endPage || page <= endPage; page++) {
const url = `https://share.dmhy.org/topics/list/page/${page}`;
listener && listener({ page, url, timestamp });
const payloads = await fetchResource(page);
let stop = false;
let inserted = 0;
for (const p of payloads) {
const createdAt = new Date(p.createdAt);
timestamp = createdAt;
if (isBefore(createdAt, limit)) {
stop = true;
break;
}
const ok = await this.createResource(p);
if (ok) {
inserted++;
}
}
listener && listener({ page, url, timestamp, ok: inserted });
if (stop || earlyStop && !inserted) {
break;
}
await sleep();
}
}
async search(keyword, option = {}) {
if (option.limit) {
const oldest = await this.timestamp();
if (isBefore(option.limit, oldest)) {
option.earlyStop = false;
option.limit = subDays(option.limit, 1);
await this.index(option);
}
}
const keywords = typeof keyword === "string" ? [keyword] : keyword;
const titleWhere = [
...keywords,
...keywords.map((k) => simpleToTrad(k))
].map((w) => ({
title: { contains: w }
}));
const keywordsWhere = keywords.map(this.parser.normalize).map((w) => ({ keywords: { contains: w } }));
debug$1(keywords.map(this.parser.normalize).join("\n"));
const result = await this.prisma.resource.findMany({
where: {
OR: [...keywordsWhere, ...titleWhere],
type: "\u52D5\u756B",
createdAt: {
gt: option.limit
}
},
include: {
Video: option.Video,
Episode: option.Episode
}
});
return result;
}
async createResource(payload) {
if (payload.type === "\u52D5\u756B") {
payload.keywords = this.parser.normalizeMagnetTitle(payload.title);
}
try {
const resp = await this.prisma.resource.create({ data: payload });
await this.timestamp(new Date(payload.createdAt));
return resp;
} catch (error) {
if (error instanceof Prisma.Prisma.PrismaClientKnownRequestError) {
if (error.code === "P2002") {
debug$1(`Found title: ${payload.title}`);
if (payload.keywords) {
try {
await this.prisma.resource.update({
where: {
id: payload.id
},
data: {
keywords: payload.keywords
}
});
} catch (error2) {
debug$1(error2);
}
}
} else {
debug$1(error);
}
} else {
debug$1(error);
}
}
}
async createResources(payloads) {
return (await Promise.all(payloads.map((p) => this.createResource(p)))).filter(Boolean);
}
async findById(id) {
return await this.prisma.resource.findUnique({
where: {
id
}
}) ?? void 0;
}
async timestamp(newValue) {
if (!this._timestamp) {
const t = await this.prisma.resource.aggregate({
_min: {
createdAt: true
}
});
this._timestamp = t._min.createdAt ?? /* @__PURE__ */ new Date();
debug$1("Init oldest timestamp: " + this._timestamp.toLocaleDateString());
}
if (newValue) {
return isBefore(newValue, this._timestamp) ? this._timestamp = newValue : this._timestamp;
} else {
return this._timestamp;
}
}
async list(option = {}) {
return await this.prisma.resource.findMany({
skip: option.skip,
take: option.take,
orderBy: {
createdAt: option.order ?? "desc"
}
});
}
async destroy() {
await this.prisma.$disconnect();
}
idToLink(magnetId) {
return `https://share.dmhy.org/topics/view/${magnetId}.html`;
}
}
const debug = createDebug("anime:database");
class EpisodeStore extends AbstractDatabase {
constructor() {
super(...arguments);
this.parser = new MagnetParser();
}
toEpisode(raw) {
const attrs = JSON.parse(raw.attrs);
return {
bgmId: String(raw.bgmId),
magnet: raw.magnet,
ep: raw.ep,
fansub: raw.fansub,
quality: this.parser.quality({ tags: attrs }),
language: this.parser.language({ tags: attrs })
};
}
async createEpisode(bgmId, magnet) {
const parsed = this.parser.parse(magnet.title);
debug(magnet.title + " => " + JSON.stringify(parsed, null, 2));
try {
return this.toEpisode(
await this.prisma.episode.create({
data: {
magnetId: magnet.id,
bgmId: +bgmId,
ep: parsed.ep,
fansub: magnet.fansub,
attrs: JSON.stringify(parsed.tags ?? [])
},
include: {
magnet: true
}
})
);
} catch (error) {
if (error instanceof Prisma.Prisma.PrismaClientKnownRequestError) {
if (error.code === "P2002") {
debug(`Found episode: ${magnet.title}`);
} else {
debug(error);
}
} else {
debug(error);
}
}
}
async findEpisode(magnetId) {
const raw = await this.prisma.episode.findUnique({
where: {
magnetId
},
include: {
magnet: true
}
});
if (raw) {
return this.toEpisode(raw);
} else {
return void 0;
}
}
async listEpisodes(bgmId) {
const eps = await this.prisma.episode.findMany({
where: {
bgmId: +bgmId
},
include: {
magnet: true
}
});
return eps.map((ep) => this.toEpisode(ep));
}
}
export { EpisodeStore, MagnetParser, MagnetStore, VideoStore, fetchResource, proxy };