enka-network-api
Version:
Enka-network API wrapper for Genshin Impact.
498 lines (497 loc) • 25 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.CachedAssetsManager = void 0;
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const axios_1 = __importDefault(require("axios"));
const unzip_stream_1 = __importDefault(require("unzip-stream"));
const config_file_js_1 = require("config_file.js");
const fetch_utils_1 = require("../utils/fetch_utils");
const ObjectKeysManager_1 = require("./ObjectKeysManager");
const cache_utils_1 = require("../utils/cache_utils");
const ExcelTransformer_1 = require("./ExcelTransformer");
const initialExcelDataMemory = Object.fromEntries(ExcelTransformer_1.excels.map(content => [content, null]));
let excelDataMemory = Object.assign({}, initialExcelDataMemory);
const languages = ["chs", "cht", "de", "en", "es", "fr", "id", "jp", "kr", "pt", "ru", "th", "vi"];
const initialLangDataMemory = Object.fromEntries(languages.map(lang => [lang, null]));
let langDataMemory = Object.assign({}, initialLangDataMemory);
let objectKeysManager;
const textMapWhiteList = [
2329553598, // Aether
3241049361, // Lumine
];
const manualTextMapWhiteList = [
"EquipType",
"EQUIP_BRACER",
"EQUIP_DRESS",
"EQUIP_SHOES",
"EQUIP_RING",
"EQUIP_NECKLACE",
"ElementType",
"None",
"Fire",
"Water",
"Grass",
"Electric",
"Wind",
"Ice",
"Rock",
"WeaponType",
];
const getGitRemoteAPIUrl = (useRawGenshinData, rawDate, date) => useRawGenshinData
? `https://gitlab.com/api/v4/projects/53216109/repository/commits?since=${rawDate.toISOString()}`
: `https://api.github.com/repos/yuko1101/enka-network-api/commits?sha=main&path=cache.zip&since=${date.toISOString()}`;
class CachedAssetsManager {
constructor(enka) {
var _a;
this.enka = enka;
this.defaultCacheDirectoryPath = path_1.default.resolve(__dirname, "..", "..", "cache");
this.cacheDirectoryPath = (_a = enka.options.cacheDirectory) !== null && _a !== void 0 ? _a : this.defaultCacheDirectoryPath;
this.gameDataBaseUrl = enka.options.gameDataBaseUrl;
this._cacheUpdater = null;
this._githubCache = null;
this._isFetching = false;
}
/** Create the necessary folders and files, and if the directory [cacheDirectoryPath](#cacheDirectoryPath) did not exist, move the cache files from the default path. */
async cacheDirectorySetup() {
if (!fs_1.default.existsSync(this.cacheDirectoryPath)) {
fs_1.default.mkdirSync(this.cacheDirectoryPath);
if (fs_1.default.existsSync(this.defaultCacheDirectoryPath) && fs_1.default.readdirSync(this.defaultCacheDirectoryPath).length > 0) {
try {
(0, config_file_js_1.move)(this.defaultCacheDirectoryPath, this.cacheDirectoryPath);
}
catch (e) {
console.error(`Auto-Moving cache data failed with error: ${e}`);
}
}
}
if (!fs_1.default.existsSync(path_1.default.resolve(this.cacheDirectoryPath, "data"))) {
fs_1.default.mkdirSync(path_1.default.resolve(this.cacheDirectoryPath, "data"));
}
if (!fs_1.default.existsSync(path_1.default.resolve(this.cacheDirectoryPath, "langs"))) {
fs_1.default.mkdirSync(path_1.default.resolve(this.cacheDirectoryPath, "langs"));
}
if (!fs_1.default.existsSync(path_1.default.resolve(this.cacheDirectoryPath, "github"))) {
fs_1.default.mkdirSync(path_1.default.resolve(this.cacheDirectoryPath, "github"));
}
const githubCachePath = path_1.default.resolve(this.cacheDirectoryPath, "github", "genshin_data.json");
if (!fs_1.default.existsSync(githubCachePath) || !this._githubCache) {
this._githubCache = await new config_file_js_1.ConfigFile(githubCachePath, config_file_js_1.defaultJsonOptions, {
"lastUpdate": 0,
"rawLastUpdate": 0,
}).load();
}
}
/** Obtains a text map for a specific language. */
async fetchLanguageData(lang) {
await this.cacheDirectorySetup();
const enka = this.enka;
// TODO: better handling for languages with splitted files
if (lang === "th") {
const json1 = JSON.parse(await (0, fetch_utils_1.fetchString)({ url: `${this.gameDataBaseUrl}/TextMap/TextMap${lang.toUpperCase()}_0.json`, enka, allowLocalFile: true }));
const json2 = JSON.parse(await (0, fetch_utils_1.fetchString)({ url: `${this.gameDataBaseUrl}/TextMap/TextMap${lang.toUpperCase()}_1.json`, enka, allowLocalFile: true }));
return Object.assign(Object.assign({}, json1), json2);
}
const url = `${this.gameDataBaseUrl}/TextMap/TextMap${lang.toUpperCase()}.json`;
const json = JSON.parse(await (0, fetch_utils_1.fetchString)({ url, enka, allowLocalFile: true }));
return json;
}
/**
* @param useRawGenshinData Whether to fetch from gitlab repo ({@link https://gitlab.com/Dimbreath/AnimeGameData}) instead of downloading cache.zip
* @returns Whether the game data update is available or not.
*/
async checkForUpdates(useRawGenshinData = false) {
var _a, _b, _c, _d;
await this.cacheDirectorySetup();
const url = getGitRemoteAPIUrl(useRawGenshinData, new Date((_b = (_a = this._githubCache) === null || _a === void 0 ? void 0 : _a.getValue("rawLastUpdate")) !== null && _b !== void 0 ? _b : 0), new Date((_d = (_c = this._githubCache) === null || _c === void 0 ? void 0 : _c.getValue("lastUpdate")) !== null && _d !== void 0 ? _d : 0));
const newCommits = JSON.parse(await (0, fetch_utils_1.fetchString)({ url, enka: this.enka }));
return newCommits.length !== 0;
}
/**
* @param options.useRawGenshinData Whether to fetch from gitlab repo ({@link https://gitlab.com/Dimbreath/AnimeGameData}) instead of downloading cache.zip
* @param options.ghproxy Whether to use ghproxy.com
*/
async fetchAllContents(options = {}) {
var _a, _b;
if (this._isFetching) {
throw new Error("You are already fetching assets.");
}
const mergedOptions = (0, config_file_js_1.bindOptions)({
useRawGenshinData: false,
}, options);
await this.cacheDirectorySetup();
this._isFetching = true;
if (!mergedOptions.useRawGenshinData) {
if (this.enka.options.showFetchCacheLog) {
console.info("Downloading cache.zip...");
}
await this._downloadCacheZip();
await ((_a = this._githubCache) === null || _a === void 0 ? void 0 : _a.set("lastUpdate", Date.now()).save());
if (this.enka.options.showFetchCacheLog) {
console.info("Download completed");
}
}
else {
if (this.enka.options.showFetchCacheLog) {
console.info("Downloading structure data files...");
}
const promises = [];
// TODO: use ExcelDataMap (but not readonly) type instead.
const excelOutputData = Object.assign({}, initialExcelDataMemory);
for (const excel of ExcelTransformer_1.excels) {
const fileName = `${excel}.json`;
const url = `${this.gameDataBaseUrl}/ExcelBinOutput/${fileName}`;
promises.push((async () => {
const json = JSON.parse(await (0, fetch_utils_1.fetchString)({ url, enka: this.enka, allowLocalFile: true }));
if (this.enka.options.showFetchCacheLog) {
console.info(`Downloaded data/${fileName}`);
}
excelOutputData[excel] = this.formatExcel(excel, json);
})());
}
await Promise.all(promises);
if (this.enka.options.showFetchCacheLog) {
console.info("> Downloaded all structure data files");
console.info("Downloading language files...");
}
const langsData = Object.assign({}, initialLangDataMemory);
const langPromises = [];
for (const lang of languages) {
langPromises.push((async () => {
const data = await this.fetchLanguageData(lang);
if (this.enka.options.showFetchCacheLog) {
console.info(`Downloaded langs/${lang}.json`);
}
langsData[lang] = data;
})());
}
await Promise.all(langPromises);
if (this.enka.options.showFetchCacheLog) {
console.info("> Downloaded all language files");
console.info("Parsing data...");
}
const clearLangsData = this.removeUnusedTextData(excelOutputData, langsData);
if (this.enka.options.showFetchCacheLog) {
console.info("> Parsing completed");
console.info("Saving into files...");
}
for (const lang of Object.keys(clearLangsData)) {
fs_1.default.writeFileSync(this.getLanguageDataPath(lang), JSON.stringify(clearLangsData[lang]));
}
for (const key in excelOutputData) {
fs_1.default.writeFileSync(this.getJSONDataPath(key), JSON.stringify(excelOutputData[key]));
}
await ((_b = this._githubCache) === null || _b === void 0 ? void 0 : _b.set("rawLastUpdate", Date.now()).save());
if (this.enka.options.showFetchCacheLog) {
console.info(">> All Completed");
}
}
this._isFetching = false;
}
/**
* @returns whether all genshin cache data files exist.
*/
hasAllContents() {
for (const lang of languages) {
if (!fs_1.default.existsSync(path_1.default.resolve(this.cacheDirectoryPath, "langs", `${lang}.json`)))
return false;
}
for (const excel of ExcelTransformer_1.excels) {
const fileName = `${excel}.json`;
if (!fs_1.default.existsSync(path_1.default.resolve(this.cacheDirectoryPath, "data", fileName)))
return false;
}
return true;
}
/**
* @param options.useRawGenshinData Whether to fetch from gitlab repo ({@link https://gitlab.com/Dimbreath/AnimeGameData}) instead of downloading cache.zip
* @param options.ghproxy Whether to use ghproxy.com
* @returns true if there were any updates, false if there were no updates.
*/
async updateContents(options = {}) {
var _a, _b;
const mergedOptions = (0, config_file_js_1.bindOptions)({
useRawGenshinData: false,
ghproxy: false,
onUpdateStart: undefined,
onUpdateEnd: undefined,
}, options);
if (await this.checkForUpdates(mergedOptions.useRawGenshinData)) {
await ((_a = mergedOptions.onUpdateStart) === null || _a === void 0 ? void 0 : _a.call(mergedOptions));
// fetch all because large file diff cannot be retrieved
await this.fetchAllContents({ useRawGenshinData: mergedOptions.useRawGenshinData, ghproxy: mergedOptions.ghproxy });
await ((_b = mergedOptions.onUpdateEnd) === null || _b === void 0 ? void 0 : _b.call(mergedOptions));
}
}
/**
* @param options.useRawGenshinData Whether to fetch from gitlab repo ({@link https://gitlab.com/Dimbreath/AnimeGameData}) instead of downloading cache.zip
* @param options.ghproxy Whether to use ghproxy.com
* @param options.timeout in milliseconds
*/
activateAutoCacheUpdater(options = {}) {
const mergedOptions = (0, config_file_js_1.bindOptions)({
useRawGenshinData: false,
instant: true,
ghproxy: false,
timeout: 60 * 60 * 1000,
onUpdateStart: null,
onUpdateEnd: null,
onError: null,
}, options);
if (mergedOptions.timeout < 60 * 1000)
throw new Error("timeout cannot be shorter than 1 minute.");
if (mergedOptions.instant)
this.updateContents({ onUpdateStart: mergedOptions.onUpdateStart, onUpdateEnd: mergedOptions.onUpdateEnd, useRawGenshinData: mergedOptions.useRawGenshinData, ghproxy: mergedOptions.ghproxy });
this._cacheUpdater = setInterval(async () => {
var _a;
if (this._isFetching)
return;
try {
this.updateContents({ onUpdateStart: mergedOptions.onUpdateStart, onUpdateEnd: mergedOptions.onUpdateEnd, useRawGenshinData: mergedOptions.useRawGenshinData, ghproxy: mergedOptions.ghproxy });
}
catch (e) {
if (e instanceof Error)
(_a = mergedOptions.onError) === null || _a === void 0 ? void 0 : _a.call(mergedOptions, e);
}
}, mergedOptions.timeout);
}
/**
* Disables the updater activated by [activateAutoCacheUpdater](#activateAutoCacheUpdater)
*/
deactivateAutoCacheUpdater() {
if (this._cacheUpdater !== null) {
clearInterval(this._cacheUpdater);
this._cacheUpdater = null;
}
}
/**
* @returns text map file path for a specific language
*/
getLanguageDataPath(lang, directory) {
const relativeDir = directory ? ["langs", directory] : ["langs"];
return path_1.default.resolve(this.cacheDirectoryPath, ...relativeDir, `${lang}.json`);
}
/**
* @param name without extensions (.json)
* @returns excel bin file path
*/
getJSONDataPath(name) {
return path_1.default.resolve(this.cacheDirectoryPath, "data", `${name}.json`);
}
_getExcelDataPath(excel) {
return path_1.default.resolve(this.cacheDirectoryPath, "data", `${excel}.json`);
}
_getExcelData(excel) {
var _a;
(_a = excelDataMemory[excel]) !== null && _a !== void 0 ? _a : (excelDataMemory[excel] = JSON.parse(fs_1.default.readFileSync(this._getExcelDataPath(excel), "utf-8")));
const excelData = excelDataMemory[excel];
if (!excelData)
throw new Error(`Failed to load ${excel} excel.`);
return excelData;
}
getExcelData(excel, ...keys) {
const excelData = this._getExcelData(excel);
return (0, ExcelTransformer_1.indexBy)(excelData, ...keys);
}
formatExcel(excel, data) {
const transformer = new ExcelTransformer_1.ExcelTransformer();
return transformer.transform(excel, data);
}
/**
* @returns text map for a specific language
*/
getLanguageData(lang, directory) {
var _a, _b;
// Avoid "js/prototype-polluting-assignment" just in case. (https://github.com/yuko1101/enka-network-api/security/code-scanning/252)
if (["__proto__", "constructor", "prototype"].includes(lang))
return {};
(_a = langDataMemory[lang]) !== null && _a !== void 0 ? _a : (langDataMemory[lang] = JSON.parse(fs_1.default.readFileSync(this.getLanguageDataPath(lang), "utf-8")));
if (directory) {
const loadedJson = JSON.parse(fs_1.default.readFileSync(this.getLanguageDataPath(lang, directory), "utf-8"));
langDataMemory[lang] = Object.assign(langDataMemory[lang], loadedJson);
}
return (_b = langDataMemory[lang]) !== null && _b !== void 0 ? _b : {};
}
/**
* @returns ObjectKeysManager of this
*/
getObjectKeysManager() {
if (!objectKeysManager)
objectKeysManager = new ObjectKeysManager_1.ObjectKeysManager(this);
return objectKeysManager;
}
/**
* Clean memory of cache data.
* Then reload data that was loaded before the clean if `reload` is true.
* If `reload` is false, load each file as needed.
*/
refreshAllData(reload = false) {
const loadedData = reload ? Object.keys(excelDataMemory) : [];
const loadedLangs = reload ? Object.keys(langDataMemory) : null;
excelDataMemory = Object.assign({}, initialExcelDataMemory);
langDataMemory = Object.assign({}, initialLangDataMemory);
objectKeysManager = null;
if (reload && loadedData && loadedLangs) {
for (const name of loadedData) {
this._getExcelData(name);
}
for (const lang of loadedLangs) {
this.getLanguageData(lang);
}
objectKeysManager = new ObjectKeysManager_1.ObjectKeysManager(this);
}
}
/**
* Remove all unused text map entries
*/
removeUnusedTextData(data, langsData, showLog = true) {
const required = [];
function push(...keys) {
const len = keys.length;
for (let i = 0; i < len; i++) {
const key = keys[i];
if (!required.includes(key))
required.push(key);
}
}
push(...textMapWhiteList);
Object.values(data["AvatarExcelConfigData"]).forEach(c => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, c);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
Object.values(data["FetterInfoExcelConfigData"]).forEach(c => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, c);
push(json.getAsNumber("avatarNativeTextMapHash"), json.getAsNumber("avatarVisionBeforTextMapHash"), json.getAsNumber("avatarConstellationAfterTextMapHash"), json.getAsNumber("avatarConstellationBeforTextMapHash"), json.getAsNumber("avatarTitleTextMapHash"), json.getAsNumber("avatarDetailTextMapHash"), json.getAsNumber("cvChineseTextMapHash"), json.getAsNumber("cvJapaneseTextMapHash"), json.getAsNumber("cvEnglishTextMapHash"), json.getAsNumber("cvKoreanTextMapHash"));
});
Object.values(data["AvatarCostumeExcelConfigData"]).forEach(c => {
Object.values(c).forEach(s => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, s);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
});
Object.values(data["AvatarSkillExcelConfigData"]).forEach(s => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, s);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
Object.values(data["ProudSkillExcelConfigData"]).forEach(g => {
Object.values(g).forEach(p => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, p);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"), ...(json.has("paramDescList") ? json.get("paramDescList").mapArray((_, e) => e.getAsNumber()) : []));
});
});
Object.values(data["AvatarTalentExcelConfigData"]).forEach(c => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, c);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
Object.values(data["WeaponExcelConfigData"]).forEach(w => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, w);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
Object.values(data["EquipAffixExcelConfigData"]).forEach(a => {
Object.values(a).forEach(l => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, l);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
});
Object.values(data["ReliquaryExcelConfigData"]).forEach(a => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, a);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
Object.values(data["ManualTextMapConfigData"]).forEach(m => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, m);
const id = json.getAsString("textMapId");
if (!manualTextMapWhiteList.includes(id) && !id.startsWith("FIGHT_REACTION_") && !id.startsWith("FIGHT_PROP_") && !id.startsWith("PROP_") && !id.startsWith("WEAPON_"))
return;
push(json.getAsNumber("textMapContentTextMapHash"));
});
Object.values(data["MaterialExcelConfigData"]).forEach(m => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, m);
push(json.getAsNumber("nameTextMapHash"), json.getAsNumber("descTextMapHash"));
});
Object.values(data["ProfilePictureExcelConfigData"]).forEach(p => {
const json = new config_file_js_1.JsonReader(ExcelTransformer_1.excelJsonOptions, p);
push(json.getAsNumber("nameTextMapHash"));
});
const requiredStringKeys = required.filter(key => key).map(key => key.toString());
const keyCount = requiredStringKeys.length;
if (showLog)
console.info(`Required keys have been loaded (${keyCount.toLocaleString()} keys)`);
const clearLangsData = Object.assign({}, initialLangDataMemory);
for (const lang of Object.keys(langsData)) {
if (showLog)
console.info(`Modifying language "${lang}"...`);
clearLangsData[lang] = {};
for (let i = 0; i < keyCount; i++) {
const key = requiredStringKeys[i];
const text = langsData[lang][key];
if (text) {
clearLangsData[lang][key] = text;
}
else {
// console.warn(`Required key ${key} was not found in language ${lang}.`);
}
}
}
if (showLog)
console.info("Removing unused keys completed.");
return clearLangsData;
}
/**
* Download the zip file, which contains genshin cache data, from {@link https://raw.githubusercontent.com/yuko1101/enka-network-api/main/cache.zip}
* @param options.ghproxy Whether to use ghproxy.com
*/
async _downloadCacheZip(options = {}) {
options = (0, config_file_js_1.bindOptions)({
ghproxy: false,
}, options);
const url = (options.ghproxy ? "https://ghproxy.com/" : "") + "https://raw.githubusercontent.com/yuko1101/enka-network-api/main/cache.zip";
const res = await axios_1.default.get(url, {
responseType: "stream",
}).catch(e => {
throw new Error(`Failed to download genshin data from ${url} with an error: ${e}`);
});
if (res.status == 200) {
await new Promise(resolve => {
res.data
.pipe(unzip_stream_1.default.Parse())
.on("entry", (entry) => {
const entryPath = entry.path.replace(/^cache\/?/, "");
const extractPath = path_1.default.resolve(this.cacheDirectoryPath, entryPath);
if (this.enka.options.showFetchCacheLog)
console.info(`- Downloading ${entryPath}`);
if (entry.type === "Directory") {
if (!fs_1.default.existsSync(extractPath))
fs_1.default.mkdirSync(extractPath, { recursive: true });
entry.autodrain();
}
else if (entryPath.startsWith("github/")) {
if (fs_1.default.existsSync(extractPath)) {
entry.autodrain();
return;
}
entry.pipe(fs_1.default.createWriteStream(extractPath));
}
else {
entry.pipe(fs_1.default.createWriteStream(extractPath));
}
});
res.data.on("close", () => {
resolve();
});
});
}
else {
throw new Error(`Failed to download genshin data from ${url} with status ${res.status} - ${res.statusText}`);
}
}
/**
* @returns whether the cache is valid or not
*/
_validateCache(showLog = true) {
return (0, cache_utils_1.validateCache)(this.enka, showLog);
}
}
exports.CachedAssetsManager = CachedAssetsManager;