shamela/dist/index.js

Library to interact with the Maktabah Shamela v4 APIs

import{DEFAULT_MAPPING_RULES as e,DEFAULT_MASTER_METADATA_VERSION as t,FOOTNOTE_MARKER as n,FOREWORD_MARKER as r,UNKNOWN_VALUE_PLACEHOLDER as i}from"./utils/constants.js";import{convertContentToMarkdown as a,htmlToMarkdown as o,mapPageCharacterContent as s,moveContentAfterLineBreakIntoSpan as ee,normalizeHtml as te,normalizeLineEndings as ne,normalizeTitleSpans as re,parseContentRobust as ie,removeArabicNumericPageMarkers as ae,removeTagsExceptSpan as oe,splitPageBodyFromFooter as se,stripHtmlTags as ce}from"./content.js";import{denormalizeBooks as le}from"./transform.js";import"./types.js";import ue from"sql.js";import{unzipSync as de}from"fflate";var c=(e=>typeof require<`u`?require:typeof Proxy<`u`?new Proxy(e,{get:(e,t)=>(typeof require<`u`?require:e)[t]}):e)(function(e){if(typeof require<`u`)return require.apply(this,arguments);throw Error('Calling `require` for "'+e+"\" in an environment that doesn't expose the `require` function. See https://rolldown.rs/in-depth/bundling-cjs#require-external-modules for more details.")});const l=Object.freeze({debug:()=>{},error:()=>{},info:()=>{},warn:()=>{}});let u=l;const fe=e=>{if(!e){u=l;return}let t=[`debug`,`error`,`info`,`warn`].find(t=>typeof e[t]!=`function`);if(t)throw Error(`Logger must implement debug, error, info, and warn methods. Missing: ${String(t)}`);u=e},d=()=>u,f=()=>{u=l},p=new Proxy({},{get:(e,t)=>{let n=d(),r=n[t];return typeof r==`function`?(...e)=>r.apply(n,e):r}});let m={};const h={apiKey:`SHAMELA_API_KEY`,booksEndpoint:`SHAMELA_API_BOOKS_ENDPOINT`,masterPatchEndpoint:`SHAMELA_API_MASTER_PATCH_ENDPOINT`,sqlJsWasmUrl:`SHAMELA_SQLJS_WASM_URL`},pe=typeof process<`u`&&!!process?.env,g=e=>{let t=m[e];if(t!==void 0)return t;let n=h[e];if(pe)return process.env[n]},me=e=>{let{logger:t,...n}=e;`logger`in e&&fe(t),m={...m,...n}},_=e=>e===`fetchImplementation`?m.fetchImplementation:g(e),v=()=>({apiKey:g(`apiKey`),booksEndpoint:g(`booksEndpoint`),fetchImplementation:m.fetchImplementation,masterPatchEndpoint:g(`masterPatchEndpoint`),sqlJsWasmUrl:g(`sqlJsWasmUrl`)}),y=e=>{if(e===`fetchImplementation`)throw Error(`fetchImplementation must be provided via configure().`);let t=_(e);if(!t)throw Error(`${h[e]} environment variable not set`);return t},he=()=>{m={},f()},b=(e,t)=>e.query(`PRAGMA table_info(${t})`).all(),x=(e,t)=>!!e.query(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?1`).get(t),S=(e,t)=>x(e,t)?e.query(`SELECT * FROM ${t}`).all():[],C=e=>String(e.is_deleted)===`1`,w=(e,t,n)=>{let r={};for(let i of n){if(i===`id`){r.id=(t??e)?.id??null;continue}if(t&&i in t){let e=t[i];if(e!==`#`&&e!=null){r[i]=e;continue}}if(e&&i in e){r[i]=e[i];continue}r[i]=null}return r},ge=(e,t,n)=>{let r=new Set,i=new Map;for(let t of e)r.add(String(t.id));for(let e of t)i.set(String(e.id),e);let a=[];for(let t of e){let e=i.get(String(t.id));e&&C(e)||a.push(w(t,e,n))}for(let e of t){let t=String(e.id);r.has(t)||C(e)||a.push(w(void 0,e,n))}return a},_e=(e,t,n,r)=>{if(r.length===0)return;let i=n.map(()=>`?`).join(`,`),a=e.prepare(`INSERT INTO ${t} (${n.join(`,`)}) VALUES (${i})`);r.forEach(e=>{let t=n.map(t=>t in e?e[t]:null);a.run(...t)}),a.finalize()},ve=(e,t,n)=>{let r=t.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?1`).get(n);return r?.sql?(e.run(`DROP TABLE IF EXISTS ${n}`),e.run(r.sql),!0):(p.warn(`${n} table definition missing in source database`),!1)},T=(e,t,n,r)=>{if(!x(t,r)){p.warn(`${r} table missing in source database`);return}if(!ve(e,t,r))return;let i=b(t,r),a=n&&x(n,r)?b(n,r):[],o=i.map(e=>e.name);for(let t of a)if(!o.includes(t.name)){let n=t.type&&t.type.length>0?t.type:`TEXT`;e.run(`ALTER TABLE ${r} ADD COLUMN ${t.name} ${n}`),o.push(t.name)}_e(e,r,o,ge(S(t,r),n?S(n,r):[],o))},ye=(e,t,n)=>{e.transaction(()=>{T(e,t,n,`page`),T(e,t,n,`title`)})()},be=(e,t)=>{e.transaction(()=>{T(e,t,null,`page`),T(e,t,null,`title`)})()},xe=e=>{e.run(`CREATE TABLE page (
            id INTEGER,
            content TEXT,
            part TEXT,
            page TEXT,
            number TEXT,
            services TEXT,
            is_deleted TEXT
        )`),e.run(`CREATE TABLE title (
            id INTEGER,
            content TEXT,
            page INTEGER,
            parent INTEGER,
            is_deleted TEXT
        )`)},Se=e=>e.query(`SELECT * FROM page`).all(),E=e=>e.query(`SELECT * FROM title`).all(),D=e=>({pages:Se(e),titles:E(e)}),O=e=>{try{return c(`node:fs`).existsSync(e)}catch{return!1}},k=()=>{try{let e=c(`node:path`),t=process.cwd(),n=[e.join(t,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`..`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`../..`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`),e.join(t,`.next`,`server`,`node_modules`,`sql.js`,`dist`,`sql-wasm.wasm`)];for(let e of n)if(O(e))return e}catch{}},A=()=>{try{let e=c.resolve(`sql.js`),t=c(`node:path`),n=t.dirname(e),r=t.join(n,`dist`,`sql-wasm.wasm`);if(O(r))return r}catch{}},j=()=>{try{let e=c(`node:path`),t=c.resolve.paths(`sql.js`)||[];for(let n of t){let t=e.join(n,`sql.js`,`dist`,`sql-wasm.wasm`);if(O(t))return t}}catch{}},Ce=()=>{try{let e=new URL(`../../node_modules/sql.js/dist/sql-wasm.wasm`,import.meta.url),t=decodeURIComponent(e.pathname),n=process.platform===`win32`&&t.startsWith(`/`)?t.slice(1):t;if(O(n))return n}catch{}},we=()=>{let e=``;return c?.resolve!==void 0&&(e=A()),!e&&`cwd`in process&&(e=k()),!e&&c?.resolve?.paths!==void 0&&(e=j()),!e&&import.meta.url&&(e=Ce()),e};var Te=class{statement;constructor(e){this.statement=e}run=(...e)=>{e.length>0&&this.statement.bind(e),this.statement.step(),this.statement.reset()};finalize=()=>{this.statement.free()}},M=class{db;constructor(e){this.db=e}run=(e,t=[])=>{this.db.run(e,t)};prepare=e=>new Te(this.db.prepare(e));query=e=>({all:(...t)=>this.all(e,t),get:(...t)=>this.get(e,t)});transaction=e=>()=>{this.db.run(`BEGIN TRANSACTION`);try{e(),this.db.run(`COMMIT`)}catch(e){throw this.db.run(`ROLLBACK`),e}};close=()=>{this.db.close()};export=()=>this.db.export();all=(e,t)=>{let n=this.db.prepare(e);try{t.length>0&&n.bind(t);let e=[];for(;n.step();)e.push(n.getAsObject());return e}finally{n.free()}};get=(e,t)=>{let[n]=this.all(e,t);return n}};let N=null,P=null;const Ee=typeof process<`u`&&!!process?.versions?.node,De=()=>{if(!P){let e=_(`sqlJsWasmUrl`);if(e)P=e;else if(Ee){let e=we();if(e)P=e;else{let e=[`Unable to automatically locate sql-wasm.wasm file.`,`This can happen in bundled environments (Next.js, webpack, etc.).`,``,`Quick fix - add this to your code before using shamela:`,``,`  import { configure, createNodeConfig } from "shamela";`,`  configure(createNodeConfig({`,`    apiKey: process.env.SHAMELA_API_KEY,`,`    booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,`,`    masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,`,`  }));`,``,`Or manually specify the path:`,``,`  import { configure } from "shamela";`,`  import { join } from "node:path";`,`  configure({`,`    sqlJsWasmUrl: join(process.cwd(), "node_modules", "sql.js", "dist", "sql-wasm.wasm")`,`  });`].join(`
`);throw Error(e)}}else P=`https://cdn.jsdelivr.net/npm/sql.js@1.13.0/dist/sql-wasm.wasm`}return P},F=()=>(N||=ue({locateFile:()=>De()}),N),I=async()=>new M(new(await(F())).Database),L=async e=>new M(new(await(F())).Database(e)),Oe=(e,t,n)=>{let r=t.query(`SELECT sql FROM sqlite_master WHERE type='table' AND name = ?1`).get(n);if(!r?.sql)throw Error(`Missing table definition for ${n} in source database`);e.run(`DROP TABLE IF EXISTS ${n}`),e.run(r.sql)},ke=async(e,t)=>{let n={author:`author`,book:`book`,category:`category`},r={};for(let e of t){let t=n[(e.name.split(`/`).pop()?.split(`\\`).pop()??e.name).replace(/\.(sqlite|db)$/i,``).toLowerCase()];t&&(r[t]=await L(e.data))}try{let t=Object.entries(r);e.transaction(()=>{for(let[n,r]of t){Oe(e,r,n);let t=r.query(`PRAGMA table_info(${n})`).all().map(e=>e.name);if(t.length===0)continue;let i=r.query(`SELECT * FROM ${n}`).all();if(i.length===0)continue;let a=t.map(()=>`?`).join(`,`),o=t.map(e=>e===`order`?`"order"`:e),s=e.prepare(`INSERT INTO ${n} (${o.join(`,`)}) VALUES (${a})`);try{for(let e of i){let n=t.map(t=>t in e?e[t]:null);s.run(...n)}}finally{s.finalize()}}})()}finally{Object.values(r).forEach(e=>e?.close())}},R=(e,t,n)=>{e.run(`DROP VIEW IF EXISTS ${t}`),e.run(`CREATE VIEW ${t} AS SELECT * FROM ${n}`)},Ae=e=>{e.run(`CREATE TABLE author (
            id INTEGER,
            is_deleted TEXT,
            name TEXT,
            biography TEXT,
            death_text TEXT,
            death_number TEXT
        )`),e.run(`CREATE TABLE book (
            id INTEGER,
            name TEXT,
            is_deleted TEXT,
            category TEXT,
            type TEXT,
            date TEXT,
            author TEXT,
            printed TEXT,
            minor_release TEXT,
            major_release TEXT,
            bibliography TEXT,
            hint TEXT,
            pdf_links TEXT,
            metadata TEXT
        )`),e.run(`CREATE TABLE category (
            id INTEGER,
            is_deleted TEXT,
            "order" TEXT,
            name TEXT
        )`),R(e,`authors`,`author`),R(e,`books`,`book`),R(e,`categories`,`category`)},je=e=>e.query(`SELECT * FROM author`).all(),Me=e=>e.query(`SELECT * FROM book`).all(),Ne=e=>e.query(`SELECT * FROM category`).all(),z=(e,t)=>({authors:je(e),books:Me(e),categories:Ne(e),version:t}),B=(e,t=[`api_key`,`token`,`password`,`secret`,`auth`])=>{let n=typeof e==`string`?new URL(e):new URL(e.toString());return t.forEach(e=>{let t=n.searchParams.get(e);if(t&&t.length>6){let r=`${t.slice(0,3)}***${t.slice(-3)}`;n.searchParams.set(e,r)}else t&&n.searchParams.set(e,`***`)}),n.toString()},Pe=e=>({content:e.content,id:e.id,...e.number&&{number:e.number},...e.page&&{page:Number(e.page)},...e.part&&{part:e.part}}),Fe=e=>{let t=Number(e.parent);return{content:e.content,id:e.id,page:Number(e.page),...t&&{parent:t}}},V=e=>{let t=new URL(e);return t.protocol=`https`,t.toString()},H=e=>/\.(sqlite|db)$/i.test(e.name),U=e=>e.find(H),W=e=>{let t=/\.([^.]+)$/.exec(e);return t?`.${t[1].toLowerCase()}`:``},G=(e,t,n=!0)=>{let r=new URL(e),i=new URLSearchParams;return Object.entries(t).forEach(([e,t])=>{i.append(e,t.toString())}),n&&i.append(`api_key`,y(`apiKey`)),r.search=i.toString(),r},K=async(e,t={})=>{let n=typeof e==`string`?e:e.toString(),r=await(t.fetchImpl??v().fetchImplementation??fetch)(n);if(!r.ok)throw Error(`Error making request: ${r.status} ${r.statusText}`);if((r.headers.get(`content-type`)??``).includes(`application/json`))return await r.json();let i=await r.arrayBuffer();return new Uint8Array(i)},Ie=typeof process<`u`&&!!process?.versions?.node,Le=async()=>{if(!Ie)throw Error(`File system operations are only supported in Node.js environments`);return import(`node:fs/promises`)},Re=async e=>{let[t,n]=await Promise.all([Le(),import(`node:path`)]),r=n.dirname(e);return await t.mkdir(r,{recursive:!0}),t},q=async e=>{let t=await K(e),n=t instanceof Uint8Array?t.length:t&&typeof t.byteLength==`number`?t.byteLength:0;return p.debug(`unzipFromUrl:bytes`,n),new Promise((e,n)=>{let r=t instanceof Uint8Array?t:new Uint8Array(t);try{let t=de(r),n=Object.entries(t).map(([e,t])=>({data:t,name:e}));p.debug(`unzipFromUrl:entries`,n.map(e=>e.name)),e(n)}catch(e){n(Error(`Error processing URL: ${e.message}`))}})},J=async(e,t)=>{if(e.writer){await e.writer(t);return}if(!e.path)throw Error(`Output options must include either a writer or a path`);let n=await Re(e.path);typeof t==`string`?await n.writeFile(e.path,t,`utf-8`):await n.writeFile(e.path,t)},Y=[`author.sqlite`,`book.sqlite`,`category.sqlite`],ze=e=>{let t=new Set(e.map(e=>e.match(/[^\\/]+$/)?.[0]??e).map(e=>e.toLowerCase()));return Y.every(e=>t.has(e.toLowerCase()))},X=async(e,t)=>{p.info(`Setting up book database for ${e}`);let n=t||await Q(e),r=n.minorReleaseUrl?q(n.minorReleaseUrl):Promise.resolve([]),[i,a]=await Promise.all([q(n.majorReleaseUrl),r]),o=U(i);if(!o)throw Error(`Unable to locate book database in archive`);let s=await I();try{p.info(`Creating tables`),xe(s);let e=await L(o.data);try{let t=U(a);if(t){p.info(`Applying patches from ${t.name} to ${o.name}`);let n=await L(t.data);try{ye(s,e,n)}finally{n.close()}}else p.info(`Copying table data from ${o.name}`),be(s,e)}finally{e.close()}return{cleanup:async()=>{s.close()},client:s}}catch(e){throw s.close(),e}},Z=async(e,t)=>{p.info(`Setting up master database`);let n=e||await $(t??0);p.info(`Downloading master database ${n.version} from: ${B(n.url)}`);let r=await q(V(n.url));if(p.debug?.(`sourceTables downloaded: ${r.map(e=>e.name).toString()}`),!ze(r.map(e=>e.name)))throw p.error(`Some source tables were not found: ${r.map(e=>e.name).toString()}`),Error(`Expected tables not found!`);let i=await I();try{return p.info(`Creating master tables`),Ae(i),p.info(`Copying data to master table`),await ke(i,r.filter(H)),{cleanup:async()=>{i.close()},client:i,version:n.version}}catch(e){throw i.close(),e}},Q=async(e,t)=>{let n=G(`${y(`booksEndpoint`)}/${e}`,{major_release:(t?.majorVersion||0).toString(),minor_release:(t?.minorVersion||0).toString()});p.info(`Fetching shamela.ws book link: ${B(n)}`);try{let e=await K(n);return{majorRelease:e.major_release,majorReleaseUrl:V(e.major_release_url),...e.minor_release_url&&{minorReleaseUrl:V(e.minor_release_url)},...e.minor_release_url&&{minorRelease:e.minor_release}}}catch(e){throw Error(`Error fetching book metadata: ${e.message}`)}},Be=async(e,t)=>{if(p.info(`downloadBook ${e} ${JSON.stringify(t)}`),!t.outputFile.path)throw Error(`outputFile.path must be provided to determine output format`);let n=W(t.outputFile.path).toLowerCase(),{client:r,cleanup:i}=await X(e,t?.bookMetadata);try{if(n===`.json`){let e=await D(r);await J(t.outputFile,JSON.stringify(e,null,2))}else if(n===`.db`||n===`.sqlite`){let e=r.export();await J(t.outputFile,e)}else throw Error(`Unsupported output extension: ${n}`)}finally{await i()}return t.outputFile.path},$=async(e=0)=>{let t=G(y(`masterPatchEndpoint`),{version:e.toString()});p.info(`Fetching shamela.ws master database patch link: ${B(t)}`);try{let e=await K(t);return{url:e.patch_url,version:e.version}}catch(e){throw Error(`Error fetching master patch: ${e.message}`)}},Ve=e=>{let t=y(`masterPatchEndpoint`),{origin:n}=new URL(t);return`${n}/covers/${e}.jpg`},He=async e=>{if(p.info(`downloadMasterDatabase ${JSON.stringify(e)}`),!e.outputFile.path)throw Error(`outputFile.path must be provided to determine output format`);let t=W(e.outputFile.path),{client:n,cleanup:r,version:i}=await Z(e.masterMetadata);try{if(t===`.json`){let t=z(n,i);await J(e.outputFile,JSON.stringify(t,null,2))}else if(t===`.db`||t===`.sqlite`)await J(e.outputFile,n.export());else throw Error(`Unsupported output extension: ${t}`)}finally{await r()}return e.outputFile.path},Ue=async e=>{p.info(`getBook ${e}`);let{client:t,cleanup:n}=await X(e);try{let e=await D(t);return{pages:e.pages.map(Pe),titles:e.titles.map(Fe)}}finally{await n()}},We=async(e=0)=>{p.info(`getMaster`);let{client:t,cleanup:n,version:r}=await Z(void 0,e);try{return z(t,r)}finally{await n()}};export{e as DEFAULT_MAPPING_RULES,t as DEFAULT_MASTER_METADATA_VERSION,n as FOOTNOTE_MARKER,r as FOREWORD_MARKER,i as UNKNOWN_VALUE_PLACEHOLDER,G as buildUrl,me as configure,a as convertContentToMarkdown,le as denormalizeBooks,Be as downloadBook,He as downloadMasterDatabase,Ue as getBook,Q as getBookMetadata,v as getConfig,_ as getConfigValue,Ve as getCoverUrl,We as getMaster,$ as getMasterMetadata,o as htmlToMarkdown,K as httpsGet,s as mapPageCharacterContent,ee as moveContentAfterLineBreakIntoSpan,te as normalizeHtml,ne as normalizeLineEndings,re as normalizeTitleSpans,ie as parseContentRobust,ae as removeArabicNumericPageMarkers,oe as removeTagsExceptSpan,y as requireConfigValue,he as resetConfig,se as splitPageBodyFromFooter,ce as stripHtmlTags};
//# sourceMappingURL=index.js.map