copy-llms-txt
Version:
A lightweight JavaScript library that lets you copy the content of a webpage as clean markdown with a simple keyboard shortcut.
803 lines (778 loc) • 19.9 kB
JavaScript
var B = Object.defineProperty;
var P = (e, t, n) => t in e ? B(e, t, { enumerable: !0, configurable: !0, writable: !0, value: n }) : e[t] = n;
var f = (e, t, n) => P(e, typeof t != "symbol" ? t + "" : t, n);
function x(e) {
for (var t = 1; t < arguments.length; t++) {
var n = arguments[t];
for (var r in n)
n.hasOwnProperty(r) && (e[r] = n[r]);
}
return e;
}
function y(e, t) {
return Array(t + 1).join(e);
}
function M(e) {
return e.replace(/^\n*/, "");
}
function I(e) {
for (var t = e.length; t > 0 && e[t - 1] === `
`; ) t--;
return e.substring(0, t);
}
var H = [
"ADDRESS",
"ARTICLE",
"ASIDE",
"AUDIO",
"BLOCKQUOTE",
"BODY",
"CANVAS",
"CENTER",
"DD",
"DIR",
"DIV",
"DL",
"DT",
"FIELDSET",
"FIGCAPTION",
"FIGURE",
"FOOTER",
"FORM",
"FRAMESET",
"H1",
"H2",
"H3",
"H4",
"H5",
"H6",
"HEADER",
"HGROUP",
"HR",
"HTML",
"ISINDEX",
"LI",
"MAIN",
"MENU",
"NAV",
"NOFRAMES",
"NOSCRIPT",
"OL",
"OUTPUT",
"P",
"PRE",
"SECTION",
"TABLE",
"TBODY",
"TD",
"TFOOT",
"TH",
"THEAD",
"TR",
"UL"
];
function E(e) {
return C(e, H);
}
var A = [
"AREA",
"BASE",
"BR",
"COL",
"COMMAND",
"EMBED",
"HR",
"IMG",
"INPUT",
"KEYGEN",
"LINK",
"META",
"PARAM",
"SOURCE",
"TRACK",
"WBR"
];
function k(e) {
return C(e, A);
}
function F(e) {
return R(e, A);
}
var b = [
"A",
"TABLE",
"THEAD",
"TBODY",
"TFOOT",
"TH",
"TD",
"IFRAME",
"SCRIPT",
"AUDIO",
"VIDEO"
];
function $(e) {
return C(e, b);
}
function V(e) {
return R(e, b);
}
function C(e, t) {
return t.indexOf(e.nodeName) >= 0;
}
function R(e, t) {
return e.getElementsByTagName && t.some(function(n) {
return e.getElementsByTagName(n).length;
});
}
var s = {};
s.paragraph = {
filter: "p",
replacement: function(e) {
return `
` + e + `
`;
}
};
s.lineBreak = {
filter: "br",
replacement: function(e, t, n) {
return n.br + `
`;
}
};
s.heading = {
filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
replacement: function(e, t, n) {
var r = Number(t.nodeName.charAt(1));
if (n.headingStyle === "setext" && r < 3) {
var i = y(r === 1 ? "=" : "-", e.length);
return `
` + e + `
` + i + `
`;
} else
return `
` + y("#", r) + " " + e + `
`;
}
};
s.blockquote = {
filter: "blockquote",
replacement: function(e) {
return e = e.replace(/^\n+|\n+$/g, ""), e = e.replace(/^/gm, "> "), `
` + e + `
`;
}
};
s.list = {
filter: ["ul", "ol"],
replacement: function(e, t) {
var n = t.parentNode;
return n.nodeName === "LI" && n.lastElementChild === t ? `
` + e : `
` + e + `
`;
}
};
s.listItem = {
filter: "li",
replacement: function(e, t, n) {
e = e.replace(/^\n+/, "").replace(/\n+$/, `
`).replace(/\n/gm, `
`);
var r = n.bulletListMarker + " ", i = t.parentNode;
if (i.nodeName === "OL") {
var a = i.getAttribute("start"), o = Array.prototype.indexOf.call(i.children, t);
r = (a ? Number(a) + o : o + 1) + ". ";
}
return r + e + (t.nextSibling && !/\n$/.test(e) ? `
` : "");
}
};
s.indentedCodeBlock = {
filter: function(e, t) {
return t.codeBlockStyle === "indented" && e.nodeName === "PRE" && e.firstChild && e.firstChild.nodeName === "CODE";
},
replacement: function(e, t, n) {
return `
` + t.firstChild.textContent.replace(/\n/g, `
`) + `
`;
}
};
s.fencedCodeBlock = {
filter: function(e, t) {
return t.codeBlockStyle === "fenced" && e.nodeName === "PRE" && e.firstChild && e.firstChild.nodeName === "CODE";
},
replacement: function(e, t, n) {
for (var r = t.firstChild.getAttribute("class") || "", i = (r.match(/language-(\S+)/) || [null, ""])[1], a = t.firstChild.textContent, o = n.fence.charAt(0), c = 3, l = new RegExp("^" + o + "{3,}", "gm"), u; u = l.exec(a); )
u[0].length >= c && (c = u[0].length + 1);
var h = y(o, c);
return `
` + h + i + `
` + a.replace(/\n$/, "") + `
` + h + `
`;
}
};
s.horizontalRule = {
filter: "hr",
replacement: function(e, t, n) {
return `
` + n.hr + `
`;
}
};
s.inlineLink = {
filter: function(e, t) {
return t.linkStyle === "inlined" && e.nodeName === "A" && e.getAttribute("href");
},
replacement: function(e, t) {
var n = t.getAttribute("href");
n && (n = n.replace(/([()])/g, "\\$1"));
var r = m(t.getAttribute("title"));
return r && (r = ' "' + r.replace(/"/g, '\\"') + '"'), "[" + e + "](" + n + r + ")";
}
};
s.referenceLink = {
filter: function(e, t) {
return t.linkStyle === "referenced" && e.nodeName === "A" && e.getAttribute("href");
},
replacement: function(e, t, n) {
var r = t.getAttribute("href"), i = m(t.getAttribute("title"));
i && (i = ' "' + i + '"');
var a, o;
switch (n.linkReferenceStyle) {
case "collapsed":
a = "[" + e + "][]", o = "[" + e + "]: " + r + i;
break;
case "shortcut":
a = "[" + e + "]", o = "[" + e + "]: " + r + i;
break;
default:
var c = this.references.length + 1;
a = "[" + e + "][" + c + "]", o = "[" + c + "]: " + r + i;
}
return this.references.push(o), a;
},
references: [],
append: function(e) {
var t = "";
return this.references.length && (t = `
` + this.references.join(`
`) + `
`, this.references = []), t;
}
};
s.emphasis = {
filter: ["em", "i"],
replacement: function(e, t, n) {
return e.trim() ? n.emDelimiter + e + n.emDelimiter : "";
}
};
s.strong = {
filter: ["strong", "b"],
replacement: function(e, t, n) {
return e.trim() ? n.strongDelimiter + e + n.strongDelimiter : "";
}
};
s.code = {
filter: function(e) {
var t = e.previousSibling || e.nextSibling, n = e.parentNode.nodeName === "PRE" && !t;
return e.nodeName === "CODE" && !n;
},
replacement: function(e) {
if (!e) return "";
e = e.replace(/\r?\n|\r/g, " ");
for (var t = /^`|^ .*?[^ ].* $|`$/.test(e) ? " " : "", n = "`", r = e.match(/`+/gm) || []; r.indexOf(n) !== -1; ) n = n + "`";
return n + t + e + t + n;
}
};
s.image = {
filter: "img",
replacement: function(e, t) {
var n = m(t.getAttribute("alt")), r = t.getAttribute("src") || "", i = m(t.getAttribute("title")), a = i ? ' "' + i + '"' : "";
return r ? "![" + n + "](" + r + a + ")" : "";
}
};
function m(e) {
return e ? e.replace(/(\n+\s*)+/g, `
`) : "";
}
function S(e) {
this.options = e, this._keep = [], this._remove = [], this.blankRule = {
replacement: e.blankReplacement
}, this.keepReplacement = e.keepReplacement, this.defaultRule = {
replacement: e.defaultReplacement
}, this.array = [];
for (var t in e.rules) this.array.push(e.rules[t]);
}
S.prototype = {
add: function(e, t) {
this.array.unshift(t);
},
keep: function(e) {
this._keep.unshift({
filter: e,
replacement: this.keepReplacement
});
},
remove: function(e) {
this._remove.unshift({
filter: e,
replacement: function() {
return "";
}
});
},
forNode: function(e) {
if (e.isBlank) return this.blankRule;
var t;
return (t = p(this.array, e, this.options)) || (t = p(this._keep, e, this.options)) || (t = p(this._remove, e, this.options)) ? t : this.defaultRule;
},
forEach: function(e) {
for (var t = 0; t < this.array.length; t++) e(this.array[t], t);
}
};
function p(e, t, n) {
for (var r = 0; r < e.length; r++) {
var i = e[r];
if (U(i, t, n)) return i;
}
}
function U(e, t, n) {
var r = e.filter;
if (typeof r == "string") {
if (r === t.nodeName.toLowerCase()) return !0;
} else if (Array.isArray(r)) {
if (r.indexOf(t.nodeName.toLowerCase()) > -1) return !0;
} else if (typeof r == "function") {
if (r.call(e, t, n)) return !0;
} else
throw new TypeError("`filter` needs to be a string, array, or function");
}
function W(e) {
var t = e.element, n = e.isBlock, r = e.isVoid, i = e.isPre || function(L) {
return L.nodeName === "PRE";
};
if (!(!t.firstChild || i(t))) {
for (var a = null, o = !1, c = null, l = N(c, t, i); l !== t; ) {
if (l.nodeType === 3 || l.nodeType === 4) {
var u = l.data.replace(/[ \r\n\t]+/g, " ");
if ((!a || / $/.test(a.data)) && !o && u[0] === " " && (u = u.substr(1)), !u) {
l = g(l);
continue;
}
l.data = u, a = l;
} else if (l.nodeType === 1)
n(l) || l.nodeName === "BR" ? (a && (a.data = a.data.replace(/ $/, "")), a = null, o = !1) : r(l) || i(l) ? (a = null, o = !0) : a && (o = !1);
else {
l = g(l);
continue;
}
var h = N(c, l, i);
c = l, l = h;
}
a && (a.data = a.data.replace(/ $/, ""), a.data || g(a));
}
}
function g(e) {
var t = e.nextSibling || e.parentNode;
return e.parentNode.removeChild(e), t;
}
function N(e, t, n) {
return e && e.parentNode === t || n(t) ? t.nextSibling || t.parentNode : t.firstChild || t.nextSibling || t.parentNode;
}
var w = typeof window < "u" ? window : {};
function z() {
var e = w.DOMParser, t = !1;
try {
new e().parseFromString("", "text/html") && (t = !0);
} catch {
}
return t;
}
function _() {
var e = function() {
};
return K() ? e.prototype.parseFromString = function(t) {
var n = new window.ActiveXObject("htmlfile");
return n.designMode = "on", n.open(), n.write(t), n.close(), n;
} : e.prototype.parseFromString = function(t) {
var n = document.implementation.createHTMLDocument("");
return n.open(), n.write(t), n.close(), n;
}, e;
}
function K() {
var e = !1;
try {
document.implementation.createHTMLDocument("").open();
} catch {
w.ActiveXObject && (e = !0);
}
return e;
}
var j = z() ? w.DOMParser : _();
function q(e, t) {
var n;
if (typeof e == "string") {
var r = G().parseFromString(
// DOM parsers arrange elements in the <head> and <body>.
// Wrapping in a custom element ensures elements are reliably arranged in
// a single element.
'<x-turndown id="turndown-root">' + e + "</x-turndown>",
"text/html"
);
n = r.getElementById("turndown-root");
} else
n = e.cloneNode(!0);
return W({
element: n,
isBlock: E,
isVoid: k,
isPre: t.preformattedCode ? X : null
}), n;
}
var v;
function G() {
return v = v || new j(), v;
}
function X(e) {
return e.nodeName === "PRE" || e.nodeName === "CODE";
}
function Y(e, t) {
return e.isBlock = E(e), e.isCode = e.nodeName === "CODE" || e.parentNode.isCode, e.isBlank = Q(e), e.flankingWhitespace = J(e, t), e;
}
function Q(e) {
return !k(e) && !$(e) && /^\s*$/i.test(e.textContent) && !F(e) && !V(e);
}
function J(e, t) {
if (e.isBlock || t.preformattedCode && e.isCode)
return { leading: "", trailing: "" };
var n = Z(e.textContent);
return n.leadingAscii && T("left", e, t) && (n.leading = n.leadingNonAscii), n.trailingAscii && T("right", e, t) && (n.trailing = n.trailingNonAscii), { leading: n.leading, trailing: n.trailing };
}
function Z(e) {
var t = e.match(/^(([ \t\r\n]*)(\s*))(?:(?=\S)[\s\S]*\S)?((\s*?)([ \t\r\n]*))$/);
return {
leading: t[1],
// whole string for whitespace-only strings
leadingAscii: t[2],
leadingNonAscii: t[3],
trailing: t[4],
// empty for whitespace-only strings
trailingNonAscii: t[5],
trailingAscii: t[6]
};
}
function T(e, t, n) {
var r, i, a;
return e === "left" ? (r = t.previousSibling, i = / $/) : (r = t.nextSibling, i = /^ /), r && (r.nodeType === 3 ? a = i.test(r.nodeValue) : n.preformattedCode && r.nodeName === "CODE" ? a = !1 : r.nodeType === 1 && !E(r) && (a = i.test(r.textContent))), a;
}
var ee = Array.prototype.reduce, te = [
[/\\/g, "\\\\"],
[/\*/g, "\\*"],
[/^-/g, "\\-"],
[/^\+ /g, "\\+ "],
[/^(=+)/g, "\\$1"],
[/^(#{1,6}) /g, "\\$1 "],
[/`/g, "\\`"],
[/^~~~/g, "\\~~~"],
[/\[/g, "\\["],
[/\]/g, "\\]"],
[/^>/g, "\\>"],
[/_/g, "\\_"],
[/^(\d+)\. /g, "$1\\. "]
];
function d(e) {
if (!(this instanceof d)) return new d(e);
var t = {
rules: s,
headingStyle: "setext",
hr: "* * *",
bulletListMarker: "*",
codeBlockStyle: "indented",
fence: "```",
emDelimiter: "_",
strongDelimiter: "**",
linkStyle: "inlined",
linkReferenceStyle: "full",
br: " ",
preformattedCode: !1,
blankReplacement: function(n, r) {
return r.isBlock ? `
` : "";
},
keepReplacement: function(n, r) {
return r.isBlock ? `
` + r.outerHTML + `
` : r.outerHTML;
},
defaultReplacement: function(n, r) {
return r.isBlock ? `
` + n + `
` : n;
}
};
this.options = x({}, t, e), this.rules = new S(this.options);
}
d.prototype = {
/**
* The entry point for converting a string or DOM node to Markdown
* @public
* @param {String|HTMLElement} input The string or DOM node to convert
* @returns A Markdown representation of the input
* @type String
*/
turndown: function(e) {
if (!ie(e))
throw new TypeError(
e + " is not a string, or an element/document/fragment node."
);
if (e === "") return "";
var t = O.call(this, new q(e, this.options));
return ne.call(this, t);
},
/**
* Add one or more plugins
* @public
* @param {Function|Array} plugin The plugin or array of plugins to add
* @returns The Turndown instance for chaining
* @type Object
*/
use: function(e) {
if (Array.isArray(e))
for (var t = 0; t < e.length; t++) this.use(e[t]);
else if (typeof e == "function")
e(this);
else
throw new TypeError("plugin must be a Function or an Array of Functions");
return this;
},
/**
* Adds a rule
* @public
* @param {String} key The unique key of the rule
* @param {Object} rule The rule
* @returns The Turndown instance for chaining
* @type Object
*/
addRule: function(e, t) {
return this.rules.add(e, t), this;
},
/**
* Keep a node (as HTML) that matches the filter
* @public
* @param {String|Array|Function} filter The unique key of the rule
* @returns The Turndown instance for chaining
* @type Object
*/
keep: function(e) {
return this.rules.keep(e), this;
},
/**
* Remove a node that matches the filter
* @public
* @param {String|Array|Function} filter The unique key of the rule
* @returns The Turndown instance for chaining
* @type Object
*/
remove: function(e) {
return this.rules.remove(e), this;
},
/**
* Escapes Markdown syntax
* @public
* @param {String} string The string to escape
* @returns A string with Markdown syntax escaped
* @type String
*/
escape: function(e) {
return te.reduce(function(t, n) {
return t.replace(n[0], n[1]);
}, e);
}
};
function O(e) {
var t = this;
return ee.call(e.childNodes, function(n, r) {
r = new Y(r, t.options);
var i = "";
return r.nodeType === 3 ? i = r.isCode ? r.nodeValue : t.escape(r.nodeValue) : r.nodeType === 1 && (i = re.call(t, r)), D(n, i);
}, "");
}
function ne(e) {
var t = this;
return this.rules.forEach(function(n) {
typeof n.append == "function" && (e = D(e, n.append(t.options)));
}), e.replace(/^[\t\r\n]+/, "").replace(/[\t\r\n\s]+$/, "");
}
function re(e) {
var t = this.rules.forNode(e), n = O.call(this, e), r = e.flankingWhitespace;
return (r.leading || r.trailing) && (n = n.trim()), r.leading + t.replacement(n, e, this.options) + r.trailing;
}
function D(e, t) {
var n = I(e), r = M(t), i = Math.max(e.length - n.length, t.length - r.length), a = `
`.substring(0, i);
return n + a + r;
}
function ie(e) {
return e != null && (typeof e == "string" || e.nodeType && (e.nodeType === 1 || e.nodeType === 9 || e.nodeType === 11));
}
class ae {
constructor() {
// Initialize with default values to fix the "no initializer" error
f(this, "options", {
key: "c",
meta: !1,
turndownOptions: {},
selectorsToRemove: [
"nav",
"header",
"footer",
"aside",
".navigation",
".nav",
".menu",
".sidebar",
".ad",
".ads",
".advertisement",
"script",
"style",
"noscript",
"iframe"
]
});
f(this, "turndownService");
f(this, "initialized", !1);
/**
* Handle keydown events
*/
f(this, "handleKeyDown", (t) => {
(this.options.meta ? t.metaKey : t.ctrlKey) && t.key.toLowerCase() === this.options.key.toLowerCase() && (t.preventDefault(), this.copyPageContent());
});
this.turndownService = new d();
}
/**
* Determine if the current platform is macOS
*/
isMac() {
return navigator.platform.toLowerCase().includes("mac") || navigator.userAgent.toLowerCase().includes("mac");
}
/**
* Initialize the CopyLlmsTxt library
* @param options Configuration options (optional)
*/
init(t) {
if (this.initialized) {
console.warn(
"CopyLlmsTxt is already initialized. Call destroy() before reinitializing."
);
return;
}
const n = {
key: "c",
meta: this.isMac(),
// Use meta (Cmd) on Mac, Ctrl elsewhere
turndownOptions: {},
selectorsToRemove: [
"nav",
"header",
"footer",
"aside",
".navigation",
".nav",
".menu",
".sidebar",
".ad",
".ads",
".advertisement",
"script",
"style",
"noscript",
"iframe"
]
};
this.options = {
...n,
...t
}, this.turndownService = new d(this.options.turndownOptions), document.addEventListener("keydown", this.handleKeyDown), this.initialized = !0, console.log(
`CopyLlmsTxt initialized. Press ${this.options.meta ? "Meta/Cmd" : "Ctrl"}+${this.options.key} to copy content as markdown.`
);
}
/**
* Destroy the CopyLlmsTxt instance and remove event listeners
*/
destroy() {
this.initialized && (document.removeEventListener("keydown", this.handleKeyDown), this.initialized = !1, console.log("CopyLlmsTxt destroyed."));
}
/**
* Copy the current page content as markdown
*/
copyPageContent() {
try {
const t = document.body.cloneNode(!0);
this.cleanupDOM(t);
const n = this.turndownService.turndown(t);
navigator.clipboard.writeText(n).then(() => {
console.log("Page content copied as markdown!");
}).catch((r) => {
console.error("Failed to copy text: ", r);
});
} catch (t) {
console.error("Error copying page content:", t);
}
}
removeNonVisibleElements(t) {
const n = document.body.querySelectorAll("*"), r = t.querySelectorAll("*");
if (n.length !== r.length)
console.warn("Element count mismatch between original and clone");
else
for (let i = r.length - 1; i >= 0; i--) {
const a = n[i], o = r[i], c = window.getComputedStyle(a), l = a.getBoundingClientRect();
(c.display === "none" || c.visibility === "hidden" || c.opacity === "0" || l.height === 0 || l.width === 0) && o.parentNode && o.parentNode.removeChild(o);
}
}
removeElementsBySelector(t) {
[
"nav",
"header",
"footer",
"aside",
".navigation",
".nav",
".menu",
".sidebar",
".ad",
".ads",
".advertisement",
"script",
"style",
"noscript",
"iframe"
].forEach((r) => {
t.querySelectorAll(r).forEach((a) => {
var o;
return (o = a.parentNode) == null ? void 0 : o.removeChild(a);
});
});
}
/**
* Clean up the DOM element by removing unwanted elements
*/
cleanupDOM(t) {
this.removeNonVisibleElements(t), this.removeElementsBySelector(t);
}
/**
* Manually copy the current page content as markdown
*/
copy() {
this.copyPageContent();
}
}
const oe = new ae();
export {
oe as default
};