@microfox/puppeteer-linkedin
Version:
Puppeteer LinkedIn - Run puppeteer on LinkedIn
1,248 lines (1,235 loc) • 67.7 kB
JavaScript
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __commonJS = (cb, mod) => function __require() {
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
};
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// node_modules/dotenv/package.json
var require_package = __commonJS({
"node_modules/dotenv/package.json"(exports2, module2) {
module2.exports = {
name: "dotenv",
version: "16.5.0",
description: "Loads environment variables from .env file",
main: "lib/main.js",
types: "lib/main.d.ts",
exports: {
".": {
types: "./lib/main.d.ts",
require: "./lib/main.js",
default: "./lib/main.js"
},
"./config": "./config.js",
"./config.js": "./config.js",
"./lib/env-options": "./lib/env-options.js",
"./lib/env-options.js": "./lib/env-options.js",
"./lib/cli-options": "./lib/cli-options.js",
"./lib/cli-options.js": "./lib/cli-options.js",
"./package.json": "./package.json"
},
scripts: {
"dts-check": "tsc --project tests/types/tsconfig.json",
lint: "standard",
pretest: "npm run lint && npm run dts-check",
test: "tap run --allow-empty-coverage --disable-coverage --timeout=60000",
"test:coverage": "tap run --show-full-coverage --timeout=60000 --coverage-report=lcov",
prerelease: "npm test",
release: "standard-version"
},
repository: {
type: "git",
url: "git://github.com/motdotla/dotenv.git"
},
homepage: "https://github.com/motdotla/dotenv#readme",
funding: "https://dotenvx.com",
keywords: [
"dotenv",
"env",
".env",
"environment",
"variables",
"config",
"settings"
],
readmeFilename: "README.md",
license: "BSD-2-Clause",
devDependencies: {
"@types/node": "^18.11.3",
decache: "^4.6.2",
sinon: "^14.0.1",
standard: "^17.0.0",
"standard-version": "^9.5.0",
tap: "^19.2.0",
typescript: "^4.8.4"
},
engines: {
node: ">=12"
},
browser: {
fs: false
}
};
}
});
// node_modules/dotenv/lib/main.js
var require_main = __commonJS({
"node_modules/dotenv/lib/main.js"(exports2, module2) {
"use strict";
var fs = require("fs");
var path = require("path");
var os = require("os");
var crypto = require("crypto");
var packageJson = require_package();
var version = packageJson.version;
var LINE = /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg;
function parse(src) {
const obj = {};
let lines = src.toString();
lines = lines.replace(/\r\n?/mg, "\n");
let match;
while ((match = LINE.exec(lines)) != null) {
const key = match[1];
let value = match[2] || "";
value = value.trim();
const maybeQuote = value[0];
value = value.replace(/^(['"`])([\s\S]*)\1$/mg, "$2");
if (maybeQuote === '"') {
value = value.replace(/\\n/g, "\n");
value = value.replace(/\\r/g, "\r");
}
obj[key] = value;
}
return obj;
}
function _parseVault(options) {
const vaultPath = _vaultPath(options);
const result = DotenvModule.configDotenv({ path: vaultPath });
if (!result.parsed) {
const err = new Error(`MISSING_DATA: Cannot parse ${vaultPath} for an unknown reason`);
err.code = "MISSING_DATA";
throw err;
}
const keys = _dotenvKey(options).split(",");
const length = keys.length;
let decrypted;
for (let i = 0; i < length; i++) {
try {
const key = keys[i].trim();
const attrs = _instructions(result, key);
decrypted = DotenvModule.decrypt(attrs.ciphertext, attrs.key);
break;
} catch (error) {
if (i + 1 >= length) {
throw error;
}
}
}
return DotenvModule.parse(decrypted);
}
function _warn(message) {
console.log(`[dotenv@${version}][WARN] ${message}`);
}
function _debug(message) {
console.log(`[dotenv@${version}][DEBUG] ${message}`);
}
function _dotenvKey(options) {
if (options && options.DOTENV_KEY && options.DOTENV_KEY.length > 0) {
return options.DOTENV_KEY;
}
if (process.env.DOTENV_KEY && process.env.DOTENV_KEY.length > 0) {
return process.env.DOTENV_KEY;
}
return "";
}
function _instructions(result, dotenvKey) {
let uri;
try {
uri = new URL(dotenvKey);
} catch (error) {
if (error.code === "ERR_INVALID_URL") {
const err = new Error("INVALID_DOTENV_KEY: Wrong format. Must be in valid uri format like dotenv://:key_1234@dotenvx.com/vault/.env.vault?environment=development");
err.code = "INVALID_DOTENV_KEY";
throw err;
}
throw error;
}
const key = uri.password;
if (!key) {
const err = new Error("INVALID_DOTENV_KEY: Missing key part");
err.code = "INVALID_DOTENV_KEY";
throw err;
}
const environment = uri.searchParams.get("environment");
if (!environment) {
const err = new Error("INVALID_DOTENV_KEY: Missing environment part");
err.code = "INVALID_DOTENV_KEY";
throw err;
}
const environmentKey = `DOTENV_VAULT_${environment.toUpperCase()}`;
const ciphertext = result.parsed[environmentKey];
if (!ciphertext) {
const err = new Error(`NOT_FOUND_DOTENV_ENVIRONMENT: Cannot locate environment ${environmentKey} in your .env.vault file.`);
err.code = "NOT_FOUND_DOTENV_ENVIRONMENT";
throw err;
}
return { ciphertext, key };
}
function _vaultPath(options) {
let possibleVaultPath = null;
if (options && options.path && options.path.length > 0) {
if (Array.isArray(options.path)) {
for (const filepath of options.path) {
if (fs.existsSync(filepath)) {
possibleVaultPath = filepath.endsWith(".vault") ? filepath : `${filepath}.vault`;
}
}
} else {
possibleVaultPath = options.path.endsWith(".vault") ? options.path : `${options.path}.vault`;
}
} else {
possibleVaultPath = path.resolve(process.cwd(), ".env.vault");
}
if (fs.existsSync(possibleVaultPath)) {
return possibleVaultPath;
}
return null;
}
function _resolveHome(envPath) {
return envPath[0] === "~" ? path.join(os.homedir(), envPath.slice(1)) : envPath;
}
function _configVault(options) {
const debug = Boolean(options && options.debug);
if (debug) {
_debug("Loading env from encrypted .env.vault");
}
const parsed = DotenvModule._parseVault(options);
let processEnv = process.env;
if (options && options.processEnv != null) {
processEnv = options.processEnv;
}
DotenvModule.populate(processEnv, parsed, options);
return { parsed };
}
function configDotenv(options) {
const dotenvPath = path.resolve(process.cwd(), ".env");
let encoding = "utf8";
const debug = Boolean(options && options.debug);
if (options && options.encoding) {
encoding = options.encoding;
} else {
if (debug) {
_debug("No encoding is specified. UTF-8 is used by default");
}
}
let optionPaths = [dotenvPath];
if (options && options.path) {
if (!Array.isArray(options.path)) {
optionPaths = [_resolveHome(options.path)];
} else {
optionPaths = [];
for (const filepath of options.path) {
optionPaths.push(_resolveHome(filepath));
}
}
}
let lastError;
const parsedAll = {};
for (const path2 of optionPaths) {
try {
const parsed = DotenvModule.parse(fs.readFileSync(path2, { encoding }));
DotenvModule.populate(parsedAll, parsed, options);
} catch (e) {
if (debug) {
_debug(`Failed to load ${path2} ${e.message}`);
}
lastError = e;
}
}
let processEnv = process.env;
if (options && options.processEnv != null) {
processEnv = options.processEnv;
}
DotenvModule.populate(processEnv, parsedAll, options);
if (lastError) {
return { parsed: parsedAll, error: lastError };
} else {
return { parsed: parsedAll };
}
}
function config(options) {
if (_dotenvKey(options).length === 0) {
return DotenvModule.configDotenv(options);
}
const vaultPath = _vaultPath(options);
if (!vaultPath) {
_warn(`You set DOTENV_KEY but you are missing a .env.vault file at ${vaultPath}. Did you forget to build it?`);
return DotenvModule.configDotenv(options);
}
return DotenvModule._configVault(options);
}
function decrypt(encrypted, keyStr) {
const key = Buffer.from(keyStr.slice(-64), "hex");
let ciphertext = Buffer.from(encrypted, "base64");
const nonce = ciphertext.subarray(0, 12);
const authTag = ciphertext.subarray(-16);
ciphertext = ciphertext.subarray(12, -16);
try {
const aesgcm = crypto.createDecipheriv("aes-256-gcm", key, nonce);
aesgcm.setAuthTag(authTag);
return `${aesgcm.update(ciphertext)}${aesgcm.final()}`;
} catch (error) {
const isRange = error instanceof RangeError;
const invalidKeyLength = error.message === "Invalid key length";
const decryptionFailed = error.message === "Unsupported state or unable to authenticate data";
if (isRange || invalidKeyLength) {
const err = new Error("INVALID_DOTENV_KEY: It must be 64 characters long (or more)");
err.code = "INVALID_DOTENV_KEY";
throw err;
} else if (decryptionFailed) {
const err = new Error("DECRYPTION_FAILED: Please check your DOTENV_KEY");
err.code = "DECRYPTION_FAILED";
throw err;
} else {
throw error;
}
}
}
function populate(processEnv, parsed, options = {}) {
const debug = Boolean(options && options.debug);
const override = Boolean(options && options.override);
if (typeof parsed !== "object") {
const err = new Error("OBJECT_REQUIRED: Please check the processEnv argument being passed to populate");
err.code = "OBJECT_REQUIRED";
throw err;
}
for (const key of Object.keys(parsed)) {
if (Object.prototype.hasOwnProperty.call(processEnv, key)) {
if (override === true) {
processEnv[key] = parsed[key];
}
if (debug) {
if (override === true) {
_debug(`"${key}" is already defined and WAS overwritten`);
} else {
_debug(`"${key}" is already defined and was NOT overwritten`);
}
}
} else {
processEnv[key] = parsed[key];
}
}
}
var DotenvModule = {
configDotenv,
_configVault,
_parseVault,
config,
decrypt,
parse,
populate
};
module2.exports.configDotenv = DotenvModule.configDotenv;
module2.exports._configVault = DotenvModule._configVault;
module2.exports._parseVault = DotenvModule._parseVault;
module2.exports.config = DotenvModule.config;
module2.exports.decrypt = DotenvModule.decrypt;
module2.exports.parse = DotenvModule.parse;
module2.exports.populate = DotenvModule.populate;
module2.exports = DotenvModule;
}
});
// src/index.ts
var index_exports = {};
__export(index_exports, {
LinkedInAuthenticated: () => LinkedInAuthenticated,
LinkedInPublic: () => LinkedInPublic,
handleModal: () => handleModal,
retry: () => retry,
scrapeArticleSection: () => scrapeArticleSection,
scrollPageToBottom: () => scrollPageToBottom,
waitForUrl: () => waitForUrl
});
module.exports = __toCommonJS(index_exports);
// src/core/public.ts
var import_puppeteer_sls = require("@microfox/puppeteer-sls");
var import_puppeteer_core = __toESM(require("puppeteer-core"), 1);
var import_dotenv = __toESM(require_main(), 1);
import_dotenv.default.config();
var LinkedInPublic = class {
constructor() {
/**
* The Puppeteer browser instance.
*/
this.browser = null;
/**
* The Puppeteer page instance.
*/
this.page = null;
}
async launch(options) {
var _a;
const launchProps = await (0, import_puppeteer_sls.puppeteerLaunchProps)();
const isLocal = process.env.IS_OFFLINE || process.env.SERVERLESS_OFFLINE;
this.browser = await import_puppeteer_core.default.launch({
...launchProps,
headless: (_a = options == null ? void 0 : options.headless) != null ? _a : true,
slowMo: isLocal ? 50 : 0
});
this.page = await this.browser.newPage();
await this.page.setUserAgent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
);
await this.page.setViewport({ width: 1920, height: 1080 });
}
async goto(url, options) {
if (!this.page) {
throw new Error("Page not initialized. Call launch() first.");
}
return this.page.goto(url, options);
}
async close() {
if (this.browser) {
await this.browser.close();
}
}
};
// src/utils/index.ts
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// src/public/articles.ts
async function scrapeCollaborativeArticleSummary(card) {
return card.evaluate((c) => {
var _a;
const linkElement = c.querySelector("a");
if (!linkElement) return null;
const titleElement = c.querySelector("h2");
const topicElements = c.querySelectorAll(
"a.tagged-topic, .content-hub-tagged-topics a"
);
const title = (_a = titleElement == null ? void 0 : titleElement.innerText.trim()) != null ? _a : "";
const url = linkElement.href;
const topics = Array.from(topicElements).map((el) => {
var _a2, _b;
return {
name: (_b = (_a2 = el.textContent) == null ? void 0 : _a2.trim()) != null ? _b : "",
url: el.href
};
});
if (title && url) {
return { title, url, topics };
}
return null;
});
}
async function scrapePerspective(contribution) {
return contribution.evaluate((c) => {
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
const authorName = (_c = (_b = (_a = c.querySelector(".contribution-author__name")) == null ? void 0 : _a.textContent) == null ? void 0 : _b.trim()) != null ? _c : "";
const headline = (_f = (_e = (_d = c.querySelector(".contribution-author__headline")) == null ? void 0 : _d.textContent) == null ? void 0 : _e.trim()) != null ? _f : "";
const text = (_i = (_h = (_g = c.querySelector(".contribution-text")) == null ? void 0 : _g.textContent) == null ? void 0 : _h.trim()) != null ? _i : "";
const likesText = (_k = (_j = c.querySelector('[data-test-id="social-actions__reactions"]')) == null ? void 0 : _j.getAttribute("data-num-reactions")) != null ? _k : "0";
const likes = parseInt(likesText, 10);
if (!authorName) {
return null;
}
return {
author: {
name: authorName,
headline
},
text,
likes
};
});
}
async function scrapeArticleSection(segment) {
const section = await segment.evaluate((s) => {
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
const sectionTitle = (_f = (_e = (_b = (_a = s.querySelector("h2 > span:last-of-type")) == null ? void 0 : _a.textContent) == null ? void 0 : _b.trim()) != null ? _e : (_d = (_c = s.querySelector("h2")) == null ? void 0 : _c.textContent) == null ? void 0 : _d.trim()) != null ? _f : "";
const content = (_i = (_h = (_g = s.querySelector(".article-main__content p")) == null ? void 0 : _g.textContent) == null ? void 0 : _h.trim()) != null ? _i : "";
if (!sectionTitle) return null;
return { title: sectionTitle, content, perspectives: [] };
});
if (!section) return null;
const perspectiveElements = await segment.$$("li.contribution-list-item");
const perspectives = [];
for (const el of perspectiveElements) {
const showMoreButton = await el.$("button:not([aria-label])");
if (showMoreButton) {
try {
await showMoreButton.click();
await delay(500);
} catch (e) {
}
}
const p = await scrapePerspective(el);
if (p) {
perspectives.push(p);
}
}
section.perspectives = perspectives;
return section;
}
async function retry(fn, maxRetries = 3, wait = 1e3) {
for (let i = 0; i < maxRetries; i++) {
try {
return await fn();
} catch (e) {
if (i < maxRetries - 1) {
await delay(wait * 2 ** i);
} else {
throw e;
}
}
}
throw new Error("Retry failed");
}
async function handleModal(page) {
try {
const modalSelector = ".modal__overlay";
const modalVisible = await page.$(modalSelector);
if (modalVisible) {
await page.mouse.click(10, 10, { delay: 50 });
await delay(1e3);
}
await page.waitForFunction(
() => !document.querySelector(".modal__overlay"),
{ timeout: 5e3 }
);
} catch (error) {
}
}
async function waitForUrl(page, urlPart, timeout = 3e4) {
await page.waitForFunction(
(part) => window.location.href.includes(part),
{ timeout },
urlPart
);
}
async function scrollPageToBottom(page) {
let lastHeight = await page.evaluate("document.body.scrollHeight");
let newHeight;
let scrollAttempts = 0;
const maxAttempts = 10;
const scrollStep = 1e3;
while (scrollAttempts < maxAttempts) {
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)");
await delay(scrollStep);
newHeight = await page.evaluate("document.body.scrollHeight");
if (newHeight === lastHeight) {
scrollAttempts++;
} else {
lastHeight = newHeight;
scrollAttempts = 0;
}
}
}
LinkedInPublic.prototype.getCollaborativeArticles = async function(options) {
const { page } = this;
if (!page) {
throw new Error("Page not initialized");
}
await retry(async () => {
await page.goto("https://www.linkedin.com/pulse/topics/home/", {
waitUntil: "domcontentloaded"
});
await page.waitForSelector("section.core-rail", { timeout: 15e3 });
});
await delay(2e3 + Math.random() * 2e3);
await handleModal(page);
let previousHeight;
let consecutiveNoChange = 0;
for (let i = 0; i < 30; i++) {
previousHeight = await page.evaluate("document.body.scrollHeight");
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)");
await delay(2e3 + Math.random() * 2e3);
const newHeight = await page.evaluate("document.body.scrollHeight");
if (newHeight === previousHeight) {
consecutiveNoChange++;
if (consecutiveNoChange > 3) {
break;
}
} else {
consecutiveNoChange = 0;
}
}
const articleCardElements = await page.$$(".content-hub-entity-card-redesign");
const articleSummaries = [];
for (const card of articleCardElements) {
if ((options == null ? void 0 : options.limit) && articleSummaries.length >= options.limit) break;
const summary = await scrapeCollaborativeArticleSummary(card);
if (summary) {
articleSummaries.push(summary);
}
}
return articleSummaries;
};
LinkedInPublic.prototype.getFullCollaborativeArticles = async function(options) {
const summaries = await this.getCollaborativeArticles(options);
const fullArticles = [];
for (const summary of summaries) {
const fullArticle = await this.scrapeArticlePage(summary);
if (fullArticle) {
fullArticles.push(fullArticle);
}
}
return fullArticles;
};
LinkedInPublic.prototype.scrapeArticlePage = async function(articleSummary) {
const { page } = this;
if (!page) {
throw new Error("Page not initialized");
}
try {
await page.goto(articleSummary.url, { waitUntil: "domcontentloaded" });
await delay(1500 + Math.random() * 1500);
await handleModal(page);
const segmentElements = await page.$$(".article-segment");
const sections = [];
for (const segment of segmentElements) {
const section = await scrapeArticleSection(segment);
if (section) {
sections.push(section);
}
}
return {
...articleSummary,
sections
};
} catch (e) {
console.error(`Failed to scrape article ${articleSummary.url}:`, e);
return null;
}
};
LinkedInPublic.prototype.scrapeMoreToExplore = async function(options) {
var _a;
const { page } = this;
if (!page) {
throw new Error("Page not initialized");
}
const summaries = await this.getCollaborativeArticles({ limit: 50 });
const allTopics = summaries.flatMap((s) => s.topics);
const uniqueTopics = Array.from(new Map(allTopics.map((item) => [item.url, item])).values());
const results = [];
const topicsToScrape = uniqueTopics.slice(
0,
(_a = options == null ? void 0 : options.topicLimit) != null ? _a : uniqueTopics.length
);
for (const topic of topicsToScrape) {
await page.goto(topic.url, { waitUntil: "domcontentloaded" });
await delay(1500 + Math.random() * 2500);
await handleModal(page);
try {
await page.waitForFunction(
"document.querySelectorAll('.content-hub-entity-card-redesign').length > 0",
{ timeout: 2e4 }
);
} catch (e) {
console.warn(`Could not find articles for topic "${topic.name}" at ${topic.url}`);
continue;
}
const articleCardElements = await page.$$(
".content-hub-entity-card-redesign"
);
const articles = [];
for (const card of articleCardElements) {
const summary = await scrapeCollaborativeArticleSummary(card);
if (summary) {
articles.push(summary);
}
}
results.push({ title: topic.name, url: topic.url, articles });
}
return results;
};
// src/auth/posts.ts
var LinkedInPosts = class {
get page() {
return this.getPage();
}
constructor(ensureLoggedIn, getPage) {
this.ensureLoggedIn = ensureLoggedIn;
this.getPage = getPage;
}
async reactToPost(postUrl, reaction) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to post: ${postUrl}`);
await this.page.goto(postUrl, { waitUntil: "domcontentloaded" });
try {
await this.page.waitForSelector(".feed-shared-social-action-bar", { timeout: 15e3 });
const likeButtonSelector = "button.react-button__trigger";
await this.page.waitForSelector(likeButtonSelector, { timeout: 1e4 });
if (reaction === "like") {
console.log("Liking the post");
await this.page.click(likeButtonSelector);
} else {
console.log(`Hovering over the like button to show other reactions`);
await this.page.hover(likeButtonSelector);
const reactionSelector = `button[aria-label="Send ${reaction}"]`;
console.log(`Waiting for reaction button: ${reactionSelector}`);
await this.page.waitForSelector(reactionSelector, { timeout: 5e3 });
console.log(`Clicking the "${reaction}" button`);
await this.page.click(reactionSelector);
}
console.log("Successfully reacted to the post.");
} catch (error) {
console.error("Failed to react to the post. Saving screenshot to reaction-failure.png");
if (this.page) {
await this.page.screenshot({ path: "reaction-failure.png" });
}
throw new Error(`Could not react to post: ${error.message}`);
}
}
async commentOnPost(postUrl, commentText) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to post: ${postUrl}`);
await this.page.goto(postUrl, { waitUntil: "domcontentloaded" });
try {
const commentButtonSelector = 'button[aria-label="Comment"]';
console.log("Waiting for the comment button...");
await this.page.waitForSelector(commentButtonSelector, { timeout: 1e4 });
await this.page.click(commentButtonSelector);
const editorSelector = '.ql-editor[contenteditable="true"]';
console.log("Waiting for the comment editor...");
await this.page.waitForSelector(editorSelector, { timeout: 1e4 });
console.log("Typing the comment...");
await this.page.type(editorSelector, commentText);
const postButtonSelector = "button.comments-comment-box__submit-button--cr";
console.log("Waiting for the post button...");
await this.page.waitForSelector(postButtonSelector, { timeout: 1e4 });
await this.page.click(postButtonSelector);
console.log("Successfully commented on the post.");
} catch (error) {
console.error("Failed to comment on the post. Saving screenshot to comment-failure.png");
if (this.page) {
await this.page.screenshot({ path: "comment-failure.png" });
}
throw new Error(`Could not comment on post: ${error.message}`);
}
}
async createPost(text) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log("Navigating to the feed to create a post...");
await this.page.goto("https://www.linkedin.com/feed/", { waitUntil: "domcontentloaded" });
try {
const startPostSelector = ".share-box-feed-entry__top-bar > button";
console.log('Waiting for the "Start a post" button...');
await this.page.waitForSelector(startPostSelector, { timeout: 1e4 });
await this.page.click(startPostSelector);
const editorSelector = '.ql-editor[contenteditable="true"]';
console.log("Waiting for the post editor...");
await this.page.waitForSelector(editorSelector, { timeout: 1e4 });
console.log("Typing the post content...");
await this.page.type(editorSelector, text);
const postButtonSelector = "button.share-actions__primary-action";
console.log("Waiting for the post button...");
await this.page.waitForSelector(postButtonSelector, { timeout: 1e4 });
await this.page.click(postButtonSelector);
console.log("Successfully created a new post.");
await new Promise((resolve) => setTimeout(resolve, 5e3));
} catch (error) {
console.error("Failed to create a post. Saving screenshot to create-post-failure.png");
if (this.page) {
await this.page.screenshot({ path: "create-post-failure.png" });
}
throw new Error(`Could not create post: ${error.message}`);
}
}
};
// src/auth/jobs.ts
var LinkedInJobs = class {
get page() {
return this.getPage();
}
constructor(ensureLoggedIn, getPage) {
this.ensureLoggedIn = ensureLoggedIn;
this.getPage = getPage;
}
async getRecommendedJobs(limit = 10) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log("Navigating to recommended jobs page...");
await this.page.goto("https://www.linkedin.com/jobs/recommender/recommended-jobs/", { waitUntil: "domcontentloaded" });
try {
const jobListSelector = ".jobs-recommender__job-listings-container";
console.log("Waiting for job listings to load...");
await this.page.waitForSelector(jobListSelector, { timeout: 15e3 });
console.log("Scraping recommended jobs...");
const jobs = await this.page.evaluate((limit2) => {
var _a, _b, _c, _d;
const jobElements = Array.from(document.querySelectorAll(".job-card-container"));
const jobData = [];
for (let i = 0; i < Math.min(jobElements.length, limit2); i++) {
const el = jobElements[i];
const title = ((_a = el.querySelector(".job-card-list__title")) == null ? void 0 : _a.innerText) || "TBD";
const company = ((_b = el.querySelector(".job-card-container__primary-description")) == null ? void 0 : _b.innerText) || "TBD";
const location = ((_c = el.querySelector(".job-card-container__metadata-item")) == null ? void 0 : _c.innerText) || "TBD";
const link = ((_d = el.querySelector("a.job-card-list__title")) == null ? void 0 : _d.href) || "TBD";
jobData.push({ title, company, location, link });
}
return jobData;
}, limit);
return jobs;
} catch (error) {
console.error("Failed to get recommended jobs. Saving screenshot to recommended-jobs-failure.png");
if (this.page) {
await this.page.screenshot({ path: "recommended-jobs-failure.png" });
}
throw new Error(`Could not get recommended jobs: ${error.message}`);
}
}
async searchJobs(options, limit = 10) {
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
const { keywords, location } = options;
const searchUrl = `https://www.linkedin.com/jobs/search/?keywords=${encodeURIComponent(keywords)}&location=${encodeURIComponent(location)}`;
console.log(`Navigating to job search page: ${searchUrl}`);
await this.page.goto(searchUrl, { waitUntil: "domcontentloaded" });
const jobListSelector = ".jobs-search-results-list";
console.log("Waiting for job list to load...");
await this.page.waitForSelector(jobListSelector, { timeout: 15e3 });
console.log("Scrolling to load all search results...");
await this.page.evaluate(async (selector) => {
const element = document.querySelector(selector);
if (!element) return;
await new Promise((resolve) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
const scrollHeight = element.scrollHeight;
element.scrollTop += distance;
totalHeight += distance;
if (totalHeight >= scrollHeight) {
clearInterval(timer);
resolve();
}
}, 100);
});
}, jobListSelector);
const jobCardSelector = ".scaffold-layout__list-container li.jobs-search-results__list-item";
const jobCards = await this.page.$$(jobCardSelector);
const jobs = [];
console.log(`Found ${jobCards.length} jobs in search. Scraping details for the first ${limit}...`);
for (let i = 0; i < Math.min(jobCards.length, limit); i++) {
try {
const card = jobCards[i];
const cardTitle = await card.$eval(".job-card-list__title", (el) => el.innerText.trim());
await card.click();
await this.page.waitForFunction(
(title) => {
var _a, _b;
const detailTitle = (_b = (_a = document.querySelector(".job-details-jobs-unified-top-card__job-title")) == null ? void 0 : _a.textContent) == null ? void 0 : _b.trim();
return detailTitle == null ? void 0 : detailTitle.includes(title);
},
{ timeout: 1e4 },
cardTitle
);
console.log(`Scraping job ${i + 1}/${limit}: ${cardTitle}`);
const job = await this.page.evaluate(() => {
var _a;
const getText = (selector) => {
var _a2;
return ((_a2 = document.querySelector(selector)) == null ? void 0 : _a2.innerText.trim()) || null;
};
const title = getText(".job-details-jobs-unified-top-card__job-title");
const company = getText("a.job-details-jobs-unified-top-card__company-name");
const location2 = getText(".job-details-jobs-unified-top-card__primary-description-container > div");
const description = getText(".jobs-description-content__text");
const applicantCount = getText(".jobs-unified-top-card__applicant-count");
const url = ((_a = document.querySelector(".job-details-jobs-unified-top-card__job-title a")) == null ? void 0 : _a.href) || window.location.href;
return { title, company, location: location2, description, applicantCount, url };
});
jobs.push(job);
} catch (error) {
console.error(`Failed to scrape job from search ${i + 1}: ${error.message}`);
}
}
return jobs;
}
// Placeholder for future job-related authenticated methods
};
// src/utils/scrolling.ts
var _autoScroll = async (page) => {
await page.evaluate(async () => {
await new Promise((resolve) => {
let totalHeight = 0;
const distance = 100;
const timer = setInterval(() => {
const scrollHeight = document.body.scrollHeight;
window.scrollBy(0, distance);
totalHeight += distance;
if (totalHeight >= scrollHeight) {
clearInterval(timer);
resolve();
}
}, 100);
});
});
};
// src/auth/users.ts
var LinkedInUsers = class {
get page() {
return this.getPage();
}
constructor(ensureLoggedIn, getPage) {
this.ensureLoggedIn = ensureLoggedIn;
this.getPage = getPage;
}
async sendConnectionRequest(profileUrl, message) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to profile for connection request: ${profileUrl}`);
await this.page.goto(profileUrl, { waitUntil: "domcontentloaded" });
try {
const connectButtonSelector = `.artdeco-card button.artdeco-button--primary:not([aria-label="Message"])`;
console.log("Waiting for the Connect button...");
await this.page.waitForSelector(connectButtonSelector, { timeout: 1e4 });
await this.page.click(connectButtonSelector);
await this._handleConnectionModal(message);
} catch (error) {
console.error("Failed to send connection request. Saving screenshot to connect-failure.png");
if (this.page) {
await this.page.screenshot({ path: "connect-failure.png" });
}
throw new Error(`Could not send connection request: ${error.message}`);
}
}
async _handleConnectionModal(message) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized.");
}
const modalSelector = 'div[role="dialog"]';
console.log("Waiting for the connection modal...");
await this.page.waitForSelector(modalSelector, { timeout: 1e4 });
if (message) {
const addNoteButtonSelector = 'button[aria-label="Add a note"]';
console.log('Waiting for the "Add a note" button...');
await this.page.waitForSelector(addNoteButtonSelector, { timeout: 1e4 });
await this.page.click(addNoteButtonSelector);
const messageEditorSelector = 'textarea[name="message"]';
console.log("Waiting for the message editor...");
await this.page.waitForSelector(messageEditorSelector, { timeout: 1e4 });
await this.page.type(messageEditorSelector, message);
}
const sendButtonSelector = 'button[aria-label^="Send"]';
console.log("Waiting for the final send button...");
await this.page.waitForSelector(sendButtonSelector, { timeout: 1e4 });
await this.page.click(sendButtonSelector);
console.log("Successfully sent connection request.");
}
async followProfile(profileUrl) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to profile to follow: ${profileUrl}`);
await this.page.goto(profileUrl, { waitUntil: "domcontentloaded" });
try {
const followButtonSelector = `.artdeco-card .pvs-profile-actions__custom button[aria-label*="Follow"]`;
console.log("Waiting for the Follow button...");
await this.page.waitForSelector(followButtonSelector, { timeout: 1e4 });
await this.page.click(followButtonSelector);
console.log("Successfully followed the profile.");
} catch (error) {
console.error("Failed to follow profile. Saving screenshot to follow-failure.png");
if (this.page) {
await this.page.screenshot({ path: "follow-failure.png" });
}
throw new Error(`Could not follow profile: ${error.message}`);
}
}
async sendConnectionRequestOrFollow(profileUrl, message) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to profile: ${profileUrl}`);
await this.page.goto(profileUrl, { waitUntil: "domcontentloaded" });
const connectButtonSelector = `.artdeco-card button.artdeco-button--primary:not([aria-label="Message"])`;
const followButtonSelector = `.artdeco-card .pvs-profile-actions__custom button[aria-label*="Follow"]`;
try {
const connectButton = await this.page.$(connectButtonSelector);
if (connectButton) {
console.log("Connect button found. Sending connection request.");
await connectButton.click();
await this._handleConnectionModal(message);
return;
}
const followButton = await this.page.$(followButtonSelector);
if (followButton) {
console.log("Follow button found. Following profile.");
await followButton.click();
console.log("Successfully followed the profile.");
return;
}
throw new Error("Neither Connect nor Follow button was found on the profile.");
} catch (error) {
const errorMessage = `Failed to connect or follow: ${error.message}`;
console.error(`${errorMessage} Saving screenshot to action-failure.png`);
if (this.page) {
await this.page.screenshot({ path: "action-failure.png" });
}
throw new Error(errorMessage);
}
}
async sendMessage(profileUrl, message) {
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to profile: ${profileUrl} to send a message.`);
await this.page.goto(profileUrl, { waitUntil: "domcontentloaded" });
try {
const messageButtonSelector = `.artdeco-card button[aria-label^="Message"]`;
console.log("Waiting for the message button...");
await this.page.waitForSelector(messageButtonSelector, { timeout: 1e4 });
await this.page.click(messageButtonSelector);
const composerSelector = 'div.msg-form__contenteditable[role="textbox"]';
console.log("Waiting for the message composer...");
await this.page.waitForSelector(composerSelector, { timeout: 1e4 });
console.log("Typing message...");
await this.page.type(composerSelector, message);
const sendButtonSelector = ".msg-form__send-button";
console.log("Waiting for the send button...");
await this.page.waitForSelector(sendButtonSelector, { timeout: 1e4 });
await this.page.click(sendButtonSelector);
console.log("Successfully sent message.");
await new Promise((resolve) => setTimeout(resolve, 2e3));
} catch (error) {
console.error("Failed to send message. Saving screenshot to message-failure.png");
if (this.page) {
await this.page.screenshot({ path: "message-failure.png" });
}
throw new Error(`Could not send message: ${error.message}`);
}
}
async getProfile(profileUrl) {
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
console.log(`Navigating to profile: ${profileUrl} to scrape data.`);
await this.page.goto(profileUrl, { waitUntil: "domcontentloaded" });
console.log("Scrolling to load all profile sections...");
await _autoScroll(this.page);
console.log("Scraping profile data...");
const profileData = await this.page.evaluate(() => {
var _a, _b, _c;
const name = ((_a = document.querySelector("h1")) == null ? void 0 : _a.innerText) || "TBD";
const headline = ((_b = document.querySelector("div.text-body-medium.break-words")) == null ? void 0 : _b.innerText) || "TBD";
const location = ((_c = document.querySelector("span.text-body-small.inline.break-words")) == null ? void 0 : _c.innerText) || "TBD";
const aboutElement = document.querySelector("div.display-flex.ph5.pv3 > div.display-flex.full-width > div > div > span");
const about = aboutElement ? aboutElement.innerText : "TBD";
const experience = [];
const experienceElements = document.querySelectorAll("#experience-section ul > li");
experienceElements.forEach((el) => {
var _a2, _b2, _c2, _d, _e, _f;
const company = (_b2 = (_a2 = el.querySelector("p.text-body-small > span:nth-child(2)")) == null ? void 0 : _a2.textContent) == null ? void 0 : _b2.trim();
const position = (_d = (_c2 = el.querySelector("h3.t-16")) == null ? void 0 : _c2.textContent) == null ? void 0 : _d.trim();
const dateRange = (_f = (_e = el.querySelector("h4.t-14 > span:nth-child(2)")) == null ? void 0 : _e.textContent) == null ? void 0 : _f.trim();
experience.push({ company, position, dateRange });
});
const education = [];
const educationElements = document.querySelectorAll("#education-section ul > li");
educationElements.forEach((el) => {
var _a2, _b2, _c2, _d, _e, _f, _g, _h;
const school = (_b2 = (_a2 = el.querySelector("h3.pv-entity__school-name")) == null ? void 0 : _a2.textContent) == null ? void 0 : _b2.trim();
const degree = (_d = (_c2 = el.querySelector("p.pv-entity__degree-name > span:nth-child(2)")) == null ? void 0 : _c2.textContent) == null ? void 0 : _d.trim();
const field = (_f = (_e = el.querySelector("p.pv-entity__fos > span:nth-child(2)")) == null ? void 0 : _e.textContent) == null ? void 0 : _f.trim();
const dateRange = (_h = (_g = el.querySelector("p.pv-entity__dates > span:nth-child(2)")) == null ? void 0 : _g.textContent) == null ? void 0 : _h.trim();
education.push({ school, degree, field, dateRange });
});
const skills = [];
const skillsElements = document.querySelectorAll(".pv-skill-category-entity__name-text");
skillsElements.forEach((el) => {
skills.push(el.innerText.trim());
});
return {
url: window.location.href,
name,
headline,
location,
about,
experience,
education,
skills
};
});
console.log(profileData);
return profileData;
}
// Placeholder for future user-related authenticated methods
};
// src/auth/sales.ts
var LinkedInSales = class {
get page() {
return this.getPage();
}
constructor(ensureLoggedIn, getPage) {
this.ensureLoggedIn = ensureLoggedIn;
this.getPage = getPage;
}
async searchForLeads(options) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login with salesNavigator type first.");
}
const searchUrl = "https://www.linkedin.com/sales/search/people?viewAllFilters=true";
console.log(`Navigating to Sales Navigator search page: ${searchUrl}`);
await this.page.goto(searchUrl, { waitUntil: "domcontentloaded" });
await this.page.waitForSelector(".artdeco-modal__header");
console.log("Applying search filters...");
const applyCheckboxFilter = async (groupName, values) => {
if (!this.page) return;
const groupSelector = `fieldset.t-black--light:has(legend:not(.hidden__visually__):-soup-contains('${groupName}'))`;
await this.page.waitForSelector(groupSelector, { visible: true });
const groupElement = await this.page.$(groupSelector);
if (groupElement) {
for (const value of values) {
const checkboxLabelSelector = `label[for*="${value}"]`;
const checkbox = await groupElement.$(checkboxLabelSelector);
if (checkbox) {
await checkbox.click();
await new Promise((resolve) => setTimeout(resolve, 200));
}
}
}
};
const applyTextInputFilter = async (buttonText, inputPlaceholder, value) => {
if (!this.page) return;
const values = Array.isArray(value) ? value : [value];
for (const val of values) {
const button = await this.page.waitForSelector(`button::-p-text(${buttonText})`);
if (button) {
await button.click();
await new Promise((resolve) => setTimeout(resolve, 500));
await this.page.type(`input[placeholder="${inputPlaceholder}"]`, val);
await this.page.keyboard.press("Enter");
await new Promise((resolve) => setTimeout(resolve, 200));
}
}
};
if (options.keywords) {
await applyTextInputFilter("Keywords", "Search by keyword", options.keywords);
}
if (options.geography) {
await applyTextInputFilter("Geography", "Add locations", options.geography);
}
if (options.connection && options.connection.length > 0) {
await applyCheckboxFilter("Connection", options.connection);
}
if (options.seniorityLevel && options.seniorityLevel.length > 0) {
await applyCheckboxFilter("Seniority level", options.seniorityLevel);
}
if (options.industry) {
await applyTextInputFilter("Industry", "Add industries", options.industry);
}
if (options.currentCompany) {
await applyTextInputFilter("Current company", "Add companies", options.currentCompany);
}
if (options.currentJobTitle) {
await applyTextInputFilter("Current job title", "Add job titles", options.currentJobTitle);
}
const searchButtonSelector = 'button[data-control-name="search_form_submit"]';
await this.page.waitForSelector(searchButtonSelector);
await this.page.click(searchButtonSelector);
console.log("All filters applied and search initiated.");
await this.page.waitForNavigation({ waitUntil: "domcontentloaded" });
}
// Placeholder for future sales-related authenticated methods
};
// src/auth/search.ts
var LinkedInSearch = class {
get page() {
return this.getPage();
}
constructor(ensureLoggedIn, getPage) {
this.ensureLoggedIn = ensureLoggedIn;
this.getPage = getPage;
}
async search(query, category) {
await this.ensureLoggedIn();
if (!this.page) {
throw new Error("Page not initialized. Please login first.");
}
const searchUrl = `https://www.linkedin.com/search/results/${category}/?keywords=${encodeURIComponent(query)}`;
console.log(`Navigating to search page: ${searchUrl}`);
await this.page.goto(searchUrl, { waitUntil: "domcontentloaded" });
try {
const resultsContainerSelector = ".search-results-container";
console.log("Waiting for search results to load...");
await this.page.waitForSelector(resultsContainerSelector, { timeout: 15e3 });
const results = await this.page.evaluate(() => {
const items = Array.from(document.querySelectorAll(".reusable-search__result-container"));
return items.map((item) => {
var _a;
const titleElement = item.querySelector(".entity-result__title-text a");
const title = titleElement ? titleElement.innerText.trim() : "TBD";
const link = titleElement ? titleElement.href : "TBD";
const summary = ((_a = item.querySelector(".entity-result__summary")) == null ? void 0 : _a.innerText.trim()) || "";
return { title, link, summary };
});
});
return results;
} catch (error) {
console.error("Failed to perform search. Saving screenshot to search-failure.png");
if (this.page) {
await this.page.screenshot({ path: "search-failure.png" });
}
throw new Error(`Could not perform search: ${error.message}`);
}
}
async scrapePeople() {
if (!this.page) {
throw new Error("Page not initialized");
}
return this.page.evaluate(() => {
const results = [];
const resultElements = document.querySelectorAll("li.reusable-search__result-container");
resultElements.forEach((element) => {
const nameElement = element.querySelector('span[aria-hidden="true"]');
const anchorElement = element.querySelector("a.app-aware-link");
if (nameElement && anchorElement) {
const name = nameElement.innerText;
const profileUrl = anchorElement.href;
const headlineElement = element.querySelector(".entity-result__primary-subtitle");
const headline = headlineElement ? headlineElement.innerText : "";