auto-gpt-ts
Version:
my take of Auto-GPT in typescript
153 lines • 6.86 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.scrapeText = exports.GetHyperlinks = exports.GetTextSummary = exports.BrowseWebsiteCommand = void 0;
const command_1 = require("./command");
const puppeteer = __importStar(require("puppeteer"));
const node_html_parser_1 = require("node-html-parser");
const text_1 = require("../processing/text");
const html_1 = require("../processing/html");
const fs = __importStar(require("fs"));
const validators_1 = require("../url-utils/validators");
let BrowseWebsiteCommand = class BrowseWebsiteCommand {
static browseWebsite(url, question) {
return __awaiter(this, void 0, void 0, function* () {
if (!(0, validators_1.isValidUrl)('http')) {
return `Error: Please provide a valid url.`;
}
const { text, links } = yield scrapeText((0, validators_1.sanitizeUrl)(url), question);
return `Answer gathered from website: ${text} \n \n Links: ${links}`;
});
}
};
BrowseWebsiteCommand = __decorate([
(0, command_1.CommandDecorator)({
name: 'browseWebsite',
description: 'Browse Website',
signature: '"url": string, "question": string'
})
], BrowseWebsiteCommand);
exports.BrowseWebsiteCommand = BrowseWebsiteCommand;
let GetTextSummary = class GetTextSummary {
static getTextSummary(url, question) {
return __awaiter(this, void 0, void 0, function* () {
if (!(0, validators_1.isValidUrl)("http")) {
return `Error: URL is not valid`;
}
const { text } = yield scrapeText((0, validators_1.sanitizeUrl)(url), question);
return `"" "Result" : ${text}""`;
});
}
};
GetTextSummary = __decorate([
(0, command_1.CommandDecorator)({
name: "getTextSummary",
description: "Get a summary of the text",
signature: '"url": string, "question": string',
})
], GetTextSummary);
exports.GetTextSummary = GetTextSummary;
let GetHyperlinks = class GetHyperlinks {
static getHyperlinks(url) {
return __awaiter(this, void 0, void 0, function* () {
if (!(0, validators_1.isValidUrl)("http")) {
return `Error: URL is not valid`;
}
const { links } = yield scrapeText((0, validators_1.sanitizeUrl)(url), "");
return `"" "Result" : ${links}""`;
});
}
};
GetHyperlinks = __decorate([
(0, command_1.CommandDecorator)({
name: "getHyperlinks",
description: "Get a list of hyperlinks from a webpage",
signature: '"url": string',
})
], GetHyperlinks);
exports.GetHyperlinks = GetHyperlinks;
function scrapeText(url, question) {
return __awaiter(this, void 0, void 0, function* () {
const browser = yield puppeteer.launch({
// headless: true,
headless: false,
timeout: 20000,
ignoreHTTPSErrors: true,
slowMo: 0,
args: [
'--disable-gpu',
'--disable-dev-shm-usage',
'--disable-setuid-sandbox',
'--no-first-run',
'--no-sandbox',
'--no-zygote',
'--window-size=1280,720',
],
});
const page = yield browser.newPage();
yield page.setViewport({ width: 1280, height: 720 });
yield page.setRequestInterception(true);
page.on('request', (interceptedRequest) => {
const blockResources = ['image', 'media', 'font'];
if (blockResources.includes(interceptedRequest.resourceType())) {
interceptedRequest.abort();
}
else {
interceptedRequest.continue();
}
});
yield page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36');
page.evaluateOnNewDocument(fs.readFileSync(require.resolve('./page-overlay.js'), 'utf8'));
yield page.goto(url, {
waitUntil: "networkidle2",
});
const html = yield page.content();
const root = (0, node_html_parser_1.parse)(html);
const text = yield (0, text_1.summarizeText)(url, root.structuredText, question, page);
const links = (0, html_1.extractHyperlinks)(root.querySelectorAll('a'), url);
yield browser.close();
return {
text,
links
};
});
}
exports.scrapeText = scrapeText;
//# sourceMappingURL=browse-web.js.map