@a11ywatch/core
Version:
a11ywatch central API
316 lines • 15.5 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var g = generator.apply(thisArg, _arguments || []), i, q = [];
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
function fulfill(value) { resume("next", value); }
function reject(value) { resume("throw", value); }
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
};
var __asyncValues = (this && this.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.crawlMultiSite = exports.entriesFromWebsiteSync = exports.crawlPage = void 0;
const website_source_builder_1 = require("@a11ywatch/website-source-builder");
const messagers_1 = require("../../messagers");
const pubsub_1 = require("../../../database/pubsub");
const static_1 = require("../../static");
const utils_1 = require("../../utils");
const websites_1 = require("../../controllers/websites");
const issues_1 = require("../../controllers/issues");
const analytics_1 = require("../../controllers/analytics");
const pages_1 = require("../../../core/controllers/pages");
const users_1 = require("../../controllers/users");
const extract_page_data_1 = require("../../utils/shapes/extract-page-data");
const runners_1 = require("../../utils/filters/runners");
const fetch_issues_1 = require("./fetch-issues");
const event_1 = require("../../../event");
const config_1 = require("../../../config/config");
const find_1 = require("../../controllers/page-actions/find");
const scan_attempt_1 = require("../../controllers/users/update/scan-attempt");
const strings_1 = require("../../strings");
const collection_upsert_1 = require("../../utils/collection-upsert");
const errors_1 = require("../../strings/errors");
const message_1 = require("../../../web/messages/message");
const watcher_crawl_1 = require("./watcher_crawl");
const shape_response_1 = require("../../models/response/shape-response");
const crawl_tracking_1 = require("../../../event/crawl-tracking");
const names_1 = require("../../../event/names");
const trackerProccess = (data, { domain, urlMap, userId, shutdown = false }, blockEvent) => {
event_1.crawlTrackingEmitter.emit("crawl-processed", {
user_id: userId,
domain,
pages: [urlMap],
shutdown,
});
if (!blockEvent && data) {
event_1.crawlEmitter.emit((0, names_1.getActiveCrawlKey)(domain, userId), data);
}
};
const getInsightsEnabled = ({ pageInsights, insightsLocked, pageSpeedApiKey, rootPage, }) => insightsLocked && !pageSpeedApiKey ? pageInsights && rootPage : pageInsights;
const crawlPage = (crawlConfig, sendEmail, blockEvent) => __awaiter(void 0, void 0, void 0, function* () {
var _a, _b;
const { url: urlMap, pageInsights = false, user: usr, sendSub: sub = true, html, standard, ignore, rules, runners, } = crawlConfig !== null && crawlConfig !== void 0 ? crawlConfig : {};
const userId = (_a = usr === null || usr === void 0 ? void 0 : usr.id) !== null && _a !== void 0 ? _a : crawlConfig === null || crawlConfig === void 0 ? void 0 : crawlConfig.userId;
const [userData, userCollection] = yield (0, users_1.UsersController)().getUser({
id: userId,
});
const { domain, pathname } = (0, website_source_builder_1.sourceBuild)(urlMap);
if (!sendEmail && (0, scan_attempt_1.validateScanEnabled)({ user: userData }) === false) {
trackerProccess(undefined, { domain, urlMap, userId, shutdown: true }, blockEvent);
return (0, shape_response_1.shapeResponse)({
data: null,
code: 300,
success: false,
message: strings_1.RATE_EXCEEDED_ERROR,
});
}
const [website, websiteCollection] = yield (0, websites_1.getWebsite)({
domain,
userId,
});
const { role: urole, pageSpeedApiKey, scanInfo } = userData !== null && userData !== void 0 ? userData : {};
const { standard: websiteStandard, pageHeaders, pageInsights: websitePageInsights, mobile, ua, ignore: websiteIgnore, rules: websiteRules, runners: websiteRunners, actionsEnabled, } = website !== null && website !== void 0 ? website : {};
const freeAccount = !urole;
const rootPage = pathname === "/";
const actions = actionsEnabled &&
(yield (0, find_1.findPageActionsByPath)({
userId,
path: pathname,
domain: website.domain,
}));
const dataSource = yield (0, fetch_issues_1.fetchPageIssues)({
pageHeaders,
url: urlMap,
userId,
pageInsights: getInsightsEnabled({
pageInsights: pageInsights || websitePageInsights,
insightsLocked: !config_1.SUPER_MODE && freeAccount,
pageSpeedApiKey: !!pageSpeedApiKey,
rootPage,
}),
mobile,
ua,
standard: standard || websiteStandard,
actions,
cv: config_1.SUPER_MODE || !!urole,
pageSpeedApiKey: pageSpeedApiKey,
html,
ignore: ignore && Array.isArray(ignore) && ignore.length ? ignore : websiteIgnore,
rules: rules && Array.isArray(rules) && rules.length ? rules : websiteRules,
runners: (0, runners_1.filterRunnerDuplicates)(runners && Array.isArray(runners) && runners.length
? runners
: websiteRunners || []),
});
let shutdown = false;
if (!config_1.SUPER_MODE) {
const ttime = (dataSource === null || dataSource === void 0 ? void 0 : dataSource.usage) || 0;
shutdown =
(0, scan_attempt_1.validateScanEnabled)({
user: {
role: urole,
scanInfo: {
usageLimit: scanInfo === null || scanInfo === void 0 ? void 0 : scanInfo.usageLimit,
totalUptime: ttime + (scanInfo === null || scanInfo === void 0 ? void 0 : scanInfo.totalUptime) || 0,
},
},
}) === false;
setImmediate(() => __awaiter(void 0, void 0, void 0, function* () {
yield (0, collection_upsert_1.collectionIncrement)({
"scanInfo.totalUptime": ttime,
}, userCollection, { id: userId });
}));
}
if (!dataSource || !((_b = dataSource === null || dataSource === void 0 ? void 0 : dataSource.webPage) === null || _b === void 0 ? void 0 : _b.issuesInfo) || shutdown) {
trackerProccess(undefined, { domain, urlMap, userId, shutdown }, blockEvent);
return (0, shape_response_1.shapeResponse)({
data: null,
code: message_1.StatusCode.BadRequest,
success: false,
message: errors_1.SCAN_TIMEOUT,
});
}
const { issues: pageIssues, webPage, issuesInfo, } = (0, extract_page_data_1.extractPageData)(dataSource);
const pageUrl = webPage.url;
const issueCount = pageIssues.issues.length;
if (website) {
const analyticsCollection = (0, analytics_1.AnalyticsController)().getCollection;
const pagesCollection = (0, pages_1.PagesController)().getCollection;
setImmediate(() => __awaiter(void 0, void 0, void 0, function* () {
if (rootPage) {
yield (0, utils_1.collectionUpsert)({
domain: webPage.domain,
url: webPage.url,
pageLoadTime: webPage.pageLoadTime,
lastScanDate: webPage.lastScanDate,
online: true,
userId,
}, [websiteCollection, !!webPage], {
searchProps: { url: pageUrl, userId },
});
}
const [issueExist, issuesCollection] = yield (0, issues_1.IssuesController)().getIssue({ pageUrl, userId, noRetries: true }, true);
const hostname = website.tld || website.tld ? webPage.domain : undefined;
yield Promise.all([
(0, utils_1.collectionUpsert)({
pageUrl,
domain: website.domain,
hostname,
userId,
possibleIssuesFixedByCdn: issuesInfo.possibleIssuesFixedByCdn,
totalIssues: issuesInfo.totalIssues,
issuesFixedByCdn: issuesInfo.issuesFixedByCdn,
errorCount: issuesInfo.errorCount,
warningCount: issuesInfo.warningCount,
noticeCount: issuesInfo.noticeCount,
accessScore: issuesInfo.accessScore,
}, [analyticsCollection, "upsert"]),
(0, utils_1.collectionUpsert)({
issues: pageIssues.issues,
documentTitle: pageIssues.documentTitle,
pageUrl: pageIssues.pageUrl,
domain: website.domain,
hostname,
webdomain: website.domain,
userId,
}, [issuesCollection, issueExist, !issueCount], {
searchProps: { pageUrl, userId },
}),
(0, utils_1.collectionUpsert)({
url: webPage.url,
domain: website.domain,
pageLoadTime: webPage.pageLoadTime,
lastScanDate: webPage.lastScanDate,
hostname,
userId,
online: true,
}, [pagesCollection, "upsert"], {
searchProps: { url: pageUrl, userId },
}),
]);
if (sendEmail && (issuesInfo === null || issuesInfo === void 0 ? void 0 : issuesInfo.errorCount) && (userData === null || userData === void 0 ? void 0 : userData.emailConfirmed)) {
yield messagers_1.emailMessager.sendMail({
userId,
data: {
issues: pageIssues.issues,
documentTitle: pageIssues.documentTitle,
pageUrl: pageIssues.pageUrl,
domain: pageIssues.domain,
userId,
issuesInfo,
},
confirmedOnly: true,
sendEmail: true,
});
}
}));
}
if (issueCount && sub) {
setImmediate(() => __awaiter(void 0, void 0, void 0, function* () {
try {
yield pubsub_1.pubsub.publish(static_1.ISSUE_ADDED, {
issueAdded: {
domain: webPage.domain,
url: webPage.url,
pageLoadTime: webPage.pageLoadTime,
lastScanDate: webPage.lastScanDate,
issue: pageIssues.issues,
issuesInfo,
userId,
online: true,
},
});
}
catch (_) {
}
}));
}
const responseData = {
data: {
domain: webPage.domain,
url: webPage.url,
pageLoadTime: webPage.pageLoadTime,
lastScanDate: webPage.lastScanDate,
issues: pageIssues.issues,
issuesInfo,
userId,
online: true,
},
};
trackerProccess(responseData, {
domain,
urlMap,
userId,
shutdown,
}, blockEvent);
return (0, shape_response_1.shapeResponse)(responseData);
});
exports.crawlPage = crawlPage;
function entriesFromWebsite(pages, userId) {
return __asyncGenerator(this, arguments, function* entriesFromWebsite_1() {
for (const url of pages) {
yield yield __await([yield __await((0, exports.crawlPage)({ url, userId }, false)), url]);
}
});
}
function entriesFromWebsiteSync(pages) {
return __asyncGenerator(this, arguments, function* entriesFromWebsiteSync_1() {
for (const { url, userId, subdomains, tld, ua, proxy } of pages) {
yield yield __await([
!crawl_tracking_1.crawlingSet.has((0, crawl_tracking_1.getKey)(url, [], userId)) &&
(yield __await((0, watcher_crawl_1.watcherCrawl)({
url,
subdomains,
tld,
userId,
scan: true,
agent: ua,
proxy,
}))),
url,
]);
}
});
}
exports.entriesFromWebsiteSync = entriesFromWebsiteSync;
const crawlMultiSite = (data) => __awaiter(void 0, void 0, void 0, function* () {
var e_1, _c;
var _d;
const { pages = [], userId: uid, user_id } = data;
const userId = uid !== null && uid !== void 0 ? uid : user_id;
const responseData = [];
try {
for (var _e = __asyncValues(entriesFromWebsite(pages, userId)), _f; _f = yield _e.next(), !_f.done;) {
const [scanResult, url] = _f.value;
responseData.push((_d = scanResult.data) !== null && _d !== void 0 ? _d : { url, online: scanResult.success });
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_f && !_f.done && (_c = _e.return)) yield _c.call(_e);
}
finally { if (e_1) throw e_1.error; }
}
return responseData;
});
exports.crawlMultiSite = crawlMultiSite;
//# sourceMappingURL=crawl.js.map