@a11ywatch/core
Version:
a11ywatch central API
69 lines • 3.56 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.crawlStream = void 0;
const extracter_1 = require("../../web/params/extracter");
const crawl_tracking_1 = require("../../event/crawl-tracking");
const message_1 = require("../../web/messages/message");
const crawl_stream_1 = require("../utils/crawl-stream");
const crawl_stream_slim_1 = require("../utils/crawl-stream-slim");
const get_user_data_1 = require("../utils/get-user-data");
const crawl_config_1 = require("./crawl-config");
const crawlStream = (req, res, slim) => __awaiter(void 0, void 0, void 0, function* () {
var _a, _b;
const body = req.body;
const baseUrl = (body === null || body === void 0 ? void 0 : body.websiteUrl) || (body === null || body === void 0 ? void 0 : body.url);
const url = baseUrl && decodeURIComponent(baseUrl);
const removeTrailing = body.removeTrailing;
if (!url) {
res.status(message_1.StatusCode.BadRequest);
return res.send([]);
}
const userNext = yield (0, get_user_data_1.getUserFromApiScan)(((_a = req === null || req === void 0 ? void 0 : req.headers) === null || _a === void 0 ? void 0 : _a.authorization) || ((_b = req === null || req === void 0 ? void 0 : req.cookies) === null || _b === void 0 ? void 0 : _b.jwt), req, res);
if (userNext) {
const uid = userNext === null || userNext === void 0 ? void 0 : userNext.id;
if ((0, extracter_1.validateUID)(uid) && crawl_tracking_1.crawlingSet.has((0, crawl_tracking_1.getKey)(url, [], uid))) {
res.status(message_1.StatusCode.Accepted);
return res.send([]);
}
if (crawl_tracking_1.crawlingSet.has((0, crawl_tracking_1.getKey)(url, [], uid))) {
res.status(message_1.StatusCode.ToManyRequest);
return res.send([]);
}
res.raw.writeHead(message_1.StatusCode.Ok, {
"Content-Type": "application/json",
"Transfer-Encoding": "chunked",
"Access-Control-Allow-Origin": "*",
});
const crawlProps = yield (0, crawl_config_1.getCrawlConfig)({
id: userNext === null || userNext === void 0 ? void 0 : userNext.id,
url,
role: userNext === null || userNext === void 0 ? void 0 : userNext.role,
subdomains: body.subdomains,
tld: body.tld,
robots: body.robots,
sitemap: body.sitemap,
});
res.raw.write("[");
if (slim) {
yield (0, crawl_stream_slim_1.crawlHttpStreamSlim)(crawlProps, res, removeTrailing);
}
else {
yield (0, crawl_stream_1.crawlHttpStream)(crawlProps, res, removeTrailing);
}
res.raw.write("]");
setImmediate(() => {
res.raw.end();
});
}
});
exports.crawlStream = crawlStream;
//# sourceMappingURL=crawl.js.map