instamancer
Version:
Scrape the Instagram API with Puppeteer
117 lines • 7.45 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.toJSON = exports.toCSV = exports.depot = void 0;
const axios_1 = __importDefault(require("axios"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const tmp = __importStar(require("tmp"));
const url_1 = require("url");
const download = __importStar(require("./download"));
async function depot(url, name, extension) {
try {
// Axios download
const downloadStream = await axios_1.default({
method: "GET",
responseType: "stream",
url,
});
// Extract headers
const contentType = downloadStream.headers["content-type"];
const contentLength = downloadStream.headers["content-length"];
// Upload path
const filePath = path.join(this.directory, name + "." + extension);
const uploadUrl = url_1.resolve(this.url, filePath);
// Axios depot
await axios_1.default({
data: downloadStream.data,
headers: {
"Content-Length": contentLength,
"Content-Type": contentType,
},
maxContentLength: Infinity,
method: "PUT",
...authURL(uploadUrl),
}).catch((error) => {
this.logger.error(`Uploading ${url} failed`, error);
});
}
catch (e) {
this.logger.error(`Uploading ${url} failed`, e);
}
}
exports.depot = depot;
function authURL(url) {
const components = new url_1.URL(url);
const auth = {
password: components.password,
username: components.username,
};
components.username = "";
components.password = "";
return {
auth,
url: components.toString(),
};
}
async function uploadFile(posts, filePath, fileFunc, contentType) {
// Create tmp file
const tmpFile = tmp.fileSync({ keep: true });
// Dump posts to file
await fileFunc(posts, tmpFile.name);
// Read file to a stream
const fileStream = fs.createReadStream(tmpFile.name);
const contentLength = fs.statSync(tmpFile.name).size;
// Upload file
const uploadUrl = url_1.resolve(this.url, filePath);
await axios_1.default({
data: fileStream,
headers: {
"Content-Length": contentLength,
"Content-Type": contentType,
},
maxContentLength: Infinity,
method: "PUT",
url: uploadUrl,
});
// Delete file
fs.unlinkSync(tmpFile.name);
}
/**
* Upload list of posts to a CSV file
*/
async function toCSV(posts, filePath) {
const uploader = uploadFile.bind(this);
await uploader(posts, filePath, download.toCSV, "text/csv");
}
exports.toCSV = toCSV;
/**
* Upload list of posts to a JSON file
*/
async function toJSON(posts, filePath) {
const uploader = uploadFile.bind(this);
await uploader(posts, filePath, download.toJSON, "text/json");
}
exports.toJSON = toJSON;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZGVwb3QuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJkZXBvdC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQUEsa0RBQTBCO0FBQzFCLHVDQUF5QjtBQUN6QiwyQ0FBNkI7QUFDN0IseUNBQTJCO0FBQzNCLDZCQUFpQztBQUVqQyxxREFBdUM7QUFRaEMsS0FBSyxVQUFVLEtBQUssQ0FFdkIsR0FBVyxFQUNYLElBQVksRUFDWixTQUFpQjtJQUVqQixJQUFJO1FBQ0EsaUJBQWlCO1FBQ2pCLE1BQU0sY0FBYyxHQUFHLE1BQU0sZUFBSyxDQUFDO1lBQy9CLE1BQU0sRUFBRSxLQUFLO1lBQ2IsWUFBWSxFQUFFLFFBQVE7WUFDdEIsR0FBRztTQUNOLENBQUMsQ0FBQztRQUVILGtCQUFrQjtRQUNsQixNQUFNLFdBQVcsR0FBRyxjQUFjLENBQUMsT0FBTyxDQUFDLGNBQWMsQ0FBQyxDQUFDO1FBQzNELE1BQU0sYUFBYSxHQUFHLGNBQWMsQ0FBQyxPQUFPLENBQUMsZ0JBQWdCLENBQUMsQ0FBQztRQUUvRCxjQUFjO1FBQ2QsTUFBTSxRQUFRLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsU0FBUyxFQUFFLElBQUksR0FBRyxHQUFHLEdBQUcsU0FBUyxDQUFDLENBQUM7UUFDbkUsTUFBTSxTQUFTLEdBQUcsYUFBTyxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsUUFBUSxDQUFDLENBQUM7UUFFOUMsY0FBYztRQUNkLE1BQU0sZUFBSyxDQUFDO1lBQ1IsSUFBSSxFQUFFLGNBQWMsQ0FBQyxJQUFJO1lBQ3pCLE9BQU8sRUFBRTtnQkFDTCxnQkFBZ0IsRUFBRSxhQUFhO2dCQUMvQixjQUFjLEVBQUUsV0FBVzthQUM5QjtZQUNELGdCQUFnQixFQUFFLFFBQVE7WUFDMUIsTUFBTSxFQUFFLEtBQUs7WUFDYixHQUFHLE9BQU8sQ0FBQyxTQUFTLENBQUM7U0FDeEIsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDLEtBQUssRUFBRSxFQUFFO1lBQ2YsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsYUFBYSxHQUFHLFNBQVMsRUFBRSxLQUFLLENBQUMsQ0FBQztRQUN4RCxDQUFDLENBQUMsQ0FBQztLQUNOO0lBQUMsT0FBTyxDQUFDLEVBQUU7UUFDUixJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxhQUFhLEdBQUcsU0FBUyxFQUFFLENBQUMsQ0FBQyxDQUFDO0tBQ25EO0FBQ0wsQ0FBQztBQXRDRCxzQkFzQ0M7QUFFRCxTQUFTLE9BQU8sQ0FDWixHQUFXO0lBRVgsTUFBTSxVQUFVLEdBQUcsSUFBSSxTQUFHLENBQUMsR0FBRyxDQUFDLENBQUM7SUFDaEMsTUFBTSxJQUFJLEdBQUc7UUFDVCxRQUFRLEVBQUUsVUFBVSxDQUFDLFFBQVE7UUFDN0IsUUFBUSxFQUFFLFVBQVUsQ0FBQyxRQUFRO0tBQ2hDLENBQUM7SUFDRixVQUFVLENBQUMsUUFBUSxHQUFHLEVBQUUsQ0FBQztJQUN6QixVQUFVLENBQUMsUUFBUSxHQUFHLEVBQUUsQ0FBQztJQUV6QixPQUFPO1FBQ0gsSUFBSTtRQUNKLEdBQUcsRUFBRSxVQUFVLENBQUMsUUFBUSxFQUFFO0tBQzdCLENBQUM7QUFDTixDQUFDO0FBRUQsS0FBSyxVQUFVLFVBQVUsQ0FFckIsS0FBZSxFQUNmLFFBQWdCLEVBQ2hCLFFBQThELEVBQzlELFdBQW1CO0lBRW5CLGtCQUFrQjtJQUNsQixNQUFNLE9BQU8sR0FBRyxHQUFHLENBQUMsUUFBUSxDQUFDLEVBQUMsSUFBSSxFQUFFLElBQUksRUFBQyxDQUFDLENBQUM7SUFFM0MscUJBQXFCO0lBQ3JCLE1BQU0sUUFBUSxDQUFDLEtBQUssRUFBRSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFcEMsd0JBQXdCO0lBQ3hCLE1BQU0sVUFBVSxHQUFHLEVBQUUsQ0FBQyxnQkFBZ0IsQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDckQsTUFBTSxhQUFhLEdBQUcsRUFBRSxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUMsSUFBSSxDQUFDO0lBRXJELGNBQWM7SUFDZCxNQUFNLFNBQVMsR0FBRyxhQUFPLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRSxRQUFRLENBQUMsQ0FBQztJQUM5QyxNQUFNLGVBQUssQ0FBQztRQUNSLElBQUksRUFBRSxVQUFVO1FBQ2hCLE9BQU8sRUFBRTtZQUNMLGdCQUFnQixFQUFFLGFBQWE7WUFDL0IsY0FBYyxFQUFFLFdBQVc7U0FDOUI7UUFDRCxnQkFBZ0IsRUFBRSxRQUFRO1FBQzFCLE1BQU0sRUFBRSxLQUFLO1FBQ2IsR0FBRyxFQUFFLFNBQVM7S0FDakIsQ0FBQyxDQUFDO0lBRUgsY0FBYztJQUNkLEVBQUUsQ0FBQyxVQUFVLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0FBQ2hDLENBQUM7QUFFRDs7R0FFRztBQUNJLEtBQUssVUFBVSxLQUFLLENBQWdCLEtBQWUsRUFBRSxRQUFnQjtJQUN4RSxNQUFNLFFBQVEsR0FBRyxVQUFVLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3ZDLE1BQU0sUUFBUSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUSxDQUFDLEtBQUssRUFBRSxVQUFVLENBQUMsQ0FBQztBQUNoRSxDQUFDO0FBSEQsc0JBR0M7QUFFRDs7R0FFRztBQUNJLEtBQUssVUFBVSxNQUFNLENBQWdCLEtBQWUsRUFBRSxRQUFnQjtJQUN6RSxNQUFNLFFBQVEsR0FBRyxVQUFVLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3ZDLE1BQU0sUUFBUSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUSxDQUFDLE1BQU0sRUFBRSxXQUFXLENBQUMsQ0FBQztBQUNsRSxDQUFDO0FBSEQsd0JBR0MifQ==