instamancer
Version:
Scrape the Instagram API with Puppeteer
104 lines • 6.77 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.toJSON = exports.toCSV = exports.s3 = void 0;
const axios_1 = __importDefault(require("axios"));
const fs = __importStar(require("fs"));
const tmp = __importStar(require("tmp"));
const download = __importStar(require("./download"));
async function s3(url, name, extension) {
try {
// Axios download
const downloadStream = await axios_1.default({
method: "GET",
responseType: "stream",
url,
});
// Extract headers
const contentType = downloadStream.headers["content-type"];
const contentLength = downloadStream.headers["content-length"];
// s3 upload
await new Promise((resolve) => {
this.s3.upload({
Body: downloadStream.data,
Bucket: this.bucket,
ContentLength: contentLength,
ContentType: contentType,
Key: this.directory + "/" + name + "." + extension,
}, (err) => {
if (err !== null) {
this.logger.error(`Uploading ${url} failed`, err);
}
resolve();
});
});
}
catch (e) {
this.logger.error(`Uploading ${url} failed`, e);
}
}
exports.s3 = s3;
async function uploadFile(posts, filePath, fileFunc, contentType) {
// Create tmp file
const tmpFile = tmp.fileSync({ keep: true });
// Dump posts to file
await fileFunc(posts, tmpFile.name);
// Read file to a stream
const fileStream = fs.createReadStream(tmpFile.name);
const contentLength = fs.statSync(tmpFile.name).size;
// s3 upload
await new Promise((resolve) => {
this.s3.upload({
Body: fileStream,
Bucket: this.bucket,
ContentLength: contentLength,
ContentType: contentType,
Key: filePath,
}, (err) => {
if (err !== null) {
this.logger.error(`Uploading ${filePath} failed`, err);
}
resolve();
});
});
// Delete file
fs.unlinkSync(tmpFile.name);
}
/**
* Upload list of posts to a CSV file
*/
async function toCSV(posts, filePath) {
const uploader = uploadFile.bind(this);
await uploader(posts, filePath, download.toCSV, "text/csv");
}
exports.toCSV = toCSV;
/**
* Upload list of posts to a JSON file
*/
async function toJSON(posts, filePath) {
const uploader = uploadFile.bind(this);
await uploader(posts, filePath, download.toJSON, "text/json");
}
exports.toJSON = toJSON;
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiczMuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyJzMy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiOzs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0FBQ0Esa0RBQTBCO0FBQzFCLHVDQUF5QjtBQUN6Qix5Q0FBMkI7QUFFM0IscURBQXVDO0FBU2hDLEtBQUssVUFBVSxFQUFFLENBRXBCLEdBQVcsRUFDWCxJQUFZLEVBQ1osU0FBaUI7SUFFakIsSUFBSTtRQUNBLGlCQUFpQjtRQUNqQixNQUFNLGNBQWMsR0FBRyxNQUFNLGVBQUssQ0FBQztZQUMvQixNQUFNLEVBQUUsS0FBSztZQUNiLFlBQVksRUFBRSxRQUFRO1lBQ3RCLEdBQUc7U0FDTixDQUFDLENBQUM7UUFFSCxrQkFBa0I7UUFDbEIsTUFBTSxXQUFXLEdBQUcsY0FBYyxDQUFDLE9BQU8sQ0FBQyxjQUFjLENBQUMsQ0FBQztRQUMzRCxNQUFNLGFBQWEsR0FBRyxjQUFjLENBQUMsT0FBTyxDQUFDLGdCQUFnQixDQUFDLENBQUM7UUFFL0QsWUFBWTtRQUNaLE1BQU0sSUFBSSxPQUFPLENBQUMsQ0FBQyxPQUFPLEVBQUUsRUFBRTtZQUMxQixJQUFJLENBQUMsRUFBRSxDQUFDLE1BQU0sQ0FDVjtnQkFDSSxJQUFJLEVBQUUsY0FBYyxDQUFDLElBQUk7Z0JBQ3pCLE1BQU0sRUFBRSxJQUFJLENBQUMsTUFBTTtnQkFDbkIsYUFBYSxFQUFFLGFBQWE7Z0JBQzVCLFdBQVcsRUFBRSxXQUFXO2dCQUN4QixHQUFHLEVBQUUsSUFBSSxDQUFDLFNBQVMsR0FBRyxHQUFHLEdBQUcsSUFBSSxHQUFHLEdBQUcsR0FBRyxTQUFTO2FBQ3JELEVBQ0QsQ0FBQyxHQUFHLEVBQUUsRUFBRTtnQkFDSixJQUFJLEdBQUcsS0FBSyxJQUFJLEVBQUU7b0JBQ2QsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsYUFBYSxHQUFHLFNBQVMsRUFBRSxHQUFHLENBQUMsQ0FBQztpQkFDckQ7Z0JBQ0QsT0FBTyxFQUFFLENBQUM7WUFDZCxDQUFDLENBQ0osQ0FBQztRQUNOLENBQUMsQ0FBQyxDQUFDO0tBQ047SUFBQyxPQUFPLENBQUMsRUFBRTtRQUNSLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBSyxDQUFDLGFBQWEsR0FBRyxTQUFTLEVBQUUsQ0FBQyxDQUFDLENBQUM7S0FDbkQ7QUFDTCxDQUFDO0FBdkNELGdCQXVDQztBQUVELEtBQUssVUFBVSxVQUFVLENBRXJCLEtBQWUsRUFDZixRQUFnQixFQUNoQixRQUE4RCxFQUM5RCxXQUFtQjtJQUVuQixrQkFBa0I7SUFDbEIsTUFBTSxPQUFPLEdBQUcsR0FBRyxDQUFDLFFBQVEsQ0FBQyxFQUFDLElBQUksRUFBRSxJQUFJLEVBQUMsQ0FBQyxDQUFDO0lBRTNDLHFCQUFxQjtJQUNyQixNQUFNLFFBQVEsQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXBDLHdCQUF3QjtJQUN4QixNQUFNLFVBQVUsR0FBRyxFQUFFLENBQUMsZ0JBQWdCLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3JELE1BQU0sYUFBYSxHQUFHLEVBQUUsQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDLElBQUksQ0FBQztJQUVyRCxZQUFZO0lBQ1osTUFBTSxJQUFJLE9BQU8sQ0FBQyxDQUFDLE9BQU8sRUFBRSxFQUFFO1FBQzFCLElBQUksQ0FBQyxFQUFFLENBQUMsTUFBTSxDQUNWO1lBQ0ksSUFBSSxFQUFFLFVBQVU7WUFDaEIsTUFBTSxFQUFFLElBQUksQ0FBQyxNQUFNO1lBQ25CLGFBQWEsRUFBRSxhQUFhO1lBQzVCLFdBQVcsRUFBRSxXQUFXO1lBQ3hCLEdBQUcsRUFBRSxRQUFRO1NBQ2hCLEVBQ0QsQ0FBQyxHQUFHLEVBQUUsRUFBRTtZQUNKLElBQUksR0FBRyxLQUFLLElBQUksRUFBRTtnQkFDZCxJQUFJLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQyxhQUFhLFFBQVEsU0FBUyxFQUFFLEdBQUcsQ0FBQyxDQUFDO2FBQzFEO1lBQ0QsT0FBTyxFQUFFLENBQUM7UUFDZCxDQUFDLENBQ0osQ0FBQztJQUNOLENBQUMsQ0FBQyxDQUFDO0lBRUgsY0FBYztJQUNkLEVBQUUsQ0FBQyxVQUFVLENBQUMsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0FBQ2hDLENBQUM7QUFFRDs7R0FFRztBQUNJLEtBQUssVUFBVSxLQUFLLENBQWdCLEtBQWUsRUFBRSxRQUFnQjtJQUN4RSxNQUFNLFFBQVEsR0FBRyxVQUFVLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3ZDLE1BQU0sUUFBUSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUSxDQUFDLEtBQUssRUFBRSxVQUFVLENBQUMsQ0FBQztBQUNoRSxDQUFDO0FBSEQsc0JBR0M7QUFFRDs7R0FFRztBQUNJLEtBQUssVUFBVSxNQUFNLENBQWdCLEtBQWUsRUFBRSxRQUFnQjtJQUN6RSxNQUFNLFFBQVEsR0FBRyxVQUFVLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO0lBQ3ZDLE1BQU0sUUFBUSxDQUFDLEtBQUssRUFBRSxRQUFRLEVBQUUsUUFBUSxDQUFDLE1BQU0sRUFBRSxXQUFXLENBQUMsQ0FBQztBQUNsRSxDQUFDO0FBSEQsd0JBR0MifQ==