flipkart-scraper
Version:
[](https://nodei.co/npm/flipkart-scraper/)
381 lines • 19.3 kB
JavaScript
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FlipkartScraper = void 0;
var axios_1 = __importDefault(require("axios"));
var events_1 = require("events");
var fastq_1 = __importDefault(require("fastq"));
var eachDeep_1 = __importDefault(require("deepdash/eachDeep"));
var util_1 = require("./util");
var numeral_1 = __importDefault(require("numeral"));
var enums_1 = require("./enums");
/**
* This the main class for Flipkart scraper
*/
var FlipkartScraper = /** @class */ (function (_super) {
__extends(FlipkartScraper, _super);
/**
* This is constructor of FlipkartScraper
* @param affiliateId
* @param affiliateToken
* @param options
*/
function FlipkartScraper(affiliateId, affiliateToken, options) {
if (options === void 0) { options = {}; }
var _this = _super.call(this) || this;
_this._baseUrl = 'https://affiliate-api.flipkart.net';
_this._requestedCount = 0;
_this._processedCount = 0;
_this._totalProductsCount = 0;
_this._retryCount = 0;
_this._errorCount = 0;
_this._retryHaltCount = 0;
_this._pendingCategory = [];
_this._completedCategory = [];
_this._downloadSize = 0;
_this._affiliateId = affiliateId;
_this._affiliateToken = affiliateToken;
_this._maxRequest = options.maxRequest || 0;
_this._concurrency = options.concurrency || 2;
_this._maxPage = options.maxPage || 0;
_this._queue = fastq_1.default(_this._worker.bind(_this), _this._concurrency);
_this._status = enums_1.ScraperStatus.READY;
_this.emit('ready');
return _this;
}
FlipkartScraper.prototype.stats = function (showAsNumbers) {
if (showAsNumbers === void 0) { showAsNumbers = false; }
var elapsed = 1;
if (this._status === enums_1.ScraperStatus.FINISHED) {
elapsed = this._endTime.getTime() - this._startTime.getTime();
}
else if (this._startTime) {
elapsed = new Date().getTime() - this._startTime.getTime();
}
var statsFormats = {
productsCount: '0.00a',
elapsed: util_1.durationFormatWithMs,
durationPerMillionProducts: util_1.durationFormatWithMs,
productsPerSec: function (value) { return numeral_1.default(value).format('0.00a') + " products/sec"; },
avgResponseTime: function (value) { return value + "ms"; },
requestPerSec: function (value) { return value + "/sec"; },
requestedCount: '0.00a',
processedCount: '0.00a',
errorCount: '0.0a',
retryCount: '0.0a',
retryHaltCount: '0.0a',
downloadSize: '0.00b',
downloadedSpeed: function (value) { return numeral_1.default(value).format('0.00b') + "/sec"; }
};
var stats = {
startTime: this._startTime,
endTime: this._endTime,
status: this._status,
concurrency: this._concurrency,
waitingRequests: this._queue.length(),
productsCount: this._totalProductsCount,
elapsed: elapsed,
durationPerMillionProducts: this._totalProductsCount ? Number(((elapsed / this._totalProductsCount) * 1000000).toFixed(0)) : 0,
productsPerSec: Number((this._totalProductsCount / (elapsed / 1000)).toFixed(0)),
avgResponseTime: Number((elapsed / (this._processedCount ? this._processedCount : 1)).toFixed(0)),
requestPerSec: Number((this._requestedCount / (elapsed / 1000)).toFixed(0)),
requestedCount: this._requestedCount,
processedCount: this._processedCount,
errorCount: this._errorCount,
retryCount: this._retryCount,
retryHaltCount: this._retryHaltCount,
pendingCategory: this._pendingCategory.length,
completedCategory: this._completedCategory.length,
downloadSize: this._downloadSize,
downloadedSpeed: Number((this._downloadSize / (elapsed / 1000)).toFixed(0)),
info: {
pendingCategories: this._pendingCategory,
completedCategories: this._completedCategory
}
};
if (!showAsNumbers) {
Object.entries(stats).forEach(function (_a) {
var key = _a[0], value = _a[1];
if (statsFormats[key]) {
if (typeof statsFormats[key] === 'function') {
stats[key] = statsFormats[key](value);
}
else
stats[key] = numeral_1.default(value).format(statsFormats[key]);
}
});
}
return stats;
};
/**
*
* @param categoriesToScrape
*/
FlipkartScraper.prototype.start = function (categoriesToScrape) {
if (categoriesToScrape === void 0) { categoriesToScrape = []; }
return __awaiter(this, void 0, void 0, function () {
var feedUrl, feedListing, categoryListing_1, finishedInfo, errorObj_1;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (this._status === enums_1.ScraperStatus.INPROGRESS || this._status === enums_1.ScraperStatus.PAUSED)
return [2 /*return*/, Promise.reject({
currentStaus: this._status,
message: 'Scraping already started. Check the status before start new scraping.'
})];
this._startTime = new Date();
this._status = enums_1.ScraperStatus.INPROGRESS;
feedUrl = this._baseUrl + "/affiliate/api/" + this._affiliateId + ".json";
_a.label = 1;
case 1:
_a.trys.push([1, 6, , 7]);
return [4 /*yield*/, this._getData(feedUrl)];
case 2:
feedListing = (_a.sent()).data;
categoryListing_1 = {};
eachDeep_1.default(feedListing, function (value, key, parent) {
if (key === 'get') {
var resourceName = parent.resourceName;
categoryListing_1[resourceName] = value;
var pendingCategoryInfo = {
category: resourceName,
startTime: new Date(),
noOfPages: 0,
elapsed: 0,
totalProducts: 0
};
if (categoriesToScrape.length) {
if (categoriesToScrape.includes(resourceName)) {
_this._pendingCategory.push(pendingCategoryInfo);
_this._enqueue({ url: value, pageNo: 1, category: resourceName, retryCount: 0 });
}
}
else {
_this._pendingCategory.push(pendingCategoryInfo);
_this._enqueue({ url: value, pageNo: 1, category: resourceName, retryCount: 0 });
}
}
});
_a.label = 3;
case 3:
if (!!this._queue.idle()) return [3 /*break*/, 5];
return [4 /*yield*/, util_1.sleep(1000)];
case 4:
_a.sent();
return [3 /*break*/, 3];
case 5:
this._endTime = new Date();
this._status = enums_1.ScraperStatus.FINISHED;
finishedInfo = { message: 'Scraping Completed', totalRequest: this._processedCount };
this.emit('finished', finishedInfo);
return [2 /*return*/, Promise.resolve('Scraping Completed.')];
case 6:
errorObj_1 = _a.sent();
this._errorCount++;
this.emit('error', errorObj_1);
return [2 /*return*/, Promise.reject(errorObj_1)];
case 7: return [2 /*return*/];
}
});
});
};
/**
*
* @param params
*/
FlipkartScraper.prototype._enqueue = function (params) {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
this._requestedCount++;
if (this._maxRequest === 0 || this._requestedCount <= this._maxRequest)
this._queue.push(params, this._onComplete.bind(this));
return [2 /*return*/];
});
});
};
FlipkartScraper.prototype._worker = function (params, cb) {
return __awaiter(this, void 0, void 0, function () {
var url, category, pageNo, retryCount, httpResponse, errorObj_2, _a, status_1, statusText, errorCode, retryParams, retryData, retryData;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
_b.trys.push([0, 2, , 3]);
url = params.url, category = params.category, pageNo = params.pageNo, retryCount = params.retryCount;
return [4 /*yield*/, this._getData(url)];
case 1:
httpResponse = _b.sent();
cb(null, { httpResponse: httpResponse, category: category, pageNo: pageNo, retryCount: retryCount });
return [3 /*break*/, 3];
case 2:
errorObj_2 = _b.sent();
this._errorCount++;
_a = errorObj_2.response || { status: -1, statusText: 'ERROR' }, status_1 = _a.status, statusText = _a.statusText;
errorCode = errorObj_2.code;
if (params.retryCount < 10 && (errorCode || status_1 >= 500)) {
retryParams = __assign(__assign({}, params), { retryCount: params.retryCount + 1 });
this._enqueue(retryParams);
retryData = __assign(__assign({}, retryParams), { status: status_1, statusText: statusText, errorCode: errorCode });
this.emit('retry', retryData);
this._retryCount++;
}
else {
retryData = __assign(__assign({}, params), { status: status_1, statusText: statusText, errorCode: errorCode });
this.emit('retryHalted', retryData);
this._retryHaltCount++;
}
cb(errorObj_2, null);
return [3 /*break*/, 3];
case 3: return [2 /*return*/];
}
});
});
};
/**
*
* @param errorObj
* @param completedResponse
*/
FlipkartScraper.prototype._onComplete = function (errorObj, completedResponse) {
this._processedCount++;
if (errorObj === null) {
var _a = completedResponse.httpResponse, apiData_1 = _a.data, url = _a.url;
var categoryStats_1 = this._pendingCategory.filter(function (item) { return item.category === completedResponse.category; })[0];
if (apiData_1.products.length) {
this._totalProductsCount += apiData_1.products.length;
this._pendingCategory.forEach(function (item) {
if (item.category === categoryStats_1.category) {
item.elapsed = new Date().getTime() - categoryStats_1.startTime.getTime();
item.totalProducts += apiData_1.products.length;
item.noOfPages += 1;
}
});
this.emit('data', {
url: url,
apiData: apiData_1,
category: completedResponse.category,
pageNo: completedResponse.pageNo,
retryCount: completedResponse.retryCount
});
}
if (apiData_1.nextUrl) {
if (this._maxPage === 0 || completedResponse.pageNo < this._maxPage)
this._enqueue({ url: apiData_1.nextUrl, category: completedResponse.category, pageNo: completedResponse.pageNo + 1, retryCount: 0 });
}
else {
var elapsed = new Date().getTime() - categoryStats_1.startTime.getTime();
var completedInfo = {
category: completedResponse.category,
noOfPages: completedResponse.pageNo,
totalProducts: apiData_1.products.length + ((completedResponse.pageNo - 1) * 500),
elapsed: elapsed
};
this._pendingCategory = this._pendingCategory.filter(function (item) { return item.category !== completedResponse.category; });
this._completedCategory.push(completedInfo);
this.emit('categoryCompleted', completedInfo);
}
}
else {
this.emit('error', errorObj);
}
};
/**
*
* @param url
*/
FlipkartScraper.prototype._getData = function (url) {
return __awaiter(this, void 0, void 0, function () {
var startTs, responseObj, duration, status_2, statusText, headers, apiData, responseInfo, errorObj_3;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
startTs = new Date().getTime();
return [4 /*yield*/, axios_1.default.get(url, {
headers: {
'Fk-Affiliate-Id': this._affiliateId,
'Fk-Affiliate-Token': this._affiliateToken,
}
})];
case 1:
responseObj = _a.sent();
duration = new Date().getTime() - startTs;
status_2 = responseObj.status, statusText = responseObj.statusText, headers = responseObj.headers, apiData = responseObj.data;
responseInfo = { url: url, status: status_2, statusText: statusText, duration: duration, headers: headers, data: apiData };
this._downloadSize += JSON.stringify(apiData).length + JSON.stringify(headers).length;
this.emit('response', responseInfo);
return [2 /*return*/, Promise.resolve(responseInfo)];
case 2:
errorObj_3 = _a.sent();
return [2 /*return*/, Promise.reject(errorObj_3)];
case 3: return [2 /*return*/];
}
});
});
};
return FlipkartScraper;
}(events_1.EventEmitter));
exports.FlipkartScraper = FlipkartScraper;
//# sourceMappingURL=index.js.map