google-trends-api-client
Version:
An fully typed and easy to use client for the google trends api.
68 lines • 3.87 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.getFromApiCall = exports.getFromApiCallWithoutRequestCaching = exports.getFromApiCallWithoutRequestCachingOrRateLimiting = void 0;
const bottleneck_1 = __importDefault(require("bottleneck"));
const crypto_1 = __importDefault(require("crypto"));
const got_scraping_1 = require("got-scraping");
const url_fns_1 = require("url-fns");
const with_simple_caching_1 = require("with-simple-caching");
const getCookiesHeadersFromCookieCacheIfPossible_1 = require("../agent/cookies/getCookiesHeadersFromCookieCacheIfPossible");
const setCookiesFromAxiosResponseIfPossible_1 = require("../agent/cookies/setCookiesFromAxiosResponseIfPossible");
const useSessionTokenForCookieCacheIfPossible_1 = require("../agent/cookies/useSessionTokenForCookieCacheIfPossible");
const getUserAgentHeaderIfPossible_1 = require("../agent/getUserAgentHeaderIfPossible");
const useProxyIfPossible_1 = require("../agent/useProxyIfPossible");
const common_1 = require("../common");
const bottleneck = new bottleneck_1.default({
maxConcurrent: 1,
minTime: 1000, // wait a min of 1 second
});
const gotScrapingTyped = got_scraping_1.gotScraping;
const getFromApiCallWithoutRequestCachingOrRateLimiting = async ({ origin = common_1.GOOGLE_TRENDS_API_HOST, path, queryParams, pathParams, agentOptions: { userAgent, proxyConfig, cookieCache } = {}, }) => {
const response = await gotScrapingTyped({
method: 'GET',
...(0, useProxyIfPossible_1.useProxyUrlIfPossible)({ proxyConfig }),
...(0, useSessionTokenForCookieCacheIfPossible_1.useSessionTokenForCookieCacheIfPossible)({ cookieCache }),
headers: {
...(0, getUserAgentHeaderIfPossible_1.getUserAgentHeadersIfPossible)({ userAgent }),
...(await (0, getCookiesHeadersFromCookieCacheIfPossible_1.getCookiesHeadersFromCookieCacheIfPossible)({ cookieCache })),
},
url: (0, url_fns_1.createUrl)({
origin,
path,
pathParams,
queryParams,
}),
throwHttpErrors: true,
});
await (0, setCookiesFromAxiosResponseIfPossible_1.setCookiesFromAxiosResponseIfPossible)({ response, cookieCache });
if (origin === common_1.GOOGLE_TRENDS_API_HOST)
return response.body.replace(/^([^{]+)/g, ''); // for somereason, resopnse starts with `)]}',\n`, so delete everything before the first `{`; NOTE: do so here, so if caching, it is saved to cache without this
return response.body;
};
exports.getFromApiCallWithoutRequestCachingOrRateLimiting = getFromApiCallWithoutRequestCachingOrRateLimiting;
const getFromApiCallWithoutRequestCaching = (args) => {
return bottleneck.schedule(() => (0, exports.getFromApiCallWithoutRequestCachingOrRateLimiting)(args));
};
exports.getFromApiCallWithoutRequestCaching = getFromApiCallWithoutRequestCaching;
const getFromApiCall = (args) => {
// if there's no request cache defined, just make the request directly
const requestCache = args.agentOptions?.requestCache;
if (!requestCache)
return (0, exports.getFromApiCallWithoutRequestCaching)(args);
// if there is a request cache, then make the request with caching
return (0, with_simple_caching_1.withSimpleCaching)(exports.getFromApiCallWithoutRequestCaching, {
cache: requestCache,
serialize: {
key: (key) => [
key[0].path.replace(/\//g, '.').replace(/:/g, '_'),
crypto_1.default.createHash('sha256').update(JSON.stringify(key)).digest('hex'),
'json',
].join('.'),
},
})(args);
};
exports.getFromApiCall = getFromApiCall;
//# sourceMappingURL=getFromApiCall.js.map