z-deduper
Version:
This library will allow you to build a Custom Zapier deduper for your advanced polling triggers use cases in your Zapier app.
111 lines • 4.14 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Storage = exports.MAX_KEYS = exports.MAX_KEY_SIZE = exports.ZAPIER_STORE_URL = void 0;
const lodash_1 = __importDefault(require("lodash"));
const axios_1 = __importDefault(require("axios"));
const utils_1 = require("./utils");
exports.ZAPIER_STORE_URL = "https://store.zapier.com";
exports.MAX_KEY_SIZE = 24500;
exports.MAX_KEYS = 495;
class Storage {
constructor(key) {
this.MAX_KEY_SIZE = exports.MAX_KEY_SIZE;
this.MAX_KEYS = exports.MAX_KEYS;
this.key = key;
this.client = axios_1.default.create({
baseURL: exports.ZAPIER_STORE_URL,
});
this.client.interceptors.response.use(undefined, (error) => {
var _a;
const errorMessage = (_a = error === null || error === void 0 ? void 0 : error.response) === null || _a === void 0 ? void 0 : _a.data;
if (errorMessage) {
error.message = error.message + "\nReason: " + errorMessage.error;
}
throw error;
});
}
async load() {
const firstPage = await this.fetchPage(0);
let total = lodash_1.default.toFinite(firstPage.total);
const pages = [firstPage];
if (total > 1) {
// fetch next pages
const promises = lodash_1.default.times(total - 1).map((i) => {
return this.fetchPage(i + 1);
});
const results = await Promise.all(promises);
pages.push(...results);
}
try {
// Parse data
let contents = lodash_1.default.flatMap(pages, (page) => page.records).join("");
contents = Buffer.from(contents, "base64").toString();
const records = JSON.parse(contents);
return records;
}
catch (error) {
error.message =
"Deduper cache seems to be corrupted\nReason: " + error.message;
throw error;
}
}
async fetchPage(page = 0) {
const response = await this.client.request({
method: "GET",
url: "/api/records",
headers: {
"X-Secret": utils_1.generateKey(`${this.key}.${page}`),
},
});
const data = response.data;
if (!data.__zapier_custom_deduper__) {
throw new Error("Invalid page. The UUID key might be used by something else.");
}
const records = Object.keys(response.data)
.filter((key) => key.startsWith("records."))
.reduce((prev, current) => {
const records = response.data[current];
return prev.concat(records);
}, []);
return {
records,
total: data.total,
};
}
async save(records) {
// Encode data
const contents = Buffer.from(JSON.stringify(records)).toString("base64");
const lines = utils_1.toFixedLengthChunks(contents, this.MAX_KEY_SIZE);
const parts = lodash_1.default.chunk(lines, this.MAX_KEYS);
let i = 0;
const pages = parts.map((part, p) => {
const page = part.reduce((prev, current) => {
prev[`records.${i++}`] = current;
return prev;
}, {});
page.total = parts.length;
page.__zapier_custom_deduper__ = true;
page.page = p;
return page;
});
// Save data
const promises = pages.map((data, page) => this.savePage(page, data));
return Promise.all(promises);
}
async savePage(page, data) {
const response = await this.client.request({
method: "POST",
url: "/api/records",
data,
headers: {
"X-Secret": utils_1.generateKey(`${this.key}.${page}`),
},
});
return response.data;
}
}
exports.Storage = Storage;
//# sourceMappingURL=storage.js.map