gphotos-scraper
Version:
A tool to extract public url and metadata from shared album
96 lines • 4.25 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.extractAlbum = void 0;
const exceptions_1 = require("../exceptions");
const getAlbum_1 = require("./getAlbum");
const getPhotos_1 = require("./getPhotos");
const extractAlbum = (albumUrl) => __awaiter(void 0, void 0, void 0, function* () {
const gAlbum = yield (0, getAlbum_1.getAlbum)(albumUrl);
if (!gAlbum)
throw new exceptions_1.AlbumNotFoundException();
const { items, key, id, title, cover, createdAt, updatedAt } = gAlbum;
// Optimized O(1) Lookup Map
const itemsMap = new Map(items.map((item) => [item.id, item.url]));
const album = {
id,
title,
url: albumUrl,
photos: [],
createdAt,
updatedAt,
};
const { default: pLimit } = yield Promise.resolve().then(() => __importStar(require("p-limit")));
const MAX_CONCURRENT_REQUESTS = 5;
const limit = pLimit(MAX_CONCURRENT_REQUESTS);
const chunkSize = 500; // Optimal batch size for Google Photos RPC
const tasks = [];
for (let i = 0; i < items.length; i += chunkSize) {
tasks.push(limit(() => __awaiter(void 0, void 0, void 0, function* () {
const chunk = items.slice(i, i + chunkSize);
const ids = chunk.map((el) => el.id);
try {
const photosInfo = yield (0, getPhotos_1.getPhotos)(key, ids);
return photosInfo.map((el) => {
const url = itemsMap.get(el.id) || "";
return Object.assign(Object.assign({}, el), { url });
});
}
catch (err) {
console.error(`Failed to fetch chunk starting at index ${i}`, err);
return []; // Fail gracefully for this chunk
}
})));
}
const results = yield Promise.all(tasks);
album.photos.push(...results.flat());
// Sorting descending by createdAt
album.photos.sort((a, b) => b.createdAt - a.createdAt);
if (cover) {
album.cover = album.photos.find((photo) => photo.url === cover.url);
}
return album;
});
exports.extractAlbum = extractAlbum;
//# sourceMappingURL=extractAlbum.js.map