gm-review-scraper
Version:
A tool to scrape Google Maps reviews
372 lines (353 loc) • 12.2 kB
JavaScript
import { hexToDec } from "hex2dec";
import { Impit } from "impit";
import { CookieJar } from "tough-cookie";
export class GMRScraper {
constructor(options = {}) {
this.url = options.url || "";
const {
version = 2,
sort_type = "relevent",
search_query = "",
pages = "max",
clean = false,
} = options;
this.version = version === 1 || version === 2 ? version : 2;
this.sort_type = sort_type;
this.search_query = search_query;
this.pages = pages;
this.clean = clean;
this.SortEnum = {
relevent: 1,
newest: 2,
highest_rating: 3,
lowest_rating: 4,
};
this.key = options.key || null;
this.client = null;
}
validateParams() {
try {
const parsedUrl = new URL(this.url);
if (!parsedUrl.host.includes("google.com")) {
throw new Error(`Invalid host: ${parsedUrl.host}`);
}
if (this.version === 1) {
if (
parsedUrl.host !== "www.google.com" ||
!parsedUrl.pathname.startsWith("/maps/place/")
) {
throw new Error(`Invalid URL for v1: ${this.url}`);
}
}
} catch (e) {
if (e instanceof TypeError)
throw new Error(`Invalid URL format: ${this.url}`);
throw e;
}
if (!this.SortEnum[this.sort_type]) {
throw new Error(
`Invalid sort type: ${this.sort_type}. Expected: ${Object.keys(this.SortEnum).join(", ")}`,
);
}
if (this.pages !== "max" && isNaN(Number(this.pages))) {
throw new Error(`Invalid pages value: ${this.pages}`);
}
if (typeof this.clean !== "boolean") {
throw new Error(`Invalid value for 'clean': ${this.clean}`);
}
}
async getUrlFromShortUrl(url) {
const resp = await fetch(url, { redirect: "manual" });
const locationHeader = resp.headers.get("location");
return locationHeader || url;
}
// ---------- Version 1: fetch API, fixed token ----------
parseReviewURL(url, p = "") {
const m = url.match(/!1s([a-zA-Z0-9_:]+)!/);
if (!m || !m[1]) throw new Error("Invalid URL");
const [h1, h2] = m[1].split(":").map(hexToDec);
const pS = p ? `!2m2!2i10!3s${p}` : `!2m1!2i10`;
return `https://www.google.com/maps/preview/review/listentitiesreviews?authuser=0&hl=en&gl=in&pb=!1m2!1y${h1}!2y${h2}${pS}!3e1!4m5!3b1!4b1!5b1!6b1!7b1!5m2!1sdzvaXrvAMImImAXHsLPICA!7e81`;
}
buildListUGCURLV1(url, so, pg = "", sq = "") {
const matches = [...url.matchAll(/!1s([a-zA-Z0-9_:]+)!/g)];
if (!matches || !matches[0][1]) throw new Error("Invalid URL");
const placeId = matches[1]?.[1] || matches[0][1];
return `https://www.google.com/maps/rpc/listugcposts?authuser=0&hl=en&gl=in&pb=!1m7!1s${placeId}!3s${sq}!6m4!4m1!1e1!4m1!1e3!2m2!1i10!2s${pg}!5m2!1sBnOwZvzePPfF4-EPy7LK0Ak!7e81!8m5!1b1!2b1!3b1!5b1!7b1!11m6!1e3!2e1!3sen!4slk!6m1!1i2!13m1!1e${so}`;
}
async fetchReviewsPageV1(pg = "", so, sq = "") {
const apiUrl = this.buildListUGCURLV1(this.url, so, pg, sq);
const resp = await fetch(apiUrl);
if (!resp.ok)
throw new Error(`Failed to fetch reviews: ${resp.statusText}`);
const text = await resp.text();
const raw = text.split(")]}'")[1];
return JSON.parse(raw);
}
async paginateV1(initialData, so) {
let reviews = initialData[2];
let nextPage = initialData[1]?.replace(/"/g, "");
let pageNum = 2;
while (nextPage && (this.pages === "max" || pageNum <= +this.pages)) {
const data = await this.fetchReviewsPageV1(
nextPage,
so,
this.search_query,
);
reviews = reviews.concat(data[2]);
nextPage = data[1]?.replace(/"/g, "") || "";
if (!nextPage) break;
await new Promise((r) => setTimeout(r, 1000));
pageNum++;
}
return reviews;
}
// ---------- Version 2: Impit + session token ----------
async initializeClient() {
if (!this.client) {
const cookieJar = new CookieJar();
this.client = new Impit({
cookieJar,
browser: "chrome",
headers: {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
Accept: "*/*",
"Accept-Language": "en-US,en;q=0.9",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
},
});
}
return this.client;
}
extractPlaceId(url) {
const m = [...url.matchAll(/!1s([a-zA-Z0-9_:]+)!/g)];
if (!m?.length || !m[0][1]) throw new Error("Invalid URL");
return m[1]?.[1] ?? m[0][1];
}
async fetchSessionToken(placeId) {
try {
const client = await this.initializeClient();
const sourceUrl = `https://maps.google.com/maps/place/${placeId}?hl=en&gl=US`;
const sourceRes = await client.fetch(sourceUrl);
const html = await sourceRes.text();
const token = html.split("var kEI='")[1]?.split("'")[0];
if (!token)
throw new Error("Could not find session token (kEI) in source.");
return token;
} catch (error) {
console.error("[-] Fetch error:", error.message);
return null;
}
}
buildListUGCURLV2(placeId, so, pg = "", sq = "", sessionToken) {
const type = sq ? "1m7" : "1m6";
const _sq = sq ? `!3s${sq}` : "";
return `https://www.google.com/maps/rpc/listugcposts?authuser=0&hl=en&gl=US&pb=!${type}!1s${placeId}${_sq}!6m4!4m1!1e1!4m1!1e3!2m2!1i10!2s${pg}!5m2!1s${sessionToken}!7e81!8m9!2b1!3b1!5b1!7b1!12m4!1b1!2b1!4m1!1e1!11m4!1e3!2e1!6m1!1i2!13m1!1e${so}`;
}
async fetchReviewsV2(
placeId,
sort,
nextPage = "",
search_query = "",
sessionToken,
) {
const client = await this.initializeClient();
const apiUrl = this.buildListUGCURLV2(
placeId,
sort,
nextPage,
search_query,
sessionToken,
);
const response = await client.fetch(apiUrl);
if (!response.ok) {
throw new Error(
`Failed to fetch: ${response.status} ${response.statusText}`,
);
}
const textData = await response.text();
const parts = textData.split(")]}'");
const rawJson = parts.length > 1 ? parts[1] : parts[0];
if (!rawJson) throw new Error("No valid JSON data found in the response.");
return JSON.parse(rawJson);
}
async paginateReviewsV2(placeId, sort, pages, search_query, sessionToken) {
const initialData = await this.fetchReviewsV2(
placeId,
sort,
"",
search_query,
sessionToken,
);
if (!initialData?.[2]?.length) return [];
let allReviews = [...(initialData[2] || [])];
let nextToken = initialData[1]?.toString().replace(/"/g, "");
if (!nextToken || Number(pages) === 1) {
return this.clean ? this.parseReviews(allReviews) : allReviews;
}
const max = pages === "max" ? Infinity : Number(pages);
let pageCount = 1;
while (nextToken && pageCount < max) {
try {
const data = await this.fetchReviewsV2(
placeId,
sort,
nextToken,
search_query,
sessionToken,
);
if (data[2]?.length) allReviews.push(...data[2]);
const newNextToken = data[1]?.toString().replace(/"/g, "");
if (!newNextToken || newNextToken === nextToken) break;
nextToken = newNextToken;
pageCount++;
await new Promise((r) => setTimeout(r, 1000));
} catch (error) {
console.error("Error fetching page:", error);
break;
}
}
return this.clean ? this.parseReviews(allReviews) : allReviews;
}
// ---------- Shared ----------
parseReviews(reviews) {
if (!Array.isArray(reviews)) return [];
const parsedReviews = reviews
.map((item) => {
const review = Array.isArray(item[0]) ? item[0] : item;
if (!review) return null;
const responseData = review[3];
const hasResponse = !!responseData?.[14]?.[0]?.[0];
return {
review_id: review[0],
time: {
published: review[1]?.[2],
last_edited: review[1]?.[3],
},
author: {
name: review[1]?.[4]?.[5]?.[0],
profile_url: review[1]?.[4]?.[5]?.[1],
url: review[1]?.[4]?.[5]?.[2]?.[0],
id: review[1]?.[4]?.[5]?.[3],
},
review: {
rating: review[2]?.[0]?.[0],
text: review[2]?.[15]?.[0]?.[0] || null,
language: review[2]?.[14]?.[0] || null,
},
images:
review[2]?.[2]?.map((image) => ({
id: image[0],
url: image[1]?.[6]?.[0],
size: {
width: image[1]?.[6]?.[2]?.[0],
height: image[1]?.[6]?.[2]?.[1],
},
location: {
friendly: image[1]?.[21]?.[3]?.[7]?.[0],
lat: image[1]?.[8]?.[0]?.[2],
long: image[1]?.[8]?.[0]?.[1],
},
caption: image[1]?.[21]?.[3]?.[5]?.[0] || null,
})) || null,
source: review[1]?.[13]?.[0],
response: hasResponse
? {
text: responseData[14][0][0] || null,
time: {
published: responseData[1] || null,
last_edited: responseData[2] || null,
},
}
: null,
};
})
.filter((r) => r !== null);
return parsedReviews;
}
async fetchStatistical(url) {
try {
const [lat, lng] = url.split("/@")[1].split("/")[0].split(",");
const input = url.split("place/")[1].split("/")[0];
const apiUrl = `https://maps.googleapis.com/maps/api/place/findplacefromtext/json?input=${input}&inputtype=textquery&locationbias=point:${lat},${lng}&fields=place_id,name,rating,user_ratings_total,formatted_address&key=${this.key}`;
const resp = await fetch(apiUrl);
if (!resp.ok) {
return {
rating: 0,
total_reviews: 0,
location: { lat: 0, lng: 0 },
name: "",
address: "",
};
}
const data = await resp.json();
const result = data.candidates?.[0];
return {
rating: result?.rating ?? 0,
total_reviews: result?.user_ratings_total ?? 0,
location: { lat, lng },
name: result?.name ?? input ?? "",
address: result?.formatted_address ?? "",
};
} catch {
return {
rating: 0,
total_reviews: 0,
location: { lat: 0, lng: 0 },
name: "",
address: "",
};
}
}
async scrape(url) {
this.url = url;
if (this.url.includes("maps.app.goo.gl")) {
this.url = await this.getUrlFromShortUrl(this.url);
}
this.validateParams();
const so = this.SortEnum[this.sort_type];
let statistical = {
rating: 0,
total_reviews: 0,
name: "",
address: "",
location: { lat: 0, lng: 0 },
};
if (this.key) {
statistical = await this.fetchStatistical(this.url);
}
if (this.version === 1) {
if (this.pages === 0) {
return { reviews: [], statistical };
}
const initial = await this.fetchReviewsPageV1("", so, this.search_query);
if (!initial?.[2]?.length) return { reviews: [], statistical };
if (!initial[1] || this.pages === 1) {
const reviews = this.clean ? this.parseReviews(initial[2]) : initial[2];
return { reviews, statistical };
}
const allReviews = await this.paginateV1(initial, so);
const reviews = this.clean ? this.parseReviews(allReviews) : allReviews;
return { reviews, statistical };
}
// version === 2
const placeId = this.extractPlaceId(this.url);
const sessionToken = await this.fetchSessionToken(placeId);
if (!sessionToken) throw new Error("Could not fetch session token.");
await new Promise((r) => setTimeout(r, 2000));
const reviews = await this.paginateReviewsV2(
placeId,
so,
this.pages,
this.search_query,
sessionToken,
);
return {
reviews: Array.isArray(reviews) ? reviews : [],
statistical,
};
}
}