chromium-helper
Version:
CLI tool for searching and exploring Chromium source code via Google's official APIs
1,166 lines (1,165 loc) • 175 kB
JavaScript
import fetch from 'node-fetch';
import { chromium } from 'playwright';
export class ChromiumSearchError extends Error {
cause;
constructor(message, cause) {
super(message);
this.cause = cause;
this.name = 'ChromiumSearchError';
}
}
export class GerritAPIError extends Error {
statusCode;
cause;
constructor(message, statusCode, cause) {
super(message);
this.statusCode = statusCode;
this.cause = cause;
this.name = 'GerritAPIError';
}
}
export class ChromiumAPI {
apiKey;
cache = new Map();
debugMode = false;
constructor(apiKey) {
this.apiKey = apiKey || process.env.CHROMIUM_SEARCH_API_KEY || 'AIzaSyCqPSptx9mClE5NU4cpfzr6cgdO_phV1lM';
}
setDebugMode(enabled) {
this.debugMode = enabled;
}
debug(...args) {
if (this.debugMode) {
console.log(...args);
}
}
async searchCode(params) {
const { query, caseSensitive = false, language, filePattern, searchType, excludeComments = false, limit = 20 } = params;
// Build the enhanced search query using Code Search syntax
let searchQuery = query;
// Add case sensitivity if requested
if (caseSensitive) {
searchQuery = `case:yes ${searchQuery}`;
}
// Add language filter if specified
if (language) {
searchQuery = `lang:${language} ${searchQuery}`;
}
// Add file pattern filter if specified
if (filePattern) {
searchQuery = `file:${filePattern} ${searchQuery}`;
}
// Add search type filter if specified
if (searchType) {
switch (searchType) {
case 'content':
searchQuery = `content:${query}`;
break;
case 'function':
searchQuery = `function:${query}`;
break;
case 'class':
searchQuery = `class:${query}`;
break;
case 'symbol':
searchQuery = `symbol:${query}`;
break;
case 'comment':
searchQuery = `comment:${query}`;
break;
}
// Apply other filters to the type-specific query
if (caseSensitive)
searchQuery = `case:yes ${searchQuery}`;
if (language)
searchQuery = `lang:${language} ${searchQuery}`;
if (filePattern)
searchQuery = `file:${filePattern} ${searchQuery}`;
}
// Add usage filter to exclude comments if requested
if (excludeComments && !searchType) {
searchQuery = `usage:${query}`;
if (caseSensitive)
searchQuery = `case:yes ${searchQuery}`;
if (language)
searchQuery = `lang:${language} ${searchQuery}`;
if (filePattern)
searchQuery = `file:${filePattern} ${searchQuery}`;
}
try {
const response = await this.callChromiumSearchAPI(searchQuery, limit);
return this.parseChromiumAPIResponse(response);
}
catch (error) {
throw new ChromiumSearchError(`Search failed: ${error.message}`, error);
}
}
async findSymbol(symbol, filePath) {
try {
// Search for symbol definitions using Code Search syntax
const symbolResults = await this.callChromiumSearchAPI(`symbol:${symbol}`, 10);
const symbolParsed = this.parseChromiumAPIResponse(symbolResults);
// Search for class definitions
const classResults = await this.callChromiumSearchAPI(`class:${symbol}`, 5);
const classParsed = this.parseChromiumAPIResponse(classResults);
// Search for function definitions
const functionResults = await this.callChromiumSearchAPI(`function:${symbol}`, 5);
const functionParsed = this.parseChromiumAPIResponse(functionResults);
// Search for general usage in content (excluding comments)
const usageResults = await this.callChromiumSearchAPI(`usage:${symbol}`, 10);
const usageParsed = this.parseChromiumAPIResponse(usageResults);
return {
symbol,
symbolResults: symbolParsed,
classResults: classParsed,
functionResults: functionParsed,
usageResults: usageParsed,
estimatedUsageCount: usageResults.estimatedResultCount
};
}
catch (error) {
throw new ChromiumSearchError(`Symbol lookup failed: ${error.message}`, error);
}
}
async getFile(params) {
const { filePath, lineStart, lineEnd } = params;
try {
// Check if this is a submodule file
if (filePath.startsWith('v8/')) {
return await this.getV8FileViaGitHub(filePath, lineStart, lineEnd);
}
if (filePath.startsWith('third_party/webrtc/')) {
return await this.getWebRTCFileViaGitiles(filePath, lineStart, lineEnd);
}
// Fetch from Gitiles API
const gitileUrl = `https://chromium.googlesource.com/chromium/src/+/main/${filePath}?format=TEXT`;
const response = await fetch(gitileUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
if (!response.ok) {
throw new Error(`Failed to fetch file: HTTP ${response.status}`);
}
// The response is base64 encoded
const base64Content = await response.text();
const fileContent = Buffer.from(base64Content, 'base64').toString('utf-8');
// Split into lines for line number processing
const lines = fileContent.split('\n');
let displayLines = lines;
let startLine = 1;
// Apply line range if specified
if (lineStart) {
const start = Math.max(1, lineStart) - 1; // Convert to 0-based
const end = lineEnd ? Math.min(lines.length, lineEnd) : lines.length;
displayLines = lines.slice(start, end);
startLine = start + 1;
}
// Format content with line numbers
const numberedLines = displayLines.map((line, index) => {
const lineNum = (startLine + index).toString().padStart(4, ' ');
return `${lineNum} ${line}`;
}).join('\n');
// Create browser URL for reference
let browserUrl = `https://source.chromium.org/chromium/chromium/src/+/main:${filePath}`;
if (lineStart) {
browserUrl += `;l=${lineStart}`;
if (lineEnd) {
browserUrl += `-${lineEnd}`;
}
}
return {
filePath,
content: numberedLines,
totalLines: lines.length,
displayedLines: displayLines.length,
lineStart,
lineEnd,
browserUrl
};
}
catch (error) {
throw new ChromiumSearchError(`File fetch failed: ${error.message}`, error);
}
}
async getGerritCLStatus(clNumber) {
try {
// Extract CL number from URL if needed
const clNum = this.extractCLNumber(clNumber);
const gerritUrl = `https://chromium-review.googlesource.com/changes/${clNum}?o=CURRENT_REVISION&o=DETAILED_ACCOUNTS&o=SUBMIT_REQUIREMENTS&o=CURRENT_COMMIT`;
const response = await fetch(gerritUrl);
if (!response.ok) {
throw new GerritAPIError(`Failed to fetch CL status: ${response.status}`, response.status);
}
const text = await response.text();
// Remove XSSI protection prefix
const jsonText = text.replace(/^\)\]\}'\n/, '');
return JSON.parse(jsonText);
}
catch (error) {
throw new GerritAPIError(`Gerrit API error: ${error.message}`, undefined, error);
}
}
async getGerritCLComments(params) {
try {
const clNum = this.extractCLNumber(params.clNumber);
const gerritUrl = `https://chromium-review.googlesource.com/changes/${clNum}/comments`;
const response = await fetch(gerritUrl);
if (!response.ok) {
throw new GerritAPIError(`Failed to fetch CL comments: ${response.status}`, response.status);
}
const text = await response.text();
const jsonText = text.replace(/^\)\]\}'\n/, '');
return JSON.parse(jsonText);
}
catch (error) {
throw new GerritAPIError(`Gerrit comments API error: ${error.message}`, undefined, error);
}
}
async getGerritCLDiff(params) {
const clId = this.extractCLNumber(params.clNumber);
try {
// First get CL details to know current patchset if not specified
const clDetailsUrl = `https://chromium-review.googlesource.com/changes/?q=change:${clId}&o=CURRENT_REVISION`;
const clResponse = await fetch(clDetailsUrl, {
headers: {
'Accept': 'application/json',
},
});
if (!clResponse.ok) {
throw new Error(`Failed to fetch CL details: ${clResponse.status}`);
}
const responseText = await clResponse.text();
const jsonText = responseText.replace(/^\)\]\}'\n/, '');
const clData = JSON.parse(jsonText);
if (!clData || clData.length === 0) {
throw new Error(`CL ${clId} not found`);
}
const cl = clData[0];
const targetPatchset = params.patchset || cl.current_revision_number || 1;
const revision = cl.revisions[cl.current_revision];
// Get the files list first to understand what changed
const filesUrl = `https://chromium-review.googlesource.com/changes/${clId}/revisions/${targetPatchset}/files`;
const filesResponse = await fetch(filesUrl, {
headers: {
'Accept': 'application/json',
},
});
if (!filesResponse.ok) {
throw new Error(`Failed to fetch files: ${filesResponse.status}`);
}
const filesText = await filesResponse.text();
const filesJsonText = filesText.replace(/^\)\]\}'\n/, '');
const filesData = JSON.parse(filesJsonText);
const changedFiles = Object.keys(filesData).filter(f => f !== '/COMMIT_MSG');
const result = {
clId,
subject: cl.subject,
patchset: targetPatchset,
author: cl.owner.name,
changedFiles,
filesData,
revision
};
if (params.filePath) {
// Get diff for specific file
if (!filesData[params.filePath]) {
result.error = `File ${params.filePath} not found in patchset ${targetPatchset}`;
return result;
}
const diffUrl = `https://chromium-review.googlesource.com/changes/${clId}/revisions/${targetPatchset}/files/${encodeURIComponent(params.filePath)}/diff?base=${targetPatchset - 1}&context=ALL&intraline`;
const diffResponse = await fetch(diffUrl, {
headers: {
'Accept': 'application/json',
},
});
if (diffResponse.ok) {
const diffText = await diffResponse.text();
const diffJsonText = diffText.replace(/^\)\]\}'\n/, '');
result.diffData = JSON.parse(diffJsonText);
}
}
return result;
}
catch (error) {
throw new GerritAPIError(`Failed to get CL diff: ${error.message}`, undefined, error);
}
}
async getGerritPatchsetFile(params) {
const clId = this.extractCLNumber(params.clNumber);
try {
// First get CL details to know current patchset if not specified
const clDetailsUrl = `https://chromium-review.googlesource.com/changes/?q=change:${clId}&o=CURRENT_REVISION`;
const clResponse = await fetch(clDetailsUrl, {
headers: {
'Accept': 'application/json',
},
});
if (!clResponse.ok) {
throw new Error(`Failed to fetch CL details: ${clResponse.status}`);
}
const responseText = await clResponse.text();
const jsonText = responseText.replace(/^\)\]\}'\n/, '');
const clData = JSON.parse(jsonText);
if (!clData || clData.length === 0) {
throw new Error(`CL ${clId} not found`);
}
const cl = clData[0];
const targetPatchset = params.patchset || cl.current_revision_number || 1;
// Get the file content from the patchset
const fileUrl = `https://chromium-review.googlesource.com/changes/${clId}/revisions/${targetPatchset}/files/${encodeURIComponent(params.filePath)}/content`;
const fileResponse = await fetch(fileUrl, {
headers: {
'Accept': 'text/plain',
},
});
if (!fileResponse.ok) {
if (fileResponse.status === 404) {
throw new Error(`File ${params.filePath} not found in patchset ${targetPatchset}`);
}
throw new Error(`Failed to fetch file content: ${fileResponse.status}`);
}
// Gerrit returns base64 encoded content
const base64Content = await fileResponse.text();
const content = Buffer.from(base64Content, 'base64').toString('utf-8');
return {
clId,
subject: cl.subject,
patchset: targetPatchset,
author: cl.owner.name,
filePath: params.filePath,
content,
lines: content.split('\n').length
};
}
catch (error) {
throw new GerritAPIError(`Failed to get file content: ${error.message}`, undefined, error);
}
}
async getGerritCLTrybotStatus(params) {
const clId = this.extractCLNumber(params.clNumber);
try {
// Get CL messages to find LUCI Change Verifier URLs
const messagesUrl = `https://chromium-review.googlesource.com/changes/${clId}/messages`;
const response = await fetch(messagesUrl, {
headers: {
'Accept': 'application/json',
},
});
if (!response.ok) {
throw new Error(`Failed to fetch messages: ${response.status}`);
}
const text = await response.text();
const jsonText = text.replace(/^\)\]\}'\n/, '');
const messages = JSON.parse(jsonText);
// Find LUCI Change Verifier URLs from messages
const luciUrls = this.extractLuciVerifierUrls(messages, params.patchset);
if (luciUrls.length === 0) {
return {
clId,
patchset: params.patchset || 'latest',
totalBots: 0,
failedBots: 0,
passedBots: 0,
runningBots: 0,
bots: [],
message: 'No LUCI runs found for this CL'
};
}
// Get detailed bot status from the most recent LUCI run
const latestLuciUrl = luciUrls[0];
const detailedBots = await this.fetchLuciRunDetails(latestLuciUrl.url);
// Filter by failed only if requested
const filteredBots = params.failedOnly
? detailedBots.filter(bot => bot.status === 'FAILED')
: detailedBots;
return {
clId,
patchset: latestLuciUrl.patchset,
runId: latestLuciUrl.runId,
luciUrl: latestLuciUrl.url,
totalBots: detailedBots.length,
failedBots: detailedBots.filter(bot => bot.status === 'FAILED').length,
passedBots: detailedBots.filter(bot => bot.status === 'PASSED').length,
runningBots: detailedBots.filter(bot => bot.status === 'RUNNING').length,
canceledBots: detailedBots.filter(bot => bot.status === 'CANCELED').length,
bots: filteredBots,
timestamp: latestLuciUrl.timestamp
};
}
catch (error) {
throw new GerritAPIError(`Failed to get trybot status: ${error.message}`, undefined, error);
}
}
extractLuciVerifierUrls(messages, targetPatchset) {
const luciUrls = [];
for (const msg of messages) {
// Skip if we want a specific patchset and this message is for a different one
if (targetPatchset && msg._revision_number && msg._revision_number !== targetPatchset) {
continue;
}
// Look for LUCI Change Verifier URLs in messages
if (msg.message) {
const luciMatch = msg.message.match(/Follow status at: (https:\/\/luci-change-verifier\.appspot\.com\/ui\/run\/chromium\/([^\/\s]+))/);
if (luciMatch) {
luciUrls.push({
url: luciMatch[1],
runId: luciMatch[2],
patchset: msg._revision_number || 0,
timestamp: msg.date
});
}
}
}
// Return most recent first
return luciUrls.sort((a, b) => new Date(b.timestamp).getTime() - new Date(a.timestamp).getTime());
}
async fetchLuciRunDetails(luciUrl) {
try {
// Extract run ID from URL (works for both chromium and pdfium)
const runIdMatch = luciUrl.match(/\/run\/(?:chromium|pdfium)\/([^\/\s]+)/);
if (!runIdMatch) {
throw new Error('Could not extract run ID from LUCI URL');
}
const runId = runIdMatch[1];
// Fetch the LUCI page HTML directly
const response = await fetch(luciUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
},
});
if (!response.ok) {
throw new Error(`Failed to fetch LUCI page: ${response.status}`);
}
const html = await response.text();
this.debug(`[DEBUG] Fetched LUCI HTML, length: ${html.length}`);
// Parse the HTML to extract bot information
const bots = this.parseLuciHtmlImproved(html, luciUrl, runId);
this.debug(`[DEBUG] Found ${bots.length} bots from LUCI page`);
if (bots.length > 0) {
return bots;
}
// Fallback if parsing fails
return [{
name: 'LUCI Run',
status: 'UNKNOWN',
runId: runId,
luciUrl: luciUrl,
summary: 'Could not parse bot details - view at LUCI URL'
}];
}
catch (error) {
this.debug(`[DEBUG] Failed to fetch LUCI details: ${error}`);
// Fallback to basic info if we can't fetch details
return [{
name: 'LUCI Run',
status: 'UNKNOWN',
luciUrl: luciUrl,
summary: 'View detailed bot status at LUCI URL'
}];
}
}
parseLuciHtmlImproved(html, luciUrl, runId) {
const bots = [];
const foundBots = new Set();
try {
// Simple approach: Find all <a> elements with tryjob-chip classes
// This works for both Chromium and PDFium without hardcoding patterns
const tryjobPattern = /<a[^>]*class="[^"]*tryjob-chip[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gs;
let match;
while ((match = tryjobPattern.exec(html)) !== null) {
const fullMatch = match[0];
const href = match[1];
const innerText = match[2];
// Extract bot name from the inner text (trim whitespace)
const botName = innerText.trim().replace(/\s+/g, ' ');
if (botName && botName.length > 3 && !foundBots.has(botName)) {
foundBots.add(botName);
// Extract status from class attribute
let status = 'UNKNOWN';
if (fullMatch.includes('tryjob-chip passed')) {
status = 'PASSED';
}
else if (fullMatch.includes('tryjob-chip failed')) {
status = 'FAILED';
}
else if (fullMatch.includes('tryjob-chip running')) {
status = 'RUNNING';
}
else if (fullMatch.includes('tryjob-chip canceled')) {
status = 'CANCELED';
}
// Construct the full CI build URL if we have a relative path
let buildUrl = href;
if (href && href.startsWith('/')) {
buildUrl = `https://ci.chromium.org${href}`;
}
bots.push({
name: botName,
status: status,
luciUrl: luciUrl,
runId: runId,
buildUrl: buildUrl,
summary: `${status.toLowerCase()}`
});
}
}
this.debug(`[DEBUG] Parsed ${bots.length} bots from HTML`);
this.debug(`[DEBUG] Status breakdown: ${JSON.stringify(this.getStatusCounts(bots))}`);
}
catch (error) {
this.debug(`[DEBUG] Error parsing LUCI HTML: ${error}`);
}
return bots;
}
extractBotStatus(html, matchIndex) {
// Look for status indicators around this bot
const contextStart = Math.max(0, matchIndex - 500);
const contextEnd = Math.min(html.length, matchIndex + 500);
const context = html.slice(contextStart, contextEnd);
let status = 'UNKNOWN';
// Look for various status patterns in the surrounding context
if (this.checkForStatus(context, 'SUCCESS', 'PASSED', 'success')) {
status = 'PASSED';
}
else if (this.checkForStatus(context, 'FAILURE', 'FAILED', 'failure', 'error')) {
status = 'FAILED';
}
else if (this.checkForStatus(context, 'RUNNING', 'STARTED', 'running', 'pending')) {
status = 'RUNNING';
}
else if (this.checkForStatus(context, 'CANCELED', 'CANCELLED', 'canceled')) {
status = 'CANCELED';
}
// Also check for CSS class patterns that indicate status
if (status === 'UNKNOWN') {
if (context.includes('class="green"') || context.includes('color:green') ||
context.includes('background-color:green') || context.includes('rgb(76, 175, 80)')) {
status = 'PASSED';
}
else if (context.includes('class="red"') || context.includes('color:red') ||
context.includes('background-color:red') || context.includes('rgb(244, 67, 54)')) {
status = 'FAILED';
}
else if (context.includes('class="yellow"') || context.includes('color:orange') ||
context.includes('background-color:yellow') || context.includes('rgb(255, 193, 7)')) {
status = 'RUNNING';
}
}
return status;
}
checkForStatus(context, ...statusWords) {
const lowerContext = context.toLowerCase();
return statusWords.some(word => lowerContext.includes(word.toLowerCase()));
}
getStatusCounts(bots) {
const counts = {};
bots.forEach(bot => {
counts[bot.status] = (counts[bot.status] || 0) + 1;
});
return counts;
}
async findOwners(filePath) {
try {
const ownerFiles = [];
const pathParts = filePath.split('/');
// Search up the directory tree for OWNERS files
for (let i = pathParts.length; i > 0; i--) {
const dirPath = pathParts.slice(0, i).join('/');
const ownersPath = dirPath ? `${dirPath}/OWNERS` : 'OWNERS';
try {
const result = await this.getFile({ filePath: ownersPath });
ownerFiles.push({
path: ownersPath,
content: result.content,
browserUrl: result.browserUrl
});
}
catch (error) {
// OWNERS file doesn't exist at this level, continue up the tree
}
}
return {
filePath,
ownerFiles
};
}
catch (error) {
throw new ChromiumSearchError(`Owners lookup failed: ${error.message}`, error);
}
}
async searchCommits(params) {
try {
let gitileUrl = `https://chromium.googlesource.com/chromium/src/+log/?format=JSON&n=${params.limit || 20}`;
if (params.since) {
gitileUrl += `&since=${params.since}`;
}
if (params.until) {
gitileUrl += `&until=${params.until}`;
}
if (params.author) {
gitileUrl += `&author=${encodeURIComponent(params.author)}`;
}
const response = await fetch(gitileUrl);
if (!response.ok) {
throw new Error(`Failed to fetch commits: HTTP ${response.status}`);
}
const text = await response.text();
const jsonText = text.replace(/^\)\]\}'\n/, '');
const result = JSON.parse(jsonText);
// Filter by query if provided
if (params.query) {
const query = params.query.toLowerCase();
result.log = result.log.filter((commit) => commit.message.toLowerCase().includes(query) ||
commit.author.name.toLowerCase().includes(query) ||
commit.author.email.toLowerCase().includes(query));
}
return result;
}
catch (error) {
throw new ChromiumSearchError(`Commit search failed: ${error.message}`, error);
}
}
async getIssue(issueId) {
try {
const issueNum = this.extractIssueId(issueId);
const issueUrl = `https://issues.chromium.org/issues/${issueNum}`;
// Try direct API approach first (much faster than Playwright)
try {
const directApiResult = await this.getIssueDirectAPI(issueNum);
if (directApiResult && (directApiResult.comments?.length > 0 || directApiResult.description?.length > 20)) {
return {
issueId: issueNum,
browserUrl: issueUrl,
...directApiResult,
extractionMethod: 'direct-api'
};
}
else {
this.debug(`[DEBUG] Direct API returned insufficient data, falling back to browser`);
}
}
catch (error) {
this.debug(`[DEBUG] Direct API failed, falling back to browser: ${error}`);
}
// First try HTTP-based extraction for basic info
let basicInfo = null;
try {
const response = await fetch(issueUrl, {
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
},
});
if (response.ok) {
const html = await response.text();
const jspbMatch = html.match(/defrostedResourcesJspb\s*=\s*(\[.*?\]);/s);
if (jspbMatch) {
try {
const issueData = JSON.parse(jspbMatch[1]);
basicInfo = this.extractIssueInfo(issueData, issueNum);
}
catch (e) {
// Continue to browser automation
}
}
}
}
catch (e) {
// Continue to browser automation
}
// Use browser automation for comprehensive data extraction
const browserInfo = await this.extractIssueWithBrowser(issueUrl, issueNum);
// Merge basic info with browser-extracted info
const mergedInfo = {
...basicInfo,
...browserInfo,
// Prefer browser-extracted title if it's more meaningful
title: (browserInfo.title && browserInfo.title !== 'Unknown' && !browserInfo.title.includes('Issue '))
? browserInfo.title
: basicInfo?.title || browserInfo.title,
extractionMethod: 'browser-automation'
};
return {
issueId: issueNum,
browserUrl: issueUrl,
...mergedInfo
};
}
catch (error) {
const browserUrl = `https://issues.chromium.org/issues/${this.extractIssueId(issueId)}`;
return {
issueId: this.extractIssueId(issueId),
browserUrl,
error: `Failed to fetch issue details: ${error.message}`,
message: 'Use the browser URL to view the issue manually.'
};
}
}
async callChromiumSearchAPI(query, limit) {
const searchPayload = {
queryString: query,
searchOptions: {
enableDiagnostics: false,
exhaustive: false,
numberOfContextLines: 1,
pageSize: Math.min(limit, 25),
pageToken: "",
pathPrefix: "",
repositoryScope: {
root: {
ossProject: "chromium",
repositoryName: "chromium/src"
}
},
retrieveMultibranchResults: true,
savedQuery: "",
scoringModel: "",
showPersonalizedResults: false,
suppressGitLegacyResults: false
},
snippetOptions: {
minSnippetLinesPerFile: 10,
minSnippetLinesPerPage: 60,
numberOfContextLines: 1
}
};
// Generate a boundary for multipart request
const boundary = `batch${Date.now()}${Math.random().toString().substr(2)}`;
// Create the multipart body exactly like the working curl
const multipartBody = [
`--${boundary}`,
'Content-Type: application/http',
`Content-ID: <response-${boundary}+gapiRequest@googleapis.com>`,
'',
`POST /v1/contents/search?alt=json&key=${this.apiKey}`,
'sessionid: ' + Math.random().toString(36).substr(2, 12),
'actionid: ' + Math.random().toString(36).substr(2, 12),
'X-JavaScript-User-Agent: google-api-javascript-client/1.1.0',
'X-Requested-With: XMLHttpRequest',
'Content-Type: application/json',
'X-Goog-Encode-Response-If-Executable: base64',
'',
JSON.stringify(searchPayload),
`--${boundary}--`,
''
].join('\r\n');
const response = await fetch(`https://grimoireoss-pa.clients6.google.com/batch?%24ct=multipart%2Fmixed%3B%20boundary%3D${boundary}`, {
method: 'POST',
headers: {
'accept': '*/*',
'accept-language': 'en-US,en;q=0.9',
'cache-control': 'no-cache',
'content-type': 'text/plain; charset=UTF-8',
'origin': 'https://source.chromium.org',
'pragma': 'no-cache',
'referer': 'https://source.chromium.org/',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'cross-site',
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36'
},
body: multipartBody
});
if (!response.ok) {
throw new ChromiumSearchError(`API request failed: ${response.status} ${response.statusText}`);
}
const responseText = await response.text();
// Parse the multipart response to extract JSON
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
if (!jsonMatch) {
throw new ChromiumSearchError('Could not parse API response');
}
const result = JSON.parse(jsonMatch[0]);
return result;
}
parseChromiumAPIResponse(apiResponse) {
const results = [];
if (!apiResponse.searchResults) {
return results;
}
for (const searchResult of apiResponse.searchResults) {
const fileResult = searchResult.fileSearchResult;
if (!fileResult)
continue;
const filePath = fileResult.fileSpec.path;
for (const snippet of fileResult.snippets || []) {
const snippetLines = snippet.snippetLines || [];
// Group lines by snippet for better context
if (snippetLines.length > 0) {
// Find the primary match line (the one with ranges)
const matchLines = snippetLines.filter((line) => line.ranges && line.ranges.length > 0);
if (matchLines.length > 0) {
// Use the first match line as the primary result
const primaryMatch = matchLines[0];
const lineNumber = parseInt(primaryMatch.lineNumber) || 0;
const url = `https://source.chromium.org/chromium/chromium/src/+/main:${filePath};l=${lineNumber}`;
// Build context with all lines in this snippet
const contextLines = snippetLines.map((line) => {
const lineText = line.lineText || '';
const hasMatch = line.ranges && line.ranges.length > 0;
return hasMatch ? `➤ ${lineText}` : ` ${lineText}`;
}).join('\n');
results.push({
file: filePath,
line: lineNumber,
content: contextLines,
url: url,
});
}
}
}
}
return results;
}
extractCLNumber(clInput) {
// Extract CL number from URL or return as-is if already a number
const match = clInput.match(/\/(\d+)(?:\/|$)/);
return match ? match[1] : clInput;
}
extractIssueId(issueInput) {
// Extract issue ID from URL or return as-is if already a number
const match = issueInput.match(/\/issues\/(\d+)/);
return match ? match[1] : issueInput;
}
getFileExtension(filePath) {
const ext = filePath.split('.').pop()?.toLowerCase() || '';
const extensionMap = {
'cc': 'cpp',
'cpp': 'cpp',
'cxx': 'cpp',
'h': 'cpp',
'hpp': 'cpp',
'js': 'javascript',
'ts': 'typescript',
'py': 'python',
'java': 'java',
'go': 'go',
'rs': 'rust',
'sh': 'bash',
'md': 'markdown',
'json': 'json',
'xml': 'xml',
'html': 'html',
'css': 'css',
'yml': 'yaml',
'yaml': 'yaml',
};
return extensionMap[ext] || '';
}
extractIssueInfo(issueData, issueId) {
try {
// The structure can vary, so we need to search through it
let issueArray = null;
// Try different common structures
if (issueData?.[1]?.[0]) {
issueArray = issueData[1][0];
}
else if (issueData?.[0]?.[1]?.[0]) {
issueArray = issueData[0][1][0];
}
else if (Array.isArray(issueData)) {
// Search for the issue array in the nested structure
for (const item of issueData) {
if (Array.isArray(item)) {
for (const subItem of item) {
if (Array.isArray(subItem) && subItem.length > 5) {
issueArray = subItem;
break;
}
}
if (issueArray)
break;
}
}
}
if (!issueArray) {
// Try to extract basic info from the raw data string
const dataStr = JSON.stringify(issueData);
const titleMatch = dataStr.match(/"([^"]{10,200})"/);
return {
title: titleMatch ? titleMatch[1] : 'Issue data found but structure unknown',
status: 'Unknown',
priority: 'Unknown',
type: 'Unknown',
severity: 'Unknown',
reporter: 'Unknown',
assignee: 'Unknown',
created: 'Unknown',
modified: 'Unknown',
relatedCLs: this.extractRelatedCLsFromString(dataStr)
};
}
// Extract basic issue information
const title = issueArray[1] || issueArray[0] || 'No title';
const status = this.getStatusText(issueArray[2]?.[0] || issueArray[2]);
const priority = this.getPriorityText(issueArray[3]?.[0] || issueArray[3]);
const type = this.getTypeText(issueArray[4]?.[0] || issueArray[4]);
const severity = this.getSeverityText(issueArray[5]?.[0] || issueArray[5]);
// Extract timestamps
const created = this.formatTimestamp(issueArray[8] || issueArray[6]);
const modified = this.formatTimestamp(issueArray[9] || issueArray[7]);
// Extract reporter and assignee
const reporter = this.extractUserInfo(issueArray[6] || issueArray[10]);
const assignee = this.extractUserInfo(issueArray[7] || issueArray[11]);
// Look for related CLs in the issue data
const relatedCLs = this.extractRelatedCLs(issueArray);
return {
title,
status,
priority,
type,
severity,
reporter,
assignee,
created,
modified,
relatedCLs
};
}
catch (error) {
return {
title: 'Unknown',
status: 'Unknown',
error: `Failed to parse issue data: ${error instanceof Error ? error.message : String(error)}`
};
}
}
getStatusText(status) {
const statusMap = {
1: 'NEW',
2: 'ASSIGNED',
3: 'ACCEPTED',
4: 'FIXED',
5: 'VERIFIED',
6: 'INVALID',
7: 'WONTFIX',
8: 'DUPLICATE',
9: 'ARCHIVED'
};
return statusMap[status] || `Status ${status}`;
}
getPriorityText(priority) {
const priorityMap = {
0: 'P0',
1: 'P1',
2: 'P2',
3: 'P3',
4: 'P4'
};
return priorityMap[priority] || `Priority ${priority}`;
}
getTypeText(type) {
const typeMap = {
1: 'Bug',
2: 'Feature',
3: 'Task'
};
return typeMap[type] || `Type ${type}`;
}
getSeverityText(severity) {
const severityMap = {
0: 'S0',
1: 'S1',
2: 'S2',
3: 'S3',
4: 'S4'
};
return severityMap[severity] || `Severity ${severity}`;
}
extractUserInfo(userArray) {
if (!userArray || !Array.isArray(userArray)) {
return 'Unknown';
}
// User info is typically in the first element as an email
return userArray[0] || 'Unknown';
}
formatTimestamp(timestampArray) {
if (!timestampArray || !Array.isArray(timestampArray)) {
return 'Unknown';
}
// Timestamp format: [seconds, nanoseconds]
const seconds = timestampArray[0];
if (typeof seconds === 'number') {
return new Date(seconds * 1000).toISOString();
}
return 'Unknown';
}
extractRelatedCLs(issueArray) {
return this.extractRelatedCLsFromString(JSON.stringify(issueArray));
}
extractRelatedCLsFromString(str) {
const cls = [];
// Look through the data for CL references
const clMatches = str.match(/chromium-review\.googlesource\.com\/c\/chromium\/src\/\+\/(\d+)/g);
if (clMatches) {
clMatches.forEach(match => {
const clNumber = match.match(/\/(\d+)$/)?.[1];
if (clNumber && !cls.includes(clNumber)) {
cls.push(clNumber);
}
});
}
// Also look for simple CL number patterns
const clNumberMatches = str.match(/CL[\s\-\#]*(\d{6,})/gi);
if (clNumberMatches) {
clNumberMatches.forEach(match => {
const clNumber = match.match(/(\d{6,})/)?.[1];
if (clNumber && !cls.includes(clNumber)) {
cls.push(clNumber);
}
});
}
return cls;
}
extractIssueFromHTML(html, issueId) {
// Simple HTML extraction as fallback
let title = `Issue ${issueId}`;
let status = 'Unknown';
// Try to extract title from page title, avoiding common false positives
const titleMatch = html.match(/<title[^>]*>([^<]+)</i);
if (titleMatch) {
const rawTitle = titleMatch[1].replace(/\s*-\s*Chromium\s*$/i, '').trim();
// Filter out obvious data structure names
if (rawTitle &&
!rawTitle.includes('IssueFetchResponse') &&
!rawTitle.includes('undefined') &&
!rawTitle.includes('null') &&
rawTitle.length > 5) {
title = rawTitle;
}
}
// Try multiple approaches to extract meaningful data
// Look for metadata in script tags
const scriptMatches = html.match(/<script[^>]*>(.*?)<\/script>/gis);
if (scriptMatches) {
for (const script of scriptMatches) {
// Look for various data patterns
const summaryMatch = script.match(/"summary"[^"]*"([^"]{10,})/i);
if (summaryMatch && !summaryMatch[1].includes('b.IssueFetchResponse')) {
title = summaryMatch[1];
break;
}
const titleMatch = script.match(/"title"[^"]*"([^"]{10,})/i);
if (titleMatch && !titleMatch[1].includes('b.IssueFetchResponse')) {
title = titleMatch[1];
break;
}
}
}
// Try to extract status from common patterns
const statusMatches = [
/"state"[^"]*"([^"]+)"/i,
/"status"[^"]*"([^"]+)"/i,
/status[^a-zA-Z]*([A-Z][A-Za-z]+)/i,
/Status:\s*([A-Z][A-Za-z]+)/i
];
for (const pattern of statusMatches) {
const match = html.match(pattern);
if (match && match[1] !== 'Unknown') {
status = match[1];
break;
}
}
// Extract related CLs from HTML
const relatedCLs = this.extractRelatedCLsFromString(html);
return {
title,
status,
priority: 'Unknown',
type: 'Unknown',
severity: 'Unknown',
reporter: 'Unknown',
assignee: 'Unknown',
created: 'Unknown',
modified: 'Unknown',
relatedCLs,
note: 'Basic extraction from HTML - for full details use browser URL'
};
}
async extractIssueWithBrowser(issueUrl, issueId) {
let browser = null;
try {
// Launch browser
browser = await chromium.launch({
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
});
const page = await browser.newPage({
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
});
// Set up request interception to capture batch API calls
const capturedRequests = [];
const capturedResponses = [];
// Enable request interception
await page.route('**/*', async (route) => {
const request = route.request();
const url = request.url();
// Log all requests for debugging
if (url.includes('/batch') || url.includes('googleapis.com') || url.includes('issues.chromium.org')) {
this.debug(`[DEBUG] Intercepted request: ${request.method()} ${url}`);
capturedRequests.push({
url,
method: request.method(),
headers: request.headers(),
postData: request.postData()
});
}
// Continue with the request
await route.continue();
});
// Capture responses
page.on('response', async (response) => {
const url = response.url();
if (url.includes('/batch') || url.includes('googleapis.com')) {
this.debug(`[DEBUG] Captured response: ${response.status()} ${url}`);
try {
const responseText = await response.text();
capturedResponses.push({
url,
status: response.status(),
headers: response.headers(),
body: responseText
});
}
catch (e) {
this.debug(`[DEBUG] Could not capture response body: ${e}`);
}
}
});
// Navigate to issue page
await page.goto(issueUrl, {
waitUntil: 'networkidle',
timeout: 30000
});
// Wait for page to load and batch requests to complete
await page.waitForTimeout(5000);
// Try to wait for some common elements that indicate the page has loaded
try {
await page.waitForSelector('body', { timeout: 5000 });
}
catch (e) {
// Continue anyway
}
// Try to parse batch API responses first
const batchApiData = this.parseBatchAPIResponses(capturedResponses, issueId);
// Extract issue information using multiple strategies
const issueInfo = await page.evaluate((issueId) => {
const result = {
title: `Issue ${issueId}`,
status: 'Unknown',
priority: 'Unknown',
type: 'Unknown',
severity: 'Unknown',
reporter: 'Unknown',
assignee: 'Unknown',
description: '',
relatedCLs: []
};
// Strategy 1: Try to find issue title in common selectors
const titleSelectors = [
'[data-testid="issue-title"]',
'.issue-title',
'h1',
'h2',
'[role="heading"]',