git-contribution-stats
Version:
High-performance library to generate GitHub contribution reports with timeout controls, circuit breakers, and selective processing for AWS Lambda and background jobs
657 lines (656 loc) ā¢ 30.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateGitHubReport = generateGitHubReport;
exports.generateGitHubReportLegacy = generateGitHubReportLegacy;
const octokit_1 = require("octokit");
const logger_1 = require("./logger");
async function generateGitHubReport(config, progressCallback) {
const startTime = Date.now();
const { app_id, private_key, days_to_look_back = 7, max_total_timeout, continue_on_error = false, partial_results_on_timeout = false } = config;
const logger = config.logger || new logger_1.Logger({
level: 'info',
scope: ['GitHubReport']
});
logger.start(`Initializing GitHub App with ID: ${app_id}`);
const app = await initializeGitHubApp(app_id, private_key);
let results = [];
let errors = [];
let partialTimeout = false;
try {
const installations = await getInstallations(app);
logger.info(`Found ${installations.length} installations`);
// Apply installation filtering and sorting
const filteredInstallations = filterAndSortInstallations(installations, config, logger);
logger.info(`Processing ${filteredInstallations.length} installations after filtering`);
// Setup global timeout if specified
const processPromise = processInstallationsWithOptimizations(app, filteredInstallations, config, progressCallback, logger);
if (max_total_timeout) {
const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Global timeout exceeded')), max_total_timeout));
try {
const processResult = await Promise.race([processPromise, timeoutPromise]);
results = processResult.results;
errors = processResult.errors;
}
catch (error) {
if (error.message === 'Global timeout exceeded') {
logger.warn(`Global timeout of ${max_total_timeout}ms exceeded`);
partialTimeout = true;
if (partial_results_on_timeout) {
// Get partial results from the promise if possible
results = (await processPromise.catch(() => ({ results: [], errors: [] }))).results;
errors = (await processPromise.catch(() => ({ results: [], errors: [] }))).errors;
}
else {
throw error;
}
}
else {
throw error;
}
}
}
else {
const processResult = await processPromise;
results = processResult.results;
errors = processResult.errors;
}
const executionTime = Date.now() - startTime;
const report = generateReport(results);
const processedCount = results.filter(r => !('error' in r)).length;
const failedInstallations = results.filter(r => 'error' in r).map(r => r.account);
logger.success(`GitHub activity report generated successfully`);
logger.info(`Execution time: ${executionTime}ms`);
logger.info(`Processed: ${processedCount}/${filteredInstallations.length} installations`);
return {
success: !partialTimeout && errors.length === 0,
total_installations: filteredInstallations.length,
processed_installations: processedCount,
failed_installations: failedInstallations,
partial_timeout: partialTimeout,
execution_time: executionTime,
summary: report,
detailed_results: results,
errors: errors
};
}
catch (error) {
logger.error(`Error accessing GitHub API: ${error.message}`);
const executionTime = Date.now() - startTime;
return {
success: false,
total_installations: 0,
processed_installations: 0,
failed_installations: [],
partial_timeout: false,
execution_time: executionTime,
summary: `Error: ${error.message}`,
detailed_results: results,
errors: [{ installation_id: 'global', error: error.message, timestamp: Date.now() }]
};
}
}
// Legacy function for backward compatibility
async function generateGitHubReportLegacy(config) {
const result = await generateGitHubReport(config);
return {
summary: result.summary,
detailed_results: result.detailed_results
};
}
async function initializeGitHubApp(app_id, private_key) {
return new octokit_1.App({
appId: app_id,
privateKey: private_key,
});
}
async function getInstallations(app) {
const response = await app.octokit.request('GET /app/installations', {
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
});
return response.data;
}
function filterAndSortInstallations(installations, config, logger) {
let filtered = [...installations];
// Filter by target installations
if (config.target_installations && config.target_installations.length > 0) {
filtered = filtered.filter(inst => config.target_installations.includes(inst.account.login));
logger.info(`Filtered to target installations: ${filtered.map(i => i.account.login).join(', ')}`);
}
// Exclude installations
if (config.exclude_installations && config.exclude_installations.length > 0) {
const before = filtered.length;
filtered = filtered.filter(inst => !config.exclude_installations.includes(inst.account.login));
logger.info(`Excluded ${before - filtered.length} installations: ${config.exclude_installations.join(', ')}`);
}
// Sort by priority mode
if (config.priority_mode) {
switch (config.priority_mode) {
case 'smallest_first':
// Note: We don't have repo count here, so we'll sort by account type and name
filtered.sort((a, b) => {
if (a.account.type !== b.account.type) {
return a.account.type === 'User' ? -1 : 1;
}
return a.account.login.localeCompare(b.account.login);
});
logger.info('Sorted installations: smallest first (User accounts first)');
break;
case 'largest_first':
filtered.sort((a, b) => {
if (a.account.type !== b.account.type) {
return a.account.type === 'Organization' ? -1 : 1;
}
return b.account.login.localeCompare(a.account.login);
});
logger.info('Sorted installations: largest first (Organization accounts first)');
break;
case 'sequential':
default:
logger.info('Processing installations in sequential order');
break;
}
}
return filtered;
}
async function processInstallationsWithOptimizations(app, installations, config, progressCallback, logger) {
const results = [];
const errors = [];
const { days_to_look_back = 7, timeout_per_installation, continue_on_error = false, retry_failed_installations = 0 } = config;
logger?.start(`Processing ${installations.length} installations with optimizations`);
for (const [index, installation] of installations.entries()) {
const installationStartTime = Date.now();
logger?.pending(`Processing installation ${index + 1}/${installations.length}: ${installation.account.login}`);
progressCallback?.onInstallationStart?.(installation, index + 1, installations.length);
let attempts = 0;
let success = false;
while (attempts <= retry_failed_installations && !success) {
attempts++;
try {
if (attempts > 1) {
logger?.info(`Retry attempt ${attempts} for ${installation.account.login}`);
}
const installationResult = await processInstallationWithTimeout(app, installation, config, progressCallback, logger);
const elapsed = Date.now() - installationStartTime;
logger?.complete(`Finished processing ${installation.account.login} in ${elapsed}ms`);
progressCallback?.onInstallationComplete?.(installation, installationResult);
results.push(installationResult);
success = true;
}
catch (error) {
const elapsed = Date.now() - installationStartTime;
if (error.message === 'Installation timeout') {
logger?.warn(`Installation ${installation.account.login} timed out after ${elapsed}ms`);
progressCallback?.onTimeout?.(installation, elapsed);
}
else {
logger?.error(`Error processing installation ${installation.account.login}: ${error.message}`);
progressCallback?.onInstallationError?.(installation, error);
}
const errorRecord = {
installation_id: installation.account.login,
error: error.message,
timestamp: Date.now()
};
errors.push(errorRecord);
if (attempts > retry_failed_installations || !continue_on_error) {
const installationError = {
id: installation.id,
account: installation.account.login,
error: error.message,
timestamp: Date.now()
};
results.push(installationError);
if (!continue_on_error) {
logger?.error(`Stopping processing due to error in ${installation.account.login}`);
break;
}
success = true; // Stop retrying for this installation
}
}
}
}
logger?.complete(`Processed all ${installations.length} installations`);
return { results, errors };
}
async function processInstallationWithTimeout(app, installation, config, progressCallback, logger) {
const { days_to_look_back = 7, timeout_per_installation, max_repositories_per_installation, skip_large_installations, installation_size_threshold } = config;
const processInstallation = async () => {
const octokit = await app.getInstallationOctokit(installation.id);
const repos = await getRepositories(octokit);
logger?.info(`${installation.account.login} has access to ${repos.length} repositories`);
// Circuit breaker for large installations
if (skip_large_installations && installation_size_threshold) {
if (repos.length > installation_size_threshold) {
logger?.warn(`Skipping large installation: ${installation.account.login} (${repos.length} repos > ${installation_size_threshold})`);
throw new Error(`Installation too large: ${repos.length} repositories`);
}
}
// Limit repositories per installation
let processableRepos = repos;
if (max_repositories_per_installation && repos.length > max_repositories_per_installation) {
processableRepos = repos.slice(0, max_repositories_per_installation);
logger?.warn(`Limited ${installation.account.login} to ${max_repositories_per_installation} repositories (had ${repos.length})`);
}
const cutoff_date = calculateCutoffDate(days_to_look_back);
const updated_repos = filterUpdatedRepositories(processableRepos, cutoff_date, logger);
logger?.complete(`Filtered repositories: ${updated_repos.length} matched criteria`);
if (updated_repos.length === 0) {
logger?.info(`No recent activity found for ${installation.account.login}`);
}
const user_stats = await collectUserStatsWithOptimizations(octokit, updated_repos, cutoff_date, config, installation, progressCallback, logger);
return formatInstallationStats(installation, days_to_look_back, user_stats);
};
if (timeout_per_installation) {
const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Installation timeout')), timeout_per_installation));
return await Promise.race([processInstallation(), timeoutPromise]);
}
else {
return await processInstallation();
}
}
// Legacy function for backward compatibility
async function processInstallations(app, installations, days_to_look_back, logger) {
const config = {
app_id: 0, // Not used in this context
private_key: '', // Not used in this context
days_to_look_back,
continue_on_error: true
};
const result = await processInstallationsWithOptimizations(app, installations, config, undefined, logger);
return result.results;
}
async function getRepositories(octokit) {
const { data: { repositories } } = await octokit.request('GET /installation/repositories', {
per_page: 100,
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
});
return repositories;
}
function calculateCutoffDate(days_to_look_back) {
const cutoff_date = new Date();
cutoff_date.setDate(cutoff_date.getDate() - days_to_look_back);
return cutoff_date;
}
function filterUpdatedRepositories(repos, cutoff_date, logger) {
logger.start(`Filtering ${repos.length} repositories by update date`);
const filtered = repos.filter(repo => {
const last_pushed = new Date(repo.pushed_at || 0);
const last_updated = new Date(repo.updated_at || 0);
const latest_activity = last_pushed > last_updated ? last_pushed : last_updated;
const was_updated_in_period = latest_activity >= cutoff_date;
if (!was_updated_in_period) {
logger.debug(`Skipping repository ${repo.full_name} - no updates since ${cutoff_date.toISOString()}`);
}
return was_updated_in_period;
});
logger.complete(`Filtered repositories: ${filtered.length} matched criteria`);
return filtered;
}
async function collectUserStatsWithOptimizations(octokit, updated_repos, cutoff_date, config, installation, progressCallback, logger) {
const user_stats = {};
const since = cutoff_date.toISOString();
const { max_branches_per_repository } = config;
logger?.start(`Collecting stats for ${updated_repos.length} repositories`);
for (const [index, repo] of updated_repos.entries()) {
try {
const processed_commits = new Set();
logger?.pending(`Processing repository ${index + 1}/${updated_repos.length}: ${repo.full_name}`);
progressCallback?.onRepositoryStart?.(repo, installation);
await processRepositoryBranchesWithOptimizations(octokit, repo, since, cutoff_date, processed_commits, user_stats, max_branches_per_repository, logger);
await processRepositoryPullRequests(octokit, repo, cutoff_date, user_stats, logger);
logger?.complete(`Found ${processed_commits.size} commits in ${repo.name}`);
}
catch (repo_error) {
logger?.error(`Error processing repository: ${repo_error.message}`);
}
}
logger?.complete(`Collected stats for all repositories`);
return user_stats;
}
// Legacy function for backward compatibility
async function collectUserStats(octokit, updated_repos, cutoff_date, logger) {
const config = {
app_id: 0, // Not used in this context
private_key: '' // Not used in this context
};
return await collectUserStatsWithOptimizations(octokit, updated_repos, cutoff_date, config, {}, // dummy installation
undefined, // no callback
logger);
}
async function processRepositoryBranchesWithOptimizations(octokit, repo, since, cutoff_date, processed_commits, user_stats, max_branches_per_repository, logger) {
const owner_login = repo.owner.login;
const repo_name = repo.name;
const branches = await getBranches(octokit, owner_login, repo_name);
logger?.info(`Repository has ${branches.length} branches`);
// Limit branches if specified
let processableBranches = branches;
if (max_branches_per_repository && branches.length > max_branches_per_repository) {
processableBranches = branches.slice(0, max_branches_per_repository);
logger?.warn(`Repository ${repo.full_name} has ${branches.length} branches, limiting to ${max_branches_per_repository}`);
}
for (const branch of processableBranches) {
await processBranchCommits(octokit, owner_login, repo_name, branch, since, processed_commits, user_stats, logger);
}
}
// Legacy function for backward compatibility
async function processRepositoryBranches(octokit, repo, since, cutoff_date, processed_commits, user_stats, logger) {
await processRepositoryBranchesWithOptimizations(octokit, repo, since, cutoff_date, processed_commits, user_stats, undefined, // no branch limit
logger);
}
async function getBranches(octokit, owner_login, repo_name) {
const branches_query = `
query($owner: String!, $repo: String!) {
repository(owner: $owner, name: $repo) {
refs(refPrefix: "refs/heads/", first: 100) {
nodes {
name
target {
oid
}
}
}
}
}
`;
const branches_result = await octokit.graphql(branches_query, {
owner: owner_login,
repo: repo_name
});
return branches_result.repository.refs.nodes || [];
}
async function processBranchCommits(octokit, owner_login, repo_name, branch, since, processed_commits, user_stats, logger) {
const commits_query = `
query($owner: String!, $repo: String!, $branchName: String!, $since: GitTimestamp!) {
repository(owner: $owner, name: $repo) {
ref(qualifiedName: $branchName) {
target {
... on Commit {
history(since: $since, first: 100) {
nodes {
oid
message
committedDate
author {
user {
login
id
databaseId
}
name
email
}
}
}
}
}
}
}
}
`;
const commits_result = await octokit.graphql(commits_query, {
owner: owner_login,
repo: repo_name,
branchName: `refs/heads/${branch.name}`,
since: since
});
const branch_commits = commits_result.repository.ref?.target?.history?.nodes || [];
logger.debug(`Found ${branch_commits.length} commits in the period`);
for (const commit of branch_commits) {
if (!processed_commits.has(commit.oid)) {
processed_commits.add(commit.oid);
const author_login = commit.author.user?.login || commit.author.name || commit.author.email || "Unknown";
const user_id = commit.author.user?.databaseId || null;
if (!user_stats[author_login]) {
user_stats[author_login] = {
user_id: user_id,
commits_by_repo: {},
pull_requests_by_repo: {},
commit_details_by_repo: {},
total_commits: 0,
total_prs_opened: 0,
total_prs_closed: 0
};
}
else if (user_id && !user_stats[author_login].user_id) {
user_stats[author_login].user_id = user_id;
}
if (!user_stats[author_login].commits_by_repo[repo_name]) {
user_stats[author_login].commits_by_repo[repo_name] = 0;
user_stats[author_login].commit_details_by_repo[repo_name] = [];
}
user_stats[author_login].commits_by_repo[repo_name]++;
user_stats[author_login].total_commits++;
user_stats[author_login].commit_details_by_repo[repo_name].push({
id: commit.oid,
message: commit.message,
date: commit.committedDate
});
}
}
}
async function processRepositoryPullRequests(octokit, repo, cutoff_date, user_stats, logger) {
const owner_login = repo.owner.login;
const repo_name = repo.name;
const since_date_str = cutoff_date.toISOString().split('T')[0];
try {
const search_query = `repo:${owner_login}/${repo_name} is:pr updated:>=${since_date_str}`;
logger.debug(`Searching PRs with query: ${search_query}`);
const search_prs_response = await octokit.request('GET /search/issues', {
q: search_query,
per_page: 100,
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
});
logger.info(`Search API found ${search_prs_response.data.total_count} PRs for ${repo_name}`);
await processPullRequestsFromSearch(search_prs_response.data.items, repo, cutoff_date, user_stats, logger);
if (search_prs_response.data.total_count === 0) {
logger.note(`Using fallback for pulls endpoint for ${repo_name}`);
await fetchPRsWithPagination(octokit, repo, cutoff_date, user_stats, logger);
}
}
catch (search_error) {
logger.error(`Error searching PRs for ${repo.full_name}: ${search_error.message}`);
logger.note(`Using fallback for pulls endpoint for ${repo_name}`);
await fetchPRsWithPagination(octokit, repo, cutoff_date, user_stats, logger);
}
}
async function processPullRequestsFromSearch(search_prs, repo, cutoff_date, user_stats, logger) {
const owner_login = repo.owner.login;
const repo_name = repo.name;
for (const pr of search_prs || []) {
try {
const pr_repo_regex = /\/repos\/([^\/]+)\/([^\/]+)\/pulls\/\d+/;
const pr_repo_match = pr.pull_request.url.match(pr_repo_regex);
if (pr_repo_match && pr_repo_match[1] === owner_login && pr_repo_match[2] === repo_name) {
await processPullRequest(pr, repo_name, cutoff_date, user_stats, logger);
}
}
catch (pr_error) {
logger.error(`Error processing individual PR: ${pr_error.message}`);
}
}
}
async function processPullRequest(pr, repo_name, cutoff_date, user_stats, logger) {
const author_login = pr.user ? pr.user.login : "Unknown";
const user_id = pr.user?.id || pr.user?.node_id || null;
if (!user_stats[author_login]) {
user_stats[author_login] = {
user_id: user_id,
commits_by_repo: {},
pull_requests_by_repo: {},
commit_details_by_repo: {},
total_commits: 0,
total_prs_opened: 0,
total_prs_closed: 0
};
}
else if (user_id && !user_stats[author_login].user_id) {
user_stats[author_login].user_id = user_id;
}
if (!user_stats[author_login].pull_requests_by_repo[repo_name]) {
user_stats[author_login].pull_requests_by_repo[repo_name] = {
opened: 0,
closed: 0
};
}
const created_at = new Date(pr.created_at);
if (created_at >= cutoff_date) {
user_stats[author_login].pull_requests_by_repo[repo_name].opened++;
user_stats[author_login].total_prs_opened++;
logger.debug(`PR #${pr.number} by ${author_login} counted as opened`);
}
if (pr.closed_at) {
const closed_at = new Date(pr.closed_at);
if (closed_at >= cutoff_date) {
user_stats[author_login].pull_requests_by_repo[repo_name].closed++;
user_stats[author_login].total_prs_closed++;
logger.debug(`PR #${pr.number} by ${author_login} counted as closed`);
}
}
}
async function fetchPRsWithPagination(octokit, repo, cutoff_date, user_stats, logger) {
let page = 1;
let has_more_pages = true;
let total_prs = 0;
while (has_more_pages) {
try {
if (page > 1) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
logger.pending(`Fetching PRs for ${repo.name} - page ${page}`);
const pull_requests_response = await octokit.request('GET /repos/{owner}/{repo}/pulls', {
owner: repo.owner.login,
repo: repo.name,
state: 'all',
sort: 'updated',
direction: 'desc',
per_page: 100,
page: page,
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
});
const pull_requests = pull_requests_response.data;
logger.info(`Page ${page}: Found ${pull_requests.length} PRs for ${repo.name}`);
has_more_pages = pull_requests.length === 100;
page++;
const filtered_prs = pull_requests.filter((pr) => {
try {
const updated_at = new Date(pr.updated_at);
return updated_at >= cutoff_date;
}
catch (err) {
logger.error(`Error filtering PR: ${err}`);
return false;
}
});
logger.info(`${filtered_prs.length} PRs updated in the specified period`);
total_prs += filtered_prs.length;
for (const pr of filtered_prs) {
try {
await processPullRequest(pr, repo.name, cutoff_date, user_stats, logger);
}
catch (pr_error) {
logger.error(`Error processing PR: ${pr_error.message}`);
}
}
if (filtered_prs.length === 0 && page > 2) {
logger.note(`No recent PRs found on page ${page - 1}, stopping search`);
has_more_pages = false;
}
}
catch (error) {
logger.error(`Error fetching PRs (page ${page}): ${error.message}`);
if (error.status === 403 && error.message.includes('rate limit')) {
logger.warn('Rate limit reached, waiting 60 seconds...');
await new Promise(resolve => setTimeout(resolve, 60000));
page--;
}
else {
has_more_pages = false;
}
}
}
logger.complete(`Total of ${total_prs} recent PRs found for ${repo.name}`);
return total_prs;
}
function formatInstallationStats(installation, days_to_look_back, user_stats) {
const installation_stats = {
installation_id: installation.id,
account: installation.account.login,
user_id: installation.account.id,
account_type: installation.account.type,
period_days: days_to_look_back,
user_stats: []
};
for (const [user_name, stats] of Object.entries(user_stats)) {
const user_stat = {
name: user_name,
user_id: stats.user_id,
total_commits: stats.total_commits,
total_prs_opened: stats.total_prs_opened,
total_prs_closed: stats.total_prs_closed,
repo_contributions: []
};
const repo_set = new Set([
...Object.keys(stats.commits_by_repo),
...Object.keys(stats.pull_requests_by_repo)
]);
for (const repo_name of repo_set) {
const repo_stat = {
repo_name: repo_name,
commits: stats.commits_by_repo[repo_name] || 0,
prs_opened: (stats.pull_requests_by_repo[repo_name] && stats.pull_requests_by_repo[repo_name].opened) || 0,
prs_closed: (stats.pull_requests_by_repo[repo_name] && stats.pull_requests_by_repo[repo_name].closed) || 0,
commit_details: stats.commit_details_by_repo[repo_name] || []
};
user_stat.repo_contributions.push(repo_stat);
}
installation_stats.user_stats.push(user_stat);
}
return installation_stats;
}
function generateReport(results) {
let report_lines = [];
for (const installation of results) {
if ('error' in installation) {
report_lines.push(`ā ļø Installation ${installation.account}: ${installation.error}`);
continue;
}
const period_label = installation.period_days === 7 ? "week" :
(installation.period_days === 30 ? "month" :
`${installation.period_days} days`);
report_lines.push(`š Statistics for ${installation.account} (${installation.account_type}) - Last ${period_label}:`);
if (installation.user_stats && installation.user_stats.length > 0) {
installation.user_stats.sort((a, b) => {
const total_a = a.total_commits + a.total_prs_opened + a.total_prs_closed;
const total_b = b.total_commits + b.total_prs_opened + b.total_prs_closed;
return total_b - total_a;
});
for (const user of installation.user_stats) {
const user_id_info = user.user_id ? ` (ID: ${user.user_id})` : '';
report_lines.push(`\nš¤ ${user.name}${user_id_info}:`);
report_lines.push(` Total: ${user.total_commits} commits, ${user.total_prs_opened} PRs opened, ${user.total_prs_closed} PRs closed`);
user.repo_contributions.sort((a, b) => {
const total_a = a.commits + a.prs_opened + a.prs_closed;
const total_b = b.commits + b.prs_opened + b.prs_closed;
return total_b - total_a;
});
report_lines.push(` Contributions by repository:`);
for (const repo of user.repo_contributions) {
report_lines.push(` - ${repo.repo_name}: ${repo.commits} commits, ${repo.prs_opened} PRs opened, ${repo.prs_closed} PRs closed`);
}
}
}
else {
report_lines.push(` No contributions found in the period.`);
}
report_lines.push('');
}
return report_lines.join('\n');
}