go-git-it
Version:
Download any repository or subdirectory on GitHub with support for Node.js and the CLI
432 lines (425 loc) • 18.6 kB
JavaScript
import * as __WEBPACK_EXTERNAL_MODULE_progress__ from "progress";
import * as __WEBPACK_EXTERNAL_MODULE_https__ from "https";
import * as __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__ from "fs/promises";
import * as __WEBPACK_EXTERNAL_MODULE_path__ from "path";
import * as __WEBPACK_EXTERNAL_MODULE_child_process__ from "child_process";
import * as __WEBPACK_EXTERNAL_MODULE_util__ from "util";
async function addProgressBar(text, completionCallback) {
await new Promise((resolve, reject)=>{
const contentLength = 2097152;
if (text) console.log(text);
const bar = new __WEBPACK_EXTERNAL_MODULE_progress__["default"]("[:bar] :percent :etas", {
complete: '=',
incomplete: ' ',
width: 50,
total: contentLength
});
let progress = 0;
const timer = setInterval(()=>{
const chunk = 10 * Math.random() * 1024;
progress += chunk;
bar.tick(chunk);
if (progress >= contentLength || bar.complete) clearInterval(timer);
}, 50);
completionCallback().then(()=>{
clearInterval(timer);
bar.tick(contentLength);
resolve();
}).catch((error)=>{
clearInterval(timer);
console.error('[go-git-it] An error occurred:', error);
reject(error);
});
});
}
const DEFAULT_BRANCH = 'main';
const GITHUB_DOMAIN = 'github.com';
function parseGitHubUrl(gitUrl) {
try {
const url = new URL(gitUrl);
if (!url.hostname.includes(GITHUB_DOMAIN)) throw new Error(`Unsupported domain: ${url.hostname}. Only GitHub URLs are supported.`);
const pathSegments = url.pathname.split('/').filter((segment)=>segment.length > 0);
if (pathSegments.length < 2) throw new Error('Invalid GitHub URL: Missing owner or repository name');
const owner = pathSegments[0];
const project = pathSegments[1];
if (isReleaseAssetUrl(pathSegments)) return parseReleaseAssetUrl(owner, project, pathSegments);
return parseRepositoryUrl(owner, project, pathSegments);
} catch (error) {
if (error instanceof Error) throw new Error(`Failed to parse GitHub URL: ${error.message}`);
throw error;
}
}
function isReleaseAssetUrl(pathSegments) {
return pathSegments.includes('releases') && pathSegments.includes('download');
}
function parseReleaseAssetUrl(owner, project, pathSegments) {
const releaseIndex = pathSegments.findIndex((segment)=>'releases' === segment);
const downloadIndex = pathSegments.findIndex((segment)=>'download' === segment);
if (-1 === releaseIndex || -1 === downloadIndex || downloadIndex !== releaseIndex + 1) throw new Error('Invalid release asset URL format');
const tag = pathSegments[downloadIndex + 1];
const assetName = pathSegments[downloadIndex + 2];
if (!tag || !assetName) throw new Error('Missing release tag or asset name in URL');
const filePath = pathSegments.slice(releaseIndex).join('/');
const downloadUrl = `https://github.com/${owner}/${project}/releases/download/${tag}/${assetName}`;
return {
owner,
project,
branch: DEFAULT_BRANCH,
filePath,
isMainRepo: false,
isReleaseAsset: true,
isFile: true,
downloadUrl
};
}
function parseRepositoryUrl(owner, project, pathSegments) {
const blobIndex = pathSegments.findIndex((segment)=>'blob' === segment);
const treeIndex = pathSegments.findIndex((segment)=>'tree' === segment);
const indicatorIndex = -1 !== blobIndex ? blobIndex : treeIndex;
let branch = DEFAULT_BRANCH;
let filePath = '';
let isFile = false;
if (-1 !== indicatorIndex) {
if (pathSegments.length <= indicatorIndex + 1) throw new Error('Invalid URL: Missing branch after blob/tree indicator');
branch = pathSegments[indicatorIndex + 1];
filePath = pathSegments.slice(indicatorIndex + 2).join('/');
isFile = -1 !== blobIndex;
} else if (pathSegments.length > 2) {
filePath = pathSegments.slice(2).join('/');
isFile = hasFileExtension(filePath);
}
const isMainRepo = !filePath || filePath === project;
return {
owner,
project,
branch,
filePath: filePath || project,
isMainRepo,
isReleaseAsset: false,
isFile: isFile && !isMainRepo,
downloadUrl: void 0
};
}
function hasFileExtension(path) {
const segments = path.split('/');
const lastSegment = segments[segments.length - 1];
return lastSegment.includes('.') && !lastSegment.startsWith('.');
}
function isValidGitHubUrl(url) {
try {
const parsedUrl = new URL(url);
return parsedUrl.hostname.includes(GITHUB_DOMAIN);
} catch {
return false;
}
}
function getOutputDirectoryName(data) {
if (data.isReleaseAsset) {
const segments = data.filePath.split('/');
return segments[segments.length - 1];
}
if (data.isMainRepo) return data.project;
const segments = data.filePath.split('/');
return segments[segments.length - 1];
}
const exec = (0, __WEBPACK_EXTERNAL_MODULE_util__.promisify)(__WEBPACK_EXTERNAL_MODULE_child_process__.exec);
function generateTempDirName() {
const timestamp = Date.now();
const random = Math.random().toString(36).substring(2, 8);
return `.go-git-it-temp-${timestamp}-${random}`;
}
async function removeDirectory(dirPath) {
try {
await __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].rm(dirPath, {
recursive: true,
force: true
});
} catch (error) {
if (error instanceof Error && 'code' in error && 'ENOENT' === error.code) return;
throw error;
}
}
async function cleanupGitDirectory(repoPath) {
const gitPath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(repoPath, '.git');
try {
await removeDirectory(gitPath);
} catch (error) {
console.warn(`Warning: Could not remove .git directory: ${error}`);
}
}
async function createDirectory(dirPath) {
try {
await __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].mkdir(dirPath, {
recursive: true
});
} catch (error) {
if (error instanceof Error && 'code' in error && 'EEXIST' === error.code) return;
throw error;
}
}
async function pathExists(targetPath) {
try {
await __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].access(targetPath);
return true;
} catch {
return false;
}
}
async function moveFileOrDirectory(source, destination) {
try {
await __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].rename(source, destination);
} catch (error) {
if (error instanceof Error && 'code' in error && 'EXDEV' === error.code) {
await __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].cp(source, destination, {
recursive: true
});
await removeDirectory(source);
} else throw error;
}
}
async function executeGitCommand(command, options = {
cwd: process.cwd()
}) {
const { cwd, timeout = 30000 } = options;
try {
const { stdout, stderr } = await exec(command, {
cwd,
timeout,
env: {
...process.env,
GIT_TERMINAL_PROMPT: '0'
}
});
if (stderr && !stderr.includes('warning:')) console.warn(`Git warning: ${stderr}`);
return stdout;
} catch (error) {
if (error instanceof Error) throw new Error(`Git command failed: ${command}\nError: ${error.message}`);
throw error;
}
}
async function cleanupTempDirectory(tempPath, maxRetries = 3) {
for(let attempt = 1; attempt <= maxRetries; attempt++)try {
if (await pathExists(tempPath)) await removeDirectory(tempPath);
return;
} catch {
if (attempt === maxRetries) console.warn(`Warning: Could not remove temporary directory after ${maxRetries} attempts: ${tempPath}`);
else await new Promise((resolve)=>setTimeout(resolve, 100 * attempt));
}
}
async function downloadFullRepository(outputDirectory, data) {
const projectPath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(outputDirectory, data.project);
await createDirectory(projectPath);
try {
await executeGitCommand('git init --quiet', {
cwd: projectPath
});
await executeGitCommand(`git remote add origin https://github.com/${data.owner}/${data.project}`, {
cwd: projectPath
});
const branchesToTry = [
data.branch,
'main',
'master',
'develop'
];
const uniqueBranches = [
...new Set(branchesToTry)
];
let success = false;
let lastError = null;
for (const branch of uniqueBranches)try {
await executeGitCommand(`git pull origin ${branch} --depth 1 --quiet`, {
cwd: projectPath
});
success = true;
break;
} catch (error) {
lastError = error instanceof Error ? error : new Error(String(error));
console.log(`Failed to pull from branch '${branch}', trying next...`);
}
if (!success) throw new Error(`Could not pull from any branch. Last error: ${(null == lastError ? void 0 : lastError.message) || 'Unknown error'}`);
await cleanupGitDirectory(projectPath);
} catch (error) {
await removeDirectory(projectPath);
throw error;
}
}
async function downloadPartialRepository(outputDirectory, data, tempDir) {
try {
await executeGitCommand('git init --quiet', {
cwd: tempDir
});
await executeGitCommand(`git remote add origin https://github.com/${data.owner}/${data.project}`, {
cwd: tempDir
});
await executeGitCommand('git config core.sparseCheckout true', {
cwd: tempDir
});
const sparseCheckoutPath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(tempDir, '.git', 'info', 'sparse-checkout');
const sparsePath = data.isFile ? data.filePath : `${data.filePath}/*`;
await __WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].writeFile(sparseCheckoutPath, sparsePath);
await executeGitCommand(`git pull origin ${data.branch} --depth 1 --quiet`, {
cwd: tempDir
});
const sourcePath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(tempDir, data.filePath);
const outputName = __WEBPACK_EXTERNAL_MODULE_path__["default"].basename(data.filePath);
const destinationPath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(outputDirectory, outputName);
if (!await pathExists(sourcePath)) throw new Error(`Content not found at path: ${data.filePath}`);
await moveFileOrDirectory(sourcePath, destinationPath);
} catch (error) {
throw new Error(`Failed to download partial repository: ${error instanceof Error ? error.message : String(error)}`);
}
}
async function downloadReleaseAsset(outputDirectory, data, tempDir) {
if (!data.downloadUrl) throw new Error('Release asset download URL not available');
const fileName = __WEBPACK_EXTERNAL_MODULE_path__["default"].basename(data.filePath);
const tempFilePath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(tempDir, fileName);
const finalPath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(outputDirectory, fileName);
return new Promise((resolve, reject)=>{
__WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].open(tempFilePath, 'w').then((fileHandle)=>{
const writeStream = fileHandle.createWriteStream();
__WEBPACK_EXTERNAL_MODULE_https__["default"].get(data.downloadUrl, (response)=>{
if (302 === response.statusCode || 301 === response.statusCode) {
const redirectUrl = response.headers.location;
if (redirectUrl) {
writeStream.destroy();
fileHandle.close();
__WEBPACK_EXTERNAL_MODULE_https__["default"].get(redirectUrl, (redirectResponse)=>{
if (200 !== redirectResponse.statusCode) return void reject(new Error(`Failed to download release asset: HTTP ${redirectResponse.statusCode}`));
handleDownloadStream(redirectResponse, tempFilePath, finalPath, resolve, reject);
}).on('error', reject);
return;
}
}
if (200 !== response.statusCode) {
writeStream.destroy();
fileHandle.close();
reject(new Error(`Failed to download release asset: HTTP ${response.statusCode}`));
return;
}
handleDownloadStream(response, tempFilePath, finalPath, resolve, reject);
}).on('error', (error)=>{
writeStream.destroy();
fileHandle.close();
reject(error);
});
}).catch(reject);
});
}
function handleDownloadStream(response, tempFilePath, finalPath, resolve, reject) {
__WEBPACK_EXTERNAL_MODULE_fs_promises_400951f8__["default"].open(tempFilePath, 'w').then((fileHandle)=>{
const writeStream = fileHandle.createWriteStream();
response.pipe(writeStream);
writeStream.on('finish', async ()=>{
await fileHandle.close();
try {
await moveFileOrDirectory(tempFilePath, finalPath);
resolve();
} catch (error) {
reject(error instanceof Error ? error : new Error(String(error)));
}
});
writeStream.on('error', async (error)=>{
await fileHandle.close();
await removeDirectory(tempFilePath).catch(()=>{});
reject(error);
});
}).catch(reject);
}
async function validateGitAvailability() {
try {
const output = await executeGitCommand('git --version', {
cwd: process.cwd()
});
console.log(`Using ${output.trim()}`);
} catch {
throw new Error("Git is required but not found in PATH. Please install Git and ensure it's available in your terminal.");
}
}
async function testGitHubConnectivity(owner, project, skipCheck = false) {
if (skipCheck || 'test' === process.env.NODE_ENV) return;
return new Promise((resolve, reject)=>{
const testUrl = `https://api.github.com/repos/${owner}/${project}`;
__WEBPACK_EXTERNAL_MODULE_https__["default"].get(testUrl, (response)=>{
if (200 === response.statusCode) resolve();
else if (404 === response.statusCode) reject(new Error(`Repository ${owner}/${project} not found or is private`));
else if (403 === response.statusCode) {
console.warn('GitHub API rate limit reached, continuing without connectivity check...');
resolve();
} else reject(new Error(`GitHub API returned status ${response.statusCode}`));
}).on('error', (error)=>{
reject(new Error(`Failed to connect to GitHub: ${error.message}`));
});
});
}
function cli(goGitIt) {
const args = process.argv.slice(2);
if (0 === args.length || args.includes('--help') || args.includes('-h')) {
console.log(`
go-git-it - Download GitHub repositories, folders, or files
USAGE:
go-git-it <github-url> [directory]
EXAMPLES:
# Clone entire repository (like git clone)
go-git-it https://github.com/owner/repo
go-git-it https://github.com/owner/repo ./my-projects
# Download specific folder
go-git-it https://github.com/owner/repo/tree/main/src
# Download specific file
go-git-it https://github.com/owner/repo/blob/main/README.md
# Download release asset
go-git-it https://github.com/owner/repo/releases/download/v1.0.0/asset.zip
For more information, visit: https://github.com/cezaraugusto/go-git-it
`);
process.exit(0);
}
if (args.length < 1 || args.length > 2) {
console.error('Error: Invalid number of arguments.');
console.error('Usage: go-git-it <github-url> [directory]');
console.error('Use --help for more information.');
process.exit(1);
}
const gitUrl = args[0];
const outputDirectory = args[1];
(async ()=>{
try {
await goGitIt(gitUrl, outputDirectory);
} catch (error) {
console.error('Error:', error instanceof Error ? error.message : String(error));
process.exit(1);
}
})();
}
async function cloneRemote(outputDirectory, gitUrl) {
await validateGitAvailability();
const urlData = parseGitHubUrl(gitUrl);
await testGitHubConnectivity(urlData.owner, urlData.project);
const tempDirName = generateTempDirName();
const tempDir = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(outputDirectory, tempDirName);
try {
await createDirectory(tempDir);
if (urlData.isReleaseAsset) await downloadReleaseAsset(outputDirectory, urlData, tempDir);
else if (urlData.isMainRepo) await downloadFullRepository(outputDirectory, urlData);
else await downloadPartialRepository(outputDirectory, urlData, tempDir);
} finally{
await cleanupTempDirectory(tempDir);
}
}
async function src_goGitIt(gitURL, outputDirectory, progressText) {
if (!isValidGitHubUrl(gitURL)) throw new Error('Invalid GitHub URL. Please provide a valid GitHub repository URL.');
const urlData = parseGitHubUrl(gitURL);
const outputName = getOutputDirectoryName(urlData);
const outDir = outputDirectory || process.cwd();
await createDirectory(outDir);
const remoteSource = `${urlData.owner}/${urlData.project}`;
const defaultProgressText = urlData.isMainRepo ? `Cloning ${remoteSource}...` : `Downloading ${outputName} from ${remoteSource}...`;
await addProgressBar(progressText || defaultProgressText, async ()=>{
await cloneRemote(outDir, gitURL);
});
const finalPath = __WEBPACK_EXTERNAL_MODULE_path__["default"].join(outDir, outputName);
if (!progressText) console.log(`Success! Content downloaded to ${finalPath}`);
}
if (import.meta.url === `file://${process.argv[1]}`) cli(src_goGitIt);
const src = src_goGitIt;
export { src as default };