@huggingface/hub
Version:
Utilities to interact with the Hugging Face hub
1,233 lines (1,222 loc) • 45 kB
JavaScript
import {
HUB_URL,
createBranch,
createRepo,
deleteBranch,
deleteRepo,
getJob,
listJobHardware,
listJobs,
listModels,
repoExists,
runJob,
streamJobLogs,
typedEntries,
uploadFilesWithProgress,
whoAmI
} from "./chunk-OWE7BUIC.mjs";
import "./chunk-FFYIGW52.mjs";
// cli.ts
import { parseArgs } from "util";
import { pathToFileURL } from "url";
import { stat } from "fs/promises";
import { basename, join } from "path";
// package.json
var version = "2.13.0";
// cli.ts
var UploadProgressManager = class {
multibar = null;
fileBars = /* @__PURE__ */ new Map();
isQuiet;
cliProgressAvailable = false;
constructor(isQuiet = false) {
this.isQuiet = isQuiet;
}
async initialize() {
if (this.isQuiet) {
return;
}
try {
const cliProgress = await import("./cli-progress-XP5T6RZP.mjs");
this.cliProgressAvailable = true;
this.multibar = new cliProgress.MultiBar(
{
clearOnComplete: false,
hideCursor: true,
format: " {bar} | {filename} | {percentage}% | {state}",
barCompleteChar: "\u2588",
barIncompleteChar: "\u2591"
},
cliProgress.Presets.shades_grey
);
} catch (error) {
this.cliProgressAvailable = false;
}
}
handleEvent(event) {
if (this.isQuiet) {
return;
}
if (event.event === "phase") {
this.logPhase(event.phase);
} else if (event.event === "fileProgress") {
this.updateFileProgress(event.path, event.progress, event.state);
}
}
logPhase(phase) {
if (this.isQuiet) {
return;
}
const phaseMessages = {
preuploading: "\u{1F4CB} Preparing files for upload...",
uploadingLargeFiles: "\u2B06\uFE0F Uploading files...",
committing: "\u2728 Finalizing commit..."
};
console.log(`
${phaseMessages[phase] || phase}`);
}
updateFileProgress(path, progress, state) {
if (this.isQuiet) {
return;
}
if (this.cliProgressAvailable && this.multibar) {
let bar = this.fileBars.get(path);
if (!bar) {
bar = this.multibar.create(100, 0, {
filename: this.truncateFilename(path, 100),
state
});
this.fileBars.set(path, bar);
}
if (state === "error") {
bar.update(0, { state: "\u2717 error" });
} else if (progress >= 1) {
bar.update(100, { state: state === "hashing" ? "\u2713 hashed" : "\u2713 uploaded" });
} else {
const percentage = Math.round(progress * 100);
bar.update(percentage, { state });
}
} else {
const percentage = Math.round(progress * 100);
const truncatedPath = this.truncateFilename(path, 100);
if (state === "error") {
console.error(`\u2717 error: ${truncatedPath}`);
} else if (progress >= 1) {
const statusIcon = state === "hashing" ? "\u2713 hashed" : "\u2713 uploaded";
console.log(`${statusIcon}: ${truncatedPath}`);
} else if (percentage % 25 === 0) {
console.log(`${state}: ${truncatedPath} (${percentage}%)`);
}
}
}
truncateFilename(filename, maxLength) {
if (filename.length <= maxLength) {
return filename;
}
return "..." + filename.slice(-(maxLength - 3));
}
stop() {
if (!this.isQuiet && this.cliProgressAvailable && this.multibar) {
this.multibar.stop();
}
}
};
var commands = {
upload: {
description: "Upload a folder to a repo on the Hub",
args: [
{
name: "repo-name",
description: "The name of the repo to upload to",
positional: true,
required: true
},
{
name: "local-folder",
description: "The local folder to upload. Defaults to the current working directory",
positional: true,
default: () => process.cwd()
},
{
name: "path-in-repo",
description: "The path in the repo to upload the folder to. Defaults to the root of the repo",
positional: true,
default: "."
},
{
name: "quiet",
short: "q",
description: "Suppress all output",
boolean: true
},
{
name: "repo-type",
enum: ["dataset", "model", "space", "bucket"],
description: "The type of repo to upload to. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name or buckets/username/repo-name"
},
{
name: "revision",
description: "The revision to upload to. Defaults to the main branch",
default: "main"
},
{
name: "commit-message",
description: "The commit message to use. Defaults to 'Upload files using @huggingface/hub'",
default: "Upload files using @huggingface/hub"
},
{
name: "private",
description: "If creating a new repo, make it private",
boolean: true
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
},
branch: {
description: "Manage repository branches",
subcommands: {
create: {
description: "Create a new branch in a repo, or update an existing one",
args: [
{
name: "repo-name",
description: "The name of the repo to create the branch in",
positional: true,
required: true
},
{
name: "branch",
description: "The name of the branch to create",
positional: true,
required: true
},
{
name: "repo-type",
enum: ["dataset", "model", "space"],
description: "The type of the repo to create the branch into. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name"
},
{
name: "revision",
description: "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists."
},
{
name: "empty",
boolean: true,
description: "Create an empty branch. This will erase all previous commits on the branch if it exists."
},
{
name: "force",
short: "f",
boolean: true,
description: "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists."
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
},
delete: {
description: "Delete a branch in a repo",
args: [
{
name: "repo-name",
description: "The name of the repo to delete the branch from",
positional: true,
required: true
},
{
name: "branch",
description: "The name of the branch to delete",
positional: true,
required: true
},
{
name: "repo-type",
enum: ["dataset", "model", "space"],
description: "The type of repo to delete the branch from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name"
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
}
}
},
repo: {
description: "Manage repositories on the Hub",
subcommands: {
delete: {
description: "Delete a repository from the Hub",
args: [
{
name: "repo-name",
description: "The name of the repo to delete. You can also prefix the repo name with the type, e.g. datasets/username/repo-name",
positional: true,
required: true
},
{
name: "repo-type",
enum: ["dataset", "model", "space"],
description: "The type of the repo to delete. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name"
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
}
}
},
models: {
description: "Manage models on the Hub",
subcommands: {
list: {
description: "List models on the Hub (first page)",
args: [
{
name: "search",
description: "Search query to filter models by name",
positional: true
},
{
name: "sort",
enum: [
"createdAt",
"downloads",
"likes",
"lastModified",
"likes30d",
"trendingScore",
"num_parameters"
// "mainSize",
// "id",
],
description: "Sort models by a specific field"
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
}
}
},
version: {
description: "Print the version of the CLI",
args: []
},
jobs: {
description: "Manage jobs on the Hub",
subcommands: {
run: {
description: "Run a new job",
args: [
{
name: "docker-image-or-space",
description: "The Docker image to run (e.g., python:3.12) or Space ID (e.g., hf.co/spaces/username/space-name or username/space-name)",
positional: true,
required: true
},
{
name: "command",
description: `The command to run (can be multiple arguments preceded by --, e.g., -- python -c 'import os; print(os.environ["FOO"])')`,
positional: true,
multiple: true
},
{
name: "env",
short: "e",
multiple: true,
description: "Environment variable in the format KEY=VALUE (can be specified multiple times)"
},
{
name: "secret",
short: "s",
multiple: true,
description: "Secret in the format KEY=VALUE (will be encrypted server-side, can be specified multiple times)"
},
{
name: "label",
short: "l",
multiple: true,
description: "Label in the format KEY=VALUE or KEY alone (in this case VALUE defaults to empty string). Can be specified multiple times."
},
{
name: "volume",
short: "v",
multiple: true,
description: "Volume to mount in the format SOURCE:MOUNTPATH[:OPTIONS]. SOURCE uses HuggingFace prefixes (datasets/user/repo, spaces/user/repo, buckets/user/bucket) or bare user/repo for models. OPTIONS are comma-separated: ro, revision=REV, path=SUBPATH. Can be specified multiple times."
},
{
name: "flavor",
description: "Hardware flavor to use (defaults to cpu-basic)",
default: "cpu-basic"
},
{
name: "attempts",
description: "Maximum number of attempts (defaults to 1)"
},
{
name: "namespace",
description: "The namespace (username or organization name). Defaults to the current user's username."
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
},
{
name: "detach",
short: "d",
description: "Don't stream logs after creating the job",
boolean: true
}
]
},
ps: {
description: "List jobs",
args: [
{
name: "all",
short: "a",
description: "List all jobs (not just running ones)",
boolean: true
},
{
name: "namespace",
description: "The namespace (username or organization name). Defaults to the current user's username."
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
},
hardware: {
description: "List available hardware options for jobs",
args: [
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
},
logs: {
description: "Show logs for a job",
args: [
{
name: "job-id",
description: "The job ID",
positional: true,
required: true
},
{
name: "namespace",
description: "The namespace (username or organization name). Defaults to the current user's username."
},
{
name: "token",
description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.",
default: process.env.HF_TOKEN
}
]
}
}
}
};
var mainCommandName = process.argv[2];
var subCommandName;
var cliArgs;
if (mainCommandName && mainCommandName in commands && commands[mainCommandName] && "subcommands" in commands[mainCommandName]) {
subCommandName = process.argv[3];
cliArgs = process.argv.slice(4);
} else {
cliArgs = process.argv.slice(3);
}
async function run() {
switch (mainCommandName) {
case void 0:
case "--help":
case "help": {
const helpArgs = mainCommandName === "help" ? process.argv.slice(3) : [];
if (helpArgs.length > 0) {
const cmdName = helpArgs[0];
if (cmdName && commands[cmdName]) {
const cmdDef = commands[cmdName];
if ("subcommands" in cmdDef) {
if (helpArgs.length > 1) {
const subCmdName = helpArgs[1];
if (subCmdName in cmdDef.subcommands && cmdDef.subcommands[subCmdName]) {
console.log(detailedUsageForSubcommand(cmdName, subCmdName));
break;
} else {
console.error(`Error: Unknown subcommand '${subCmdName}' for command '${cmdName}'.`);
console.log(listSubcommands(cmdName, cmdDef));
process.exitCode = 1;
break;
}
} else {
console.log(listSubcommands(cmdName, cmdDef));
break;
}
} else {
console.log(detailedUsageForCommand(cmdName));
break;
}
} else {
console.error(`Error: Unknown command '${cmdName}' for help.`);
process.exitCode = 1;
}
} else {
console.log(
`Hugging Face CLI Tools (hfjs)
Available commands:
` + typedEntries(commands).map(([name, def]) => ` ${usage(name)}: ${def.description}`).join("\n")
);
console.log("\nTo get help on a specific command, run `hfjs help <command>` or `hfjs <command> --help`");
console.log(
"For commands with subcommands (like 'branch'), run `hfjs help <command> <subcommand>` or `hfjs <command> <subcommand> --help`"
);
if (mainCommandName === void 0) {
process.exitCode = 1;
}
}
break;
}
case "upload": {
const cmdDef = commands.upload;
if (cliArgs[0] === "--help" || cliArgs[0] === "-h") {
console.log(detailedUsageForCommand("upload"));
break;
}
const parsedArgs = advParseArgs(cliArgs, cmdDef.args, "upload");
const {
repoName,
localFolder,
repoType,
revision,
token,
quiet,
commitMessage,
pathInRepo,
private: isPrivate
} = parsedArgs;
const repoId = repoType ? { type: repoType, name: repoName } : repoName;
if (!await repoExists({ repo: repoId, revision, accessToken: token, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL })) {
if (!quiet) {
console.log(`Repo ${repoName} does not exist. Creating it...`);
}
await createRepo({
repo: repoId,
accessToken: token,
private: !!isPrivate,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
}
const isFile = (await stat(localFolder)).isFile();
const files = isFile ? [
{
content: pathToFileURL(localFolder),
path: join(pathInRepo, `${basename(localFolder)}`).replace(/^[.]?\//, "")
}
] : [{ content: pathToFileURL(localFolder), path: pathInRepo.replace(/^[.]?\//, "") }];
const progressManager = new UploadProgressManager(!!quiet);
await progressManager.initialize();
try {
for await (const event of uploadFilesWithProgress({
repo: repoId,
files,
branch: revision,
accessToken: token,
commitTitle: commitMessage?.trim().split("\n")[0],
commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(),
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL,
useXet: true
})) {
progressManager.handleEvent(event);
}
if (!quiet) {
console.log("\n\u2705 Upload completed successfully!");
}
} catch (error) {
progressManager.stop();
throw error;
} finally {
progressManager.stop();
}
break;
}
case "branch": {
const branchCommandGroup = commands.branch;
const currentSubCommandName = subCommandName;
if (subCommandName === "--help" || subCommandName === "-h") {
console.log(listSubcommands("branch", branchCommandGroup));
break;
}
if (cliArgs[0] === "--help" || cliArgs[0] === "-h") {
if (currentSubCommandName && branchCommandGroup.subcommands[currentSubCommandName]) {
console.log(detailedUsageForSubcommand("branch", currentSubCommandName));
} else {
console.log(listSubcommands("branch", branchCommandGroup));
}
break;
}
if (!currentSubCommandName || !branchCommandGroup.subcommands[currentSubCommandName]) {
console.error(`Error: Missing or invalid subcommand for 'branch'.`);
console.log(listSubcommands("branch", branchCommandGroup));
process.exitCode = 1;
break;
}
const subCmdDef = branchCommandGroup.subcommands[currentSubCommandName];
switch (currentSubCommandName) {
case "create": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch create");
const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs;
await createBranch({
repo: repoType ? { type: repoType, name: repoName } : repoName,
branch,
accessToken: token,
revision,
empty: empty ?? void 0,
overwrite: force ?? void 0,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
console.log(`Branch '${branch}' created successfully in repo '${repoName}'.`);
break;
}
case "delete": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch delete");
const { repoName, branch, repoType, token } = parsedArgs;
await deleteBranch({
repo: repoType ? { type: repoType, name: repoName } : repoName,
branch,
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
console.log(`Branch '${branch}' deleted successfully from repo '${repoName}'.`);
break;
}
default:
console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'branch'.`);
console.log(listSubcommands("branch", branchCommandGroup));
process.exitCode = 1;
break;
}
break;
}
case "repo": {
const repoCommandGroup = commands.repo;
const currentSubCommandName = subCommandName;
if (subCommandName === "--help" || subCommandName === "-h") {
console.log(listSubcommands("repo", repoCommandGroup));
break;
}
if (cliArgs[0] === "--help" || cliArgs[0] === "-h") {
if (currentSubCommandName && repoCommandGroup.subcommands[currentSubCommandName]) {
console.log(detailedUsageForSubcommand("repo", currentSubCommandName));
} else {
console.log(listSubcommands("repo", repoCommandGroup));
}
break;
}
if (!currentSubCommandName || !repoCommandGroup.subcommands[currentSubCommandName]) {
console.error(`Error: Missing or invalid subcommand for 'repo'.`);
console.log(listSubcommands("repo", repoCommandGroup));
process.exitCode = 1;
break;
}
const subCmdDef = repoCommandGroup.subcommands[currentSubCommandName];
switch (currentSubCommandName) {
case "delete": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, `repo ${currentSubCommandName}`);
const { repoName, repoType, token } = parsedArgs;
const repoDesignation = repoType ? { type: repoType, name: repoName } : repoName;
await deleteRepo({
repo: repoDesignation,
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
console.log(`Repository '${repoName}' deleted successfully.`);
break;
}
default:
console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'repo'.`);
console.log(listSubcommands("repo", repoCommandGroup));
process.exitCode = 1;
break;
}
break;
}
case "models": {
const modelCommandGroup = commands.models;
const currentSubCommandName = subCommandName;
if (subCommandName === "--help" || subCommandName === "-h") {
console.log(listSubcommands("models", modelCommandGroup));
break;
}
if (cliArgs[0] === "--help" || cliArgs[0] === "-h") {
if (currentSubCommandName && modelCommandGroup.subcommands[currentSubCommandName]) {
console.log(detailedUsageForSubcommand("models", currentSubCommandName));
} else {
console.log(listSubcommands("models", modelCommandGroup));
}
break;
}
if (!currentSubCommandName || !modelCommandGroup.subcommands[currentSubCommandName]) {
console.error(`Error: Missing or invalid subcommand for 'models'.`);
console.log(listSubcommands("models", modelCommandGroup));
process.exitCode = 1;
break;
}
const subCmdDef = modelCommandGroup.subcommands[currentSubCommandName];
switch (currentSubCommandName) {
case "list": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "models list");
const { search, sort, token } = parsedArgs;
const models = [];
for await (const model of listModels({
search: search ? { query: search } : void 0,
sort,
limit: 20,
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
})) {
models.push(model);
}
if (models.length === 0) {
console.log("No models found.");
break;
}
console.log(
`${"MODEL".padEnd(45)} ${"TASK".padEnd(25)} ${"DOWNLOADS".padStart(10)} ${"LIKES".padStart(7)} ${"UPDATED"}`
);
console.log("-".repeat(110));
for (const model of models) {
const task = model.task || "N/A";
const updatedAt = model.updatedAt.toLocaleDateString();
console.log(
`${model.name.padEnd(45)} ${task.padEnd(25)} ${String(model.downloads).padStart(10)} ${String(model.likes).padStart(7)} ${updatedAt}`
);
}
break;
}
default:
console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'models'.`);
console.log(listSubcommands("models", modelCommandGroup));
process.exitCode = 1;
break;
}
break;
}
case "version": {
if (cliArgs[0] === "--help" || cliArgs[0] === "-h") {
console.log(detailedUsageForCommand("version"));
break;
}
console.log(`hfjs version: ${version}`);
break;
}
case "jobs": {
const jobsCommandGroup = commands.jobs;
const currentSubCommandName = subCommandName;
if (subCommandName === "--help" || subCommandName === "-h") {
console.log(listSubcommands("jobs", jobsCommandGroup));
break;
}
if (cliArgs[0] === "--help" || cliArgs[0] === "-h") {
if (currentSubCommandName && jobsCommandGroup.subcommands[currentSubCommandName]) {
console.log(detailedUsageForSubcommand("jobs", currentSubCommandName));
} else {
console.log(listSubcommands("jobs", jobsCommandGroup));
}
break;
}
if (!currentSubCommandName || !jobsCommandGroup.subcommands[currentSubCommandName]) {
console.error(`Error: Missing or invalid subcommand for 'jobs'.`);
console.log(listSubcommands("jobs", jobsCommandGroup));
process.exitCode = 1;
break;
}
const subCmdDef = jobsCommandGroup.subcommands[currentSubCommandName];
switch (currentSubCommandName) {
case "run": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs run");
const {
dockerImageOrSpace: firstArg,
command: commandArray,
env,
secret,
label,
volume: volumeArgs,
flavor,
attempts: attemptsStr,
namespace,
token,
detach
} = parsedArgs;
const envVars = env;
const secretVars = secret;
const labelVars = label;
let attempts;
if (attemptsStr) {
const parsed = parseInt(attemptsStr, 10);
if (isNaN(parsed) || parsed < 1) {
throw new Error("Attempts must be a positive integer");
}
attempts = parsed;
}
let dockerImage;
let spaceId;
const hfCoSpacesMatch = firstArg.match(/^hf\.co\/spaces\/(.+)$/);
if (hfCoSpacesMatch) {
spaceId = hfCoSpacesMatch[1];
} else {
dockerImage = firstArg;
}
let finalNamespace = namespace;
if (!finalNamespace) {
if (!token) {
throw new Error(
"Cannot determine namespace without authentication. Please provide --namespace or --token."
);
}
const userInfo = await whoAmI({
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
if (userInfo.type !== "user") {
throw new Error("Cannot determine namespace. Please provide --namespace explicitly.");
}
finalNamespace = userInfo.name;
}
const environment = {};
if (envVars) {
for (const envVar of envVars) {
const equalIndex = envVar.indexOf("=");
if (equalIndex === -1) {
throw new Error(`Invalid environment variable format: ${envVar}. Expected KEY=VALUE`);
}
const key = envVar.slice(0, equalIndex);
const value = envVar.slice(equalIndex + 1);
environment[key] = value;
}
}
const secrets = {};
if (secretVars) {
for (const secretVar of secretVars) {
const equalIndex = secretVar.indexOf("=");
if (equalIndex === -1) {
throw new Error(`Invalid secret format: ${secretVar}. Expected KEY=VALUE`);
}
const key = secretVar.slice(0, equalIndex);
const value = secretVar.slice(equalIndex + 1);
secrets[key] = value;
}
}
const labels = {};
if (labelVars) {
for (const labelVar of labelVars) {
const equalIndex = labelVar.indexOf("=");
const [key, value] = equalIndex > -1 ? [labelVar.slice(0, equalIndex), labelVar.slice(equalIndex + 1)] : [labelVar, ""];
labels[key] = value;
}
}
const volumes = [];
if (volumeArgs) {
for (const volumeArg of volumeArgs) {
const colonIdx = volumeArg.indexOf(":");
if (colonIdx === -1) {
throw new Error(`Invalid volume format: ${volumeArg}. Expected SOURCE:MOUNTPATH[:OPTIONS]`);
}
const source = volumeArg.slice(0, colonIdx);
const rest = volumeArg.slice(colonIdx + 1);
const secondColon = rest.indexOf(":");
const mountPath = secondColon === -1 ? rest : rest.slice(0, secondColon);
const optionsStr = secondColon === -1 ? "" : rest.slice(secondColon + 1);
if (!mountPath.startsWith("/")) {
throw new Error(`Volume mountPath must start with "/": ${mountPath}`);
}
let readOnly;
let revision;
let subPath;
if (optionsStr) {
for (const opt of optionsStr.split(",")) {
if (opt === "ro") {
readOnly = true;
} else if (opt.startsWith("revision=")) {
revision = opt.slice("revision=".length);
} else if (opt.startsWith("path=")) {
subPath = opt.slice("path=".length);
} else {
throw new Error(`Unknown volume option: ${opt}`);
}
}
}
volumes.push({
source,
mountPath,
...revision ? { revision } : {},
...readOnly ? { readOnly } : {},
...subPath ? { path: subPath } : {}
});
}
}
const jobParams = {
namespace: finalNamespace,
...dockerImage ? { dockerImage } : {},
...spaceId ? { spaceId } : {},
flavor,
command: commandArray.length > 0 ? commandArray : void 0,
environment,
secrets,
...attempts !== void 0 ? { attempts } : {},
...Object.keys(labels).length > 0 ? { labels } : {},
...volumes.length > 0 ? { volumes } : {},
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL,
...token ? { accessToken: token } : {}
};
const job = await runJob(jobParams);
console.log(`Job created: ${job.id}`);
console.log(`Status: ${job.status.stage}`);
if (!detach) {
const logsParams = {
namespace: finalNamespace,
jobId: job.id,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL,
...token ? { accessToken: token } : {}
};
for await (const logChunk of streamJobLogs(logsParams)) {
console.log(logChunk.message);
}
}
break;
}
case "ps": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs ps");
const { all, namespace, token } = parsedArgs;
let finalNamespace = namespace;
if (!finalNamespace) {
if (!token) {
throw new Error(
"Cannot determine namespace without authentication. Please provide --namespace or --token."
);
}
const userInfo = await whoAmI({
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
if (userInfo.type !== "user") {
throw new Error("Cannot determine namespace. Please provide --namespace explicitly.");
}
finalNamespace = userInfo.name;
}
const jobs = await listJobs({
namespace: finalNamespace,
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
const filteredJobs = all ? jobs : jobs.filter((job) => job.status.stage === "RUNNING");
if (filteredJobs.length === 0) {
console.log(all ? "No jobs found." : "No running jobs found.");
break;
}
console.log(`${"ID".padEnd(40)} ${"STATUS".padEnd(12)} ${"CREATED".padEnd(20)} ${"DOCKER IMAGE"}`);
console.log("-".repeat(100));
for (const job of filteredJobs) {
const createdAt = new Date(job.createdAt).toLocaleString();
const dockerImage = job.dockerImage || job.spaceId || "N/A";
const status = job.status.stage;
console.log(`${job.id.padEnd(40)} ${status.padEnd(12)} ${createdAt.padEnd(20)} ${dockerImage}`);
}
break;
}
case "hardware": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs hardware");
const { token } = parsedArgs;
const hardwareParams = {
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
};
if (token) {
hardwareParams.accessToken = token;
}
const hardware = await listJobHardware(hardwareParams);
console.log(
`${"NAME".padEnd(15)} ${"PRETTY NAME".padEnd(22)} ${"CPU".padEnd(8)} ${"RAM".padEnd(7)} ${"ACCELERATOR".padEnd(16)} ${"COST/MIN".padEnd(9)} ${"COST/HOUR"}`
);
console.log("-".repeat(100));
for (const hw of hardware) {
let accelerator = "N/A";
if (hw.accelerator) {
accelerator = `${hw.accelerator.quantity}x ${hw.accelerator.model} (${hw.accelerator.vram})`;
}
const costPerMin = (hw.unitCostMicroUSD / 1e6).toFixed(4);
const costPerHour = (hw.unitCostMicroUSD / 1e6 * 60).toFixed(2);
console.log(
`${hw.name.padEnd(15)} ${hw.prettyName.padEnd(22)} ${hw.cpu.padEnd(8)} ${hw.ram.padEnd(7)} ${accelerator.padEnd(16)} $${costPerMin.padStart(8)} $${costPerHour.padStart(9)}`
);
}
break;
}
case "logs": {
const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "jobs logs");
const { jobId, namespace, token } = parsedArgs;
let finalNamespace = namespace;
if (!finalNamespace) {
if (!token) {
throw new Error(
"Cannot determine namespace without authentication. Please provide --namespace or --token."
);
}
const userInfo = await whoAmI({
accessToken: token,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL
});
if (userInfo.type !== "user") {
throw new Error("Cannot determine namespace. Please provide --namespace explicitly.");
}
finalNamespace = userInfo.name;
}
const logsParams = {
namespace: finalNamespace,
jobId,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL,
...token ? { accessToken: token } : {}
};
const jobInfoParams = {
namespace: finalNamespace,
jobId,
hubUrl: process.env.HF_ENDPOINT ?? HUB_URL,
...token ? { accessToken: token } : {}
};
const jobInfo = await getJob(jobInfoParams);
if (jobInfo.status.stage === "ERROR" && jobInfo.status.message) {
console.error(`
\u274C Job failed: ${jobInfo.status.message}
`);
}
for await (const logChunk of streamJobLogs(logsParams)) {
console.log(logChunk.message);
}
break;
}
default:
console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'jobs'.`);
console.log(listSubcommands("jobs", jobsCommandGroup));
process.exitCode = 1;
break;
}
break;
}
default:
console.error("Command not found: " + mainCommandName);
console.log(
`
Available commands:
` + typedEntries(commands).map(([name, def]) => ` ${usage(name)}: ${def.description}`).join("\n")
);
console.log("\nTo get help on a specific command, run `hfjs help <command>` or `hfjs <command> --help`");
process.exitCode = 1;
break;
}
}
run().catch((err) => {
console.error("\x1B[31mError:\x1B[0m", err.message);
console.error(err);
process.exitCode = 1;
});
function usage(commandName, subCommandName2) {
const commandEntry = commands[commandName];
let cmdArgs;
let fullCommandName = commandName;
if ("subcommands" in commandEntry) {
if (subCommandName2 && subCommandName2 in commandEntry.subcommands) {
const subCmd = commandEntry.subcommands[subCommandName2];
cmdArgs = subCmd.args;
fullCommandName = `${commandName} ${subCommandName2}`;
} else {
return `${commandName} <subcommand>`;
}
} else {
cmdArgs = commandEntry.args;
}
return `${fullCommandName} ${(cmdArgs || []).map((arg) => {
if (arg.positional) {
return arg.required ? `<${arg.name}>` : `[${arg.name}]`;
}
return `[--${arg.name}${arg.short ? `|-${arg.short}` : ""}${arg.enum ? ` {${arg.enum.join("|")}}` : arg.boolean ? "" : ` <${arg.name.toUpperCase().replace(/-/g, "_")}>`}]`;
}).join(" ")}`.trim();
}
function _detailedUsage(args, usageLine, commandDescription) {
let ret = `usage: hfjs ${usageLine}
`;
if (commandDescription) {
ret += `
${commandDescription}
`;
}
const positionals = args.filter((p) => p.positional);
const options = args.filter((p) => !p.positional);
if (positionals.length > 0) {
ret += `
Positional arguments:
`;
for (const arg of positionals) {
ret += ` ${arg.name} ${arg.description}${arg.default ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : ""}
`;
}
}
if (options.length > 0) {
ret += `
Options:
`;
for (const arg of options) {
const nameAndAlias = `--${arg.name}${arg.short ? `, -${arg.short}` : ""}`;
const valueHint = arg.enum ? `{${arg.enum.join("|")}}` : arg.boolean ? "" : `<${arg.name.toUpperCase().replace(/-/g, "_")}>`;
ret += ` ${nameAndAlias}${valueHint ? " " + valueHint : ""} ${arg.description}${arg.default !== void 0 ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : ""}
`;
}
}
ret += `
`;
return ret;
}
function detailedUsageForCommand(commandName) {
const commandDef = commands[commandName];
if ("subcommands" in commandDef) {
return listSubcommands(commandName, commandDef);
}
return _detailedUsage(commandDef.args, usage(commandName), commandDef.description);
}
function detailedUsageForSubcommand(commandName, subCommandName2) {
const commandGroup = commands[commandName];
if (!("subcommands" in commandGroup)) {
throw new Error(`Command ${commandName} does not have subcommands`);
}
if (!(subCommandName2 in commandGroup.subcommands)) {
throw new Error(`Subcommand ${subCommandName2} not found for ${commandName}`);
}
const subCommandDef = commandGroup.subcommands[subCommandName2];
return _detailedUsage(subCommandDef.args, usage(commandName, subCommandName2), subCommandDef.description);
}
function listSubcommands(commandName, commandGroup) {
let ret = `usage: hfjs ${commandName} <subcommand> [options]
`;
ret += `${commandGroup.description}
`;
ret += `Available subcommands for '${commandName}':
`;
ret += typedEntries(commandGroup.subcommands).map(([subName, subDef]) => ` ${subName} ${subDef.description}`).join("\n");
if (commandName === "jobs") {
ret += `
Example:
hfjs jobs run -e FOO=foo -e BAR=bar python:3.12 -- python -c 'import os; print(os.environ["FOO"], os.environ["BAR"])'`;
}
ret += `
Run \`hfjs help ${commandName} <subcommand>\` for more information on a specific subcommand.`;
return ret;
}
function advParseArgs(args, argDefs, commandNameForError) {
const hasMultiplePositional = argDefs.some((arg) => arg.multiple && arg.positional);
const { tokens } = parseArgs({
options: Object.fromEntries(
argDefs.filter((arg) => !arg.positional).map((arg) => {
const optionConfig = {
type: arg.boolean ? "boolean" : "string",
...arg.short && { short: arg.short },
...arg.multiple && { multiple: true },
...arg.default !== void 0 && {
default: typeof arg.default === "function" ? arg.default() : arg.default
}
};
return [arg.name, optionConfig];
})
),
args,
allowPositionals: true,
strict: false,
// We do custom validation based on tokens and argDefs
tokens: true
});
const expectedPositionals = argDefs.filter((arg) => arg.positional);
const providedPositionalTokens = tokens.filter((token) => token.kind === "positional");
if (providedPositionalTokens.length < expectedPositionals.filter((arg) => arg.required).length) {
throw new Error(
`Command '${commandNameForError}': Missing required positional arguments. Usage: hfjs ${usage(
commandNameForError.split(" ")[0],
commandNameForError.split(" ")[1]
)}`
);
}
if (providedPositionalTokens.length > expectedPositionals.length && !hasMultiplePositional) {
throw new Error(
`Command '${commandNameForError}': Too many positional arguments. Usage: hfjs ${usage(
commandNameForError.split(" ")[0],
commandNameForError.split(" ")[1]
)}`
);
}
const result = {};
for (const argDef of argDefs) {
if (argDef.default !== void 0) {
result[argDef.name] = typeof argDef.default === "function" ? argDef.default() : argDef.default;
} else if (argDef.boolean) {
result[argDef.name] = false;
}
}
expectedPositionals.forEach((argDef, i) => {
if (argDef.multiple) {
result[argDef.name] = providedPositionalTokens.slice(i).map((token) => token.value);
} else if (providedPositionalTokens[i]) {
result[argDef.name] = providedPositionalTokens[i].value;
}
});
tokens.filter((token) => token.kind === "option").forEach((token) => {
const argDef = argDefs.find((def) => def.name === token.name || def.short === token.name);
if (!argDef) {
throw new Error(`Command '${commandNameForError}': Unknown option: ${token.rawName}`);
}
if (argDef.boolean) {
result[argDef.name] = true;
} else {
if (token.value === void 0) {
throw new Error(`Command '${commandNameForError}': Missing value for option: ${token.rawName}`);
}
if (argDef.enum && !argDef.enum.includes(token.value)) {
throw new Error(
`Command '${commandNameForError}': Invalid value '${token.value}' for option ${token.rawName}. Expected one of: ${argDef.enum.join(", ")}`
);
}
if (argDef.multiple) {
const existing = result[argDef.name] || [];
existing.push(token.value);
result[argDef.name] = existing;
} else {
result[argDef.name] = token.value;
}
}
});
for (const argDef of argDefs) {
if (argDef.required && result[argDef.name] === void 0) {
throw new Error(`Command '${commandNameForError}': Missing required argument: ${argDef.name}`);
}
}
return Object.fromEntries(
Object.entries(result).map(([name, val]) => [kebabToCamelCase(name), val])
);
}
function kebabToCamelCase(str) {
return str.replace(/-./g, (match) => match[1].toUpperCase());
}