UNPKG

curlconverter

Version:

convert curl commands to Python, JavaScript, Go, PHP and more

1,650 lines (1,514 loc) 47.9 kB
import { Word, eq, mergeWords, joinWords } from "./shell/Word.js"; import { CCError, has, isInt } from "./utils.js"; import { warnf, warnIfPartsIgnored } from "./Warnings.js"; import type { Warnings, Support } from "./Warnings.js"; import type { GlobalConfig, OperationConfig, SrcDataParam, } from "./curl/opts.js"; import { Headers, parseCookies, parseCookiesStrict } from "./Headers.js"; import type { Cookies } from "./Headers.js"; import { pickAuth, type AuthType } from "./curl/auth.js"; export { AuthType } from "./curl/auth.js"; import { parseurl, type Curl_URL } from "./curl/url.js"; import { parseQueryString, percentEncodePlus } from "./Query.js"; import type { QueryList, QueryDict } from "./Query.js"; import { parseForm } from "./curl/form.js"; import type { FormParam } from "./curl/form.js"; export type FileParamType = "data" | "binary" | "urlencode" | "json"; export type DataType = FileParamType | "raw"; export type FileDataParam = { filetype: FileParamType; // The left side of "=" for --data-urlencode, can't be empty string name?: Word; filename: Word; }; // "raw"-type SrcDataParams, and `FileParamType`s that read from stdin // when we have its contents (because it comes from a pipe) are converted // to plain strings export type DataParam = Word | FileDataParam; // struct getout // https://github.com/curl/curl/blob/curl-7_86_0/src/tool_sdecls.h#L96 export interface RequestUrl { // What it looked like in the input, used for error messages originalUrl: Word; url: Word; // the query string can contain instructions to // read the query string from a file, for example with // --url-query @filename // In that case we put "@filename" in the query string and "filename" here and // warn the user that they'll need to modify the code to read that file. queryReadsFile?: string; urlObj: Curl_URL; // If the ?query can't be losslessly parsed, then // .urlWithoutQueryList === .url // .queryList === undefined urlWithoutQueryList: Word; queryList?: QueryList; // When all (if any) repeated keys in queryList happen one after the other // ?a=1&a=1&b=2 (okay) // ?a=1&b=2&a=1 (doesn't work, queryList is defined but queryDict isn't) queryDict?: QueryDict; urlWithoutQueryArray: Word; urlWithOriginalQuery: Word; // This includes the query in the URL and the query that comes from `--get --data` or `--url-query` queryArray?: DataParam[]; // This is only the query in the URL urlQueryArray?: DataParam[]; uploadFile?: Word; output?: Word; method: Word; auth?: [Word, Word]; // TODO: should authType be per-url as well? // authType?: string; } export type ProxyType = | "http1" | "http2" | "socks4" | "socks4a" | "socks5" | "socks5-hostname"; export interface Request { // Will have at least one element (otherwise an error is raised) urls: RequestUrl[]; globoff?: boolean; disallowUsernameInUrl?: boolean; pathAsIs?: boolean; // Just the part that comes from `--get --data` or `--url-query` (not the query in the URL) // unless there's only one URL, then it will include both. queryArray?: DataParam[]; authType: AuthType; proxyAuthType: AuthType; awsSigV4?: Word; oauth2Bearer?: Word; delegation?: Word; krb?: Word; saslAuthzid?: Word; saslIr?: boolean; serviceName?: Word; // A null header means the command explicitly disabled sending this header headers: Headers; proxyHeaders: Headers; refererAuto?: boolean; // .cookies is a parsed version of the Cookie header, if it can be parsed. // Generators that use .cookies need to delete the header from .headers (usually). cookies?: Cookies; cookieFiles?: Word[]; cookieJar?: Word; junkSessionCookies?: boolean; compressed?: boolean; transferEncoding?: boolean; include?: boolean; multipartUploads?: FormParam[]; // When multipartUploads comes from parsing a string in --data // this can be set to true to say that sending the original data // as a string would be more correct. multipartUploadsDoesntRoundtrip?: boolean; formEscape?: boolean; dataArray?: DataParam[]; data?: Word; dataReadsFile?: string; isDataBinary?: boolean; isDataRaw?: boolean; ipv4?: boolean; ipv6?: boolean; proto?: Word; protoRedir?: Word; protoDefault?: Word; tcpFastopen?: boolean; localPort?: [Word, Word | null]; ignoreContentLength?: boolean; interface?: Word; ciphers?: Word; curves?: Word; insecure?: boolean; certStatus?: boolean; cert?: [Word, Word | null]; certType?: Word; key?: Word; keyType?: Word; pass?: Word; cacert?: Word; caNative?: boolean; sslAllowBeast?: boolean; capath?: Word; crlfile?: Word; pinnedpubkey?: Word; randomFile?: Word; egdFile?: Word; hsts?: Word[]; // a filename alpn?: boolean; tlsVersion?: "1" | "1.0" | "1.1" | "1.2" | "1.3"; tlsMax?: Word; tls13Ciphers?: Word; tlsauthtype?: Word; tlspassword?: Word; tlsuser?: Word; sslAutoClientCert?: boolean; sslNoRevoke?: boolean; sslReqd?: boolean; sslRevokeBestEffort?: boolean; ssl?: boolean; sslv2?: boolean; sslv3?: boolean; dohUrl?: Word; dohInsecure?: boolean; dohCertStatus?: boolean; proxy?: Word; proxyType?: ProxyType; proxyAuth?: Word; proxytunnel?: boolean; noproxy?: Word; // a list of hosts or "*" preproxy?: Word; proxyAnyauth?: boolean; proxyBasic?: boolean; proxyCaNative?: boolean; proxyCacert?: Word; // <file> proxyCapath?: Word; // <dir> proxyCertType?: Word; // <type> proxyCert?: Word; // <cert[:passwd]> proxyCiphers?: Word; // <list> proxyCrlfile?: Word; // <file> proxyDigest?: boolean; proxyHttp2?: boolean; proxyInsecure?: boolean; proxyKeyType?: Word; // <type> proxyKey?: Word; // <key> proxyNegotiate?: boolean; proxyNtlm?: boolean; proxyPass?: Word; // <phrase> proxyPinnedpubkey?: Word; // <hashes> proxyServiceName?: Word; // <name> proxySslAllowBeast?: boolean; proxySslAutoClientCert?: boolean; proxyTls13Ciphers?: Word; // <ciphersuite list> proxyTlsauthtype?: Word; // <type> proxyTlspassword?: Word; // <string> proxyTlsuser?: Word; // <name> proxyTlsv1?: boolean; proxyUser?: Word; // <user:password> proxy1?: boolean; // <host[:port]> socks4?: Word; socks4a?: Word; socks5?: Word; socks5Basic?: boolean; socks5GssapiNec?: boolean; socks5GssapiService?: Word; socks5Gssapi?: boolean; socks5Hostname?: Word; haproxyClientIp?: Word; haproxyProtocol?: boolean; timeout?: Word; // a decimal, seconds connectTimeout?: Word; // a decimal, seconds expect100Timeout?: Word; // a decimal, seconds happyEyeballsTimeoutMs?: Word; // an integer, milliseconds speedLimit?: Word; // an integer speedTime?: Word; // an integer limitRate?: Word; // an integer with an optional unit maxFilesize?: Word; // an intger with an optional unit continueAt?: Word; // an integer or "-" crlf?: boolean; useAscii?: boolean; remoteTime?: boolean; clobber?: boolean; ftpSkipPasvIp?: boolean; fail?: boolean; retry?: Word; // an integer, how many times to retry retryMaxTime?: Word; // an integer, seconds keepAlive?: boolean; keepAliveTime?: Word; // an integer, seconds altSvc?: Word; followRedirects?: boolean; followRedirectsTrusted?: boolean; maxRedirects?: Word; // an integer post301?: boolean; post302?: boolean; post303?: boolean; httpVersion?: "1.0" | "1.1" | "2" | "2-prior-knowledge" | "3" | "3-only"; http0_9?: boolean; http2?: boolean; http3?: boolean; stdin?: Word; stdinFile?: Word; resolve?: Word[]; // a list of host:port:address connectTo?: Word[]; // a list of host:port:connect-to-host:connect-to-port unixSocket?: Word; abstractUnixSocket?: Word; netrc?: "optional" | "required" | "ignored"; // undefined means implicitly "ignored" netrcFile?: Word; // if undefined defaults to ~/.netrc // Global options verbose?: boolean; silent?: boolean; } function buildURL( global_: GlobalConfig, config: OperationConfig, url: Word, uploadFile?: Word, outputFile?: Word, stdin?: Word, stdinFile?: Word, ): RequestUrl { const originalUrl = url; const u = parseurl(global_, config, url); // https://github.com/curl/curl/blob/curl-7_85_0/src/tool_operate.c#L1124 // https://github.com/curl/curl/blob/curl-7_85_0/src/tool_operhlp.c#L76 if (uploadFile) { // TODO: it's more complicated if (u.path.isEmpty()) { u.path = uploadFile.prepend("/"); } else if (u.path.endsWith("/")) { u.path = u.path.add(uploadFile); } if (config.get) { warnf(global_, [ "data-ignored", "curl doesn't let you pass --get and --upload-file together", ]); } } const urlWithOriginalQuery = mergeWords( u.scheme, "://", u.host, u.path, u.query, u.fragment, ); // curl example.com example.com?foo=bar --url-query isshared=t // will make requests for // example.com/?isshared=t // example.com/?foo=bar&isshared=t // // so the query could come from // 1. `--url` (i.e. the requested URL) // 2. `--url-query` or `--get --data` (the latter takes precedence) // // If it comes from the latter, we might need to generate code to read // from one or more files. // When there's multiple urls, the latter applies to all of them // but the query from --url only applies to that URL. // // There's 3 cases for the query: // 1. it's well-formed and can be expressed as a list of tuples (or a dict) // `?one=1&one=1&two=2` // 2. it can't, for example because one of the pieces doesn't have a '=' // `?one` // 3. we need to generate code that reads from a file // // If there's only one URL we merge the query from the URL with the shared part. // // If there's multiple URLs and a shared part that reads from a file (case 3), // we only write the file reading code once, pass it as the params= argument // and the part from the URL has to be passed as a string in the URL // and requests will combine the query in the URL with the query in params=. // // Otherwise, we print each query for each URL individually, either as a // list of tuples if we can or in the URL if we can't. // // When files are passed in through --data-urlencode or --url-query // we can usually treat them as case 1 as well (in Python), but that would // generate code slightly different from curl because curl reads the file once // upfront, whereas we would read the file multiple times and it might contain // different data each time (for example if it's /dev/urandom). let urlQueryArray: DataParam[] | null = null; let queryArray: DataParam[] | null = null; let queryStrReadsFile: string | null = null; if (u.query.toBool() || (config["url-query"] && config["url-query"].length)) { let queryStr: Word | null = null; let queryParts: SrcDataParam[] = []; if (u.query.toBool()) { // remove the leading '?' queryParts.push(["raw", u.query.slice(1)]); [queryArray, queryStr, queryStrReadsFile] = buildData( queryParts, stdin, stdinFile, ); urlQueryArray = queryArray; } if (config["url-query"]) { queryParts = queryParts.concat(config["url-query"]); [queryArray, queryStr, queryStrReadsFile] = buildData( queryParts, stdin, stdinFile, ); } // TODO: check the curl source code // TODO: curl localhost:8888/? // will request /? // but // curl localhost:8888/? --url-query '' // (or --get --data '') will request / u.query = new Word(); if (queryStr && queryStr.toBool()) { u.query = queryStr.prepend("?"); } } const urlWithoutQueryArray = mergeWords( u.scheme, "://", u.host, u.path, u.fragment, ); url = mergeWords(u.scheme, "://", u.host, u.path, u.query, u.fragment); let urlWithoutQueryList = url; // TODO: parseQueryString() doesn't accept leading '?' let [queryList, queryDict] = parseQueryString( u.query.toBool() ? u.query.slice(1) : new Word(), ); if (queryList && queryList.length) { // TODO: remove the fragment too? urlWithoutQueryList = mergeWords( u.scheme, "://", u.host, u.path, u.fragment, ); } else { queryList = null; queryDict = null; } // TODO: --path-as-is // TODO: --request-target // curl expects you to uppercase methods always. If you do -X PoSt, that's what it // will send, but most APIs will helpfully uppercase what you pass in as the method. // // There are many places where curl determines the method, this is the last one: // https://github.com/curl/curl/blob/curl-7_85_0/lib/http.c#L2032 let method = new Word("GET"); if ( config.request && // Safari adds `-X null` if it can't determine the request type // https://github.com/WebKit/WebKit/blob/f58ef38d48f42f5d7723691cb090823908ff5f9f/Source/WebInspectorUI/UserInterface/Models/Resource.js#L1250 !eq(config.request, "null") ) { method = config.request; } else if (config.head) { method = new Word("HEAD"); } else if (uploadFile && uploadFile.toBool()) { // --upload-file '' doesn't do anything. method = new Word("PUT"); } else if (!config.get && (has(config, "data") || has(config, "form"))) { method = new Word("POST"); } const requestUrl: RequestUrl = { originalUrl, urlWithoutQueryList, url, urlObj: u, urlWithOriginalQuery, urlWithoutQueryArray, method, }; if (queryStrReadsFile) { requestUrl.queryReadsFile = queryStrReadsFile; } if (queryList) { requestUrl.queryList = queryList; if (queryDict) { requestUrl.queryDict = queryDict; } } if (queryArray) { requestUrl.queryArray = queryArray; } if (urlQueryArray) { requestUrl.urlQueryArray = urlQueryArray; } if (uploadFile) { if (eq(uploadFile, "-") || eq(uploadFile, ".")) { if (stdinFile) { requestUrl.uploadFile = stdinFile; } else if (stdin) { warnf(global_, [ "upload-file-with-stdin-content", "--upload-file with stdin content is not supported", ]); requestUrl.uploadFile = uploadFile; // TODO: this is complicated, // --upload-file only applies per-URL so .data needs to become per-URL... // if you pass --data and --upload-file or --get and --upload-file, curl will error // if (config.url && config.url.length === 1) { // config.data = [["raw", stdin]]; // } else { // warnf(global_, [ // "upload-file-with-stdin-content-and-multiple-urls", // "--upload-file with stdin content and multiple URLs is not supported", // ]); // } } else { requestUrl.uploadFile = uploadFile; } } else { requestUrl.uploadFile = uploadFile; } } if (outputFile) { // TODO: get stdout redirects of command requestUrl.output = outputFile; } // --user takes precedence over the URL const auth = config.user || u.auth; if (auth) { const [user, pass] = auth.split(":", 2); requestUrl.auth = [user, pass || new Word()]; } return requestUrl; } function buildData( configData: SrcDataParam[], stdin?: Word, stdinFile?: Word, ): [DataParam[], Word, string | null] { const data: DataParam[] = []; let dataStrState = new Word(); for (const [i, x] of configData.entries()) { const type = x[0]; let value = x[1]; let name: Word | null = null; if (i > 0 && type !== "json") { dataStrState = dataStrState.append("&"); } if (type === "urlencode") { // curl checks for = before @ const splitOn = value.includes("=") || !value.includes("@") ? "=" : "@"; // If there's no = or @ then the entire content is treated as a value and encoded if (value.includes("@") || value.includes("=")) { [name, value] = value.split(splitOn, 2); } if (splitOn === "=") { if (name && name.toBool()) { dataStrState = dataStrState.add(name).append("="); } // curl's --data-urlencode percent-encodes spaces as "+" // https://github.com/curl/curl/blob/curl-7_86_0/src/tool_getparam.c#L630 dataStrState = dataStrState.add(percentEncodePlus(value)); continue; } name = name && name.toBool() ? name : null; value = value.prepend("@"); } let filename: Word | null = null; if (type !== "raw" && value.startsWith("@")) { filename = value.slice(1); if (eq(filename, "-")) { if (stdin !== undefined) { switch (type) { case "binary": case "json": value = stdin; break; case "urlencode": value = mergeWords( name && name.length ? name.append("=") : new Word(), percentEncodePlus(stdin), ); break; default: value = stdin.replace(/[\n\r]/g, ""); } filename = null; } else if (stdinFile !== undefined) { filename = stdinFile; } else { // TODO: if stdin is read twice, it will be empty the second time // TODO: `STDIN_SENTINEL` so that we can tell the difference between // a stdinFile called "-" and stdin for the error message } } } if (filename !== null) { if (dataStrState.toBool()) { data.push(dataStrState); dataStrState = new Word(); } const dataParam: DataParam = { // If `filename` isn't null, then `type` can't be "raw" filetype: type as FileParamType, filename, }; if (name) { dataParam.name = name; } data.push(dataParam); } else { dataStrState = dataStrState.add(value); } } if (dataStrState.toBool()) { data.push(dataStrState); } let dataStrReadsFile: string | null = null; const dataStr = mergeWords( ...data.map((d) => { if (!(d instanceof Word)) { dataStrReadsFile ||= d.filename.toString(); // report first file if (d.name) { return mergeWords(d.name, "=@", d.filename); } return d.filename.prepend("@"); } return d; }), ); return [data, dataStr, dataStrReadsFile]; } // Parses a Content-Type header into a type and a list of parameters function parseContentType( string: string, ): [string, Array<[string, string]>] | null { if (!string.includes(";")) { return [string, []]; } const semi = string.indexOf(";"); const type = string.slice(0, semi); const rest = string.slice(semi); // See https://www.w3.org/Protocols/rfc1341/4_Content-Type.html // TODO: could be better, like reading to the next semicolon const params = rest.match( /;\s*([^;=]+)=(?:("[^"]*")|([^()<>@,;:\\"/[\]?.=]*))/g, ); if (rest.trim() && !params) { return null; } const parsedParams: Array<[string, string]> = []; for (const param of params || []) { const parsedParam = param.match( /;\s*([^;=]+)=(?:("[^"]*")|([^()<>@,;:\\"/[\]?.=]*))/, ); if (!parsedParam) { return null; } const name = parsedParam[1]; const value = parsedParam[3] || parsedParam[2].slice(1, -1); parsedParams.push([name, value]); } return [type, parsedParams]; } // Parses out the boundary= value from a Content-Type header function parseBoundary(string: string): string | null { const header = parseContentType(string); if (!header) { return null; } for (const [name, value] of header[1]) { if (name === "boundary") { return value; } } return null; } function parseRawForm( data: string, boundary: string, ): [FormParam[], boolean] | null { const endBoundary = "\r\n--" + boundary + "--\r\n"; if (!data.endsWith(endBoundary)) { return null; } data = data.slice(0, -endBoundary.length); // TODO: if empty form should it be completely empty? boundary = "--" + boundary + "\r\n"; if (data && !data.startsWith(boundary)) { return null; } data = data.slice(boundary.length); const parts = data.split("\r\n" + boundary); const form: FormParam[] = []; let roundtrips = true; for (const part of parts) { const lines = part.split("\r\n"); if (lines.length < 2) { return null; } const formParam: FormParam = { name: new Word(), content: new Word(), }; let seenContentDisposition = false; const headers: Word[] = []; let i = 0; for (; i < lines.length; i++) { if (lines[i].length === 0) { break; } const [name, value] = lines[i].split(": ", 2); if (name === undefined || value === undefined) { return null; } if (name.toLowerCase() === "content-disposition") { if (seenContentDisposition) { // should only have one return null; } const contentDisposition = parseContentType(value); if (!contentDisposition) { return null; } const [type, params] = contentDisposition; if (type !== "form-data") { return null; } let extra = 0; for (const [paramName, paramValue] of params) { switch (paramName) { case "name": formParam.name = new Word(paramValue); break; case "filename": formParam.filename = new Word(paramValue); break; default: extra++; break; } } if (extra) { roundtrips = false; // TODO: warn? } seenContentDisposition = true; } else if (name.toLowerCase() === "content-type") { formParam.contentType = new Word(value); } else { headers.push(new Word(lines[i])); } } if (headers.length) { formParam.headers = headers; } if (!seenContentDisposition) { return null; } if (i === lines.length) { return null; } if (formParam.name.isEmpty()) { return null; } formParam.content = new Word(lines.slice(i + 1).join("\n")); form.push(formParam); } return [form, roundtrips]; } function buildRequest( global_: GlobalConfig, config: OperationConfig, stdin?: Word, stdinFile?: Word, ): Request { if (!config.url || !config.url.length) { // TODO: better error message (could be parsing fail) throw new CCError("no URL specified!"); } const headers = new Headers(config.header, global_.warnings); const proxyHeaders = new Headers( config["proxy-header"], global_.warnings, "--proxy-header", ); let cookies; const cookieFiles: Word[] = []; const cookieHeader = headers.get("cookie"); if (cookieHeader) { const parsedCookies = parseCookiesStrict(cookieHeader); if (parsedCookies) { cookies = parsedCookies; } } else if (cookieHeader === undefined && config.cookie) { // If there is a Cookie header, --cookies is ignored const cookieStrings: Word[] = []; for (const c of config.cookie) { // a --cookie without a = character reads from it as a filename if (c.includes("=")) { cookieStrings.push(c); } else { cookieFiles.push(c); } } if (cookieStrings.length) { const cookieString = joinWords(config.cookie, "; "); headers.setIfMissing("Cookie", cookieString); const parsedCookies = parseCookies(cookieString); if (parsedCookies) { cookies = parsedCookies; } } } let refererAuto = false; if (config["user-agent"]) { headers.setIfMissing("User-Agent", config["user-agent"]); } if (config.referer) { if (config.referer.includes(";auto")) { refererAuto = true; } // referer can be ";auto" or followed by ";auto", we ignore that. const referer = config.referer.replace(/;auto$/, ""); if (referer.length) { headers.setIfMissing("Referer", referer); } } if (config.range) { let range = config.range.prepend("bytes="); if (!range.includes("-")) { range = range.append("-"); } headers.setIfMissing("Range", range); } if (config["time-cond"]) { let timecond = config["time-cond"]; let header = "If-Modified-Since"; switch (timecond.charAt(0)) { case "+": timecond = timecond.slice(1); break; case "-": timecond = timecond.slice(1); header = "If-Unmodified-Since"; break; case "=": timecond = timecond.slice(1); header = "Last-Modified"; break; } // TODO: parse date headers.setIfMissing(header, timecond); } let data; let dataStr; let dataStrReadsFile; let queryArray; if (config.data && config.data.length) { if (config.get) { // https://github.com/curl/curl/blob/curl-7_85_0/src/tool_operate.c#L721 // --get --data will overwrite --url-query, but if there's no --data, for example, // curl --url-query bar --get example.com // it won't // https://daniel.haxx.se/blog/2022/11/10/append-data-to-the-url-query/ config["url-query"] = config.data; delete config.data; } else { [data, dataStr, dataStrReadsFile] = buildData( config.data, stdin, stdinFile, ); } } if (config["url-query"]) { [queryArray] = buildData(config["url-query"], stdin, stdinFile); } const urls: RequestUrl[] = []; const uploadFiles = config["upload-file"] || []; const outputFiles = config.output || []; for (const [i, url] of config.url.entries()) { urls.push( buildURL( global_, config, url, uploadFiles[i], outputFiles[i], stdin, stdinFile, ), ); } // --get moves --data into the URL's query string if (config.get && config.data) { delete config.data; } if ((config["upload-file"] || []).length > config.url.length) { warnf(global_, [ "too-many-upload-files", "Got more --upload-file/-T options than URLs: " + config["upload-file"] ?.map((f) => JSON.stringify(f.toString())) .join(", "), ]); } if ((config.output || []).length > config.url.length) { warnf(global_, [ "too-many-output-files", "Got more --output/-o options than URLs: " + config.output?.map((f) => JSON.stringify(f.toString())).join(", "), ]); } const request: Request = { urls, authType: pickAuth(config.authtype), proxyAuthType: pickAuth(config.proxyauthtype), headers, proxyHeaders, }; // TODO: warn about unused stdin? if (stdin) { request.stdin = stdin; } if (stdinFile) { request.stdinFile = stdinFile; } if (Object.prototype.hasOwnProperty.call(config, "globoff")) { request.globoff = config.globoff; } if ( Object.prototype.hasOwnProperty.call(config, "disallow-username-in-url") ) { request.disallowUsernameInUrl = config["disallow-username-in-url"]; } if (Object.prototype.hasOwnProperty.call(config, "path-as-is")) { request.pathAsIs = config["path-as-is"]; } if (refererAuto) { request.refererAuto = true; } if (cookies) { // generators that use .cookies need to do // deleteHeader(request, 'cookie') request.cookies = cookies; } if (cookieFiles.length) { request.cookieFiles = cookieFiles; } if (config["cookie-jar"]) { request.cookieJar = config["cookie-jar"]; } if (Object.prototype.hasOwnProperty.call(config, "junk-session-cookies")) { request.junkSessionCookies = config["junk-session-cookies"]; } if (Object.prototype.hasOwnProperty.call(config, "compressed")) { request.compressed = config.compressed; } if (Object.prototype.hasOwnProperty.call(config, "tr-encoding")) { request.transferEncoding = config["tr-encoding"]; } if (config.include) { request.include = true; } if (config.json) { headers.setIfMissing("Content-Type", "application/json"); headers.setIfMissing("Accept", "application/json"); } else if (config.data) { headers.setIfMissing("Content-Type", "application/x-www-form-urlencoded"); } else if (config.form) { // TODO: warn when details (;filename=, etc.) are not supported // by each converter. request.multipartUploads = parseForm(config.form, global_.warnings); //headers.setIfMissing("Content-Type", "multipart/form-data"); } const contentType = headers.getContentType(); const exactContentType = headers.get("Content-Type"); if ( config.data && !dataStrReadsFile && dataStr && dataStr.isString() && !config.form && !request.multipartUploads && contentType === "multipart/form-data" && exactContentType && exactContentType.isString() ) { const boundary = parseBoundary(exactContentType.toString()); if (boundary) { const form = parseRawForm(dataStr.toString(), boundary); if (form) { const [parsedForm, roundtrip] = form; request.multipartUploads = parsedForm; if (!roundtrip) { request.multipartUploadsDoesntRoundtrip = true; } } } } if (Object.prototype.hasOwnProperty.call(config, "form-escape")) { request.formEscape = config["form-escape"]; } if (config["aws-sigv4"]) { // https://github.com/curl/curl/blob/curl-7_86_0/lib/setopt.c#L678-L679 request.authType = "aws-sigv4"; request.awsSigV4 = config["aws-sigv4"]; } if (request.authType === "bearer" && config["oauth2-bearer"]) { const bearer = config["oauth2-bearer"].prepend("Bearer "); headers.setIfMissing("Authorization", bearer); request.oauth2Bearer = config["oauth2-bearer"]; } if (config.delegation) { request.delegation = config.delegation; } if (config.krb) { request.krb = config.krb; } if (config["sasl-authzid"]) { request.saslAuthzid = config["sasl-authzid"]; } if (Object.prototype.hasOwnProperty.call(config, "sasl-ir")) { request.saslIr = config["sasl-ir"]; } if (config.negotiate) { request.authType = "negotiate"; } if (config["service-name"]) { request.serviceName = config["service-name"]; } // TODO: ideally we should generate code that explicitly unsets the header too // no HTTP libraries allow that. headers.clearNulls(); if (config.data && config.data.length) { request.data = dataStr; if (dataStrReadsFile) { request.dataReadsFile = dataStrReadsFile; } request.dataArray = data; // TODO: remove these request.isDataRaw = false; request.isDataBinary = (data || []).some( (d) => !(d instanceof Word) && d.filetype === "binary", ); } if (queryArray) { // If we have to generate code that reads from a file, we // need to do it once for all URLs. request.queryArray = queryArray; } if (Object.prototype.hasOwnProperty.call(config, "ipv4")) { request["ipv4"] = config["ipv4"]; } if (Object.prototype.hasOwnProperty.call(config, "ipv6")) { request["ipv6"] = config["ipv6"]; } if (config.proto) { // TODO: parse request.proto = config.proto; } if (config["proto-redir"]) { // TODO: parse request.protoRedir = config["proto-redir"]; } if (config["proto-default"]) { request.protoDefault = config["proto-default"]; } if (config["tcp-fastopen"]) { request.tcpFastopen = config["tcp-fastopen"]; } if (config["local-port"]) { // TODO: check the range const [start, end] = config["local-port"].split("-", 1); if (end && end.toBool()) { request.localPort = [start, end]; } else { request.localPort = [config["local-port"], null]; } } if (Object.prototype.hasOwnProperty.call(config, "ignore-content-length")) { request.ignoreContentLength = config["ignore-content-length"]; } if (config.interface) { request.interface = config.interface; } if (config.ciphers) { request.ciphers = config.ciphers; } if (config.curves) { request.curves = config.curves; } if (config.insecure) { request.insecure = true; } if (Object.prototype.hasOwnProperty.call(config, "cert-status")) { request.certStatus = config["cert-status"]; } // TODO: if the URL doesn't start with https://, curl doesn't verify // certificates, etc. if (config.cert) { if (config.cert.startsWith("pkcs11:") || !config.cert.match(/[:\\]/)) { request.cert = [config.cert, null]; } else { // TODO: curl does more complex processing // find un-backslash-escaped colon, backslash might also be escaped with a backslash let colon = -1; try { // Safari versions older than 16.4 don't support negative lookbehind colon = config.cert.search(/(?<!\\)(?:\\\\)*:/); } catch { colon = config.cert.search(/:/); } if (colon === -1) { request.cert = [config.cert, null]; } else { const cert = config.cert.slice(0, colon); const password = config.cert.slice(colon + 1); if (password.toBool()) { request.cert = [cert, password]; } else { request.cert = [cert, null]; } } } } if (config["cert-type"]) { const certType = config["cert-type"]; request.certType = certType; if ( certType.isString() && !["PEM", "DER", "ENG", "P12"].includes(certType.toString().toUpperCase()) ) { warnf(global_, [ "cert-type-unknown", "not supported file type " + JSON.stringify(certType.toString()) + " for certificate", ]); } } if (config.key) { request.key = config.key; } if (config["key-type"]) { request.keyType = config["key-type"]; } if (config.pass) { request.pass = config.pass; } if (config.cacert) { request.cacert = config.cacert; } if (Object.prototype.hasOwnProperty.call(config, "ca-native")) { request.caNative = config["ca-native"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl-allow-beast")) { request.sslAllowBeast = config["ssl-allow-beast"]; } if (config.capath) { request.capath = config.capath; } if (config.crlfile) { request.crlfile = config.crlfile; } if (config.pinnedpubkey) { request.pinnedpubkey = config.pinnedpubkey; } if (config["random-file"]) { request.randomFile = config["random-file"]; } if (config["egd-file"]) { request.egdFile = config["egd-file"]; } if (config.hsts) { request.hsts = config.hsts; } if (Object.prototype.hasOwnProperty.call(config, "alpn")) { request.alpn = config.alpn; } if (config.tlsVersion) { request.tlsVersion = config.tlsVersion; } if (config["tls-max"]) { request.tlsMax = config["tls-max"]; } if (config["tls13-ciphers"]) { request.tls13Ciphers = config["tls13-ciphers"]; } if (config["tlsauthtype"]) { request.tlsauthtype = config["tlsauthtype"]; } if (config["tlspassword"]) { request.tlspassword = config["tlspassword"]; } if (config["tlsuser"]) { request.tlsuser = config["tlsuser"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl-allow-beast")) { request.sslAllowBeast = config["ssl-allow-beast"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl-auto-client-cert")) { request.sslAutoClientCert = config["ssl-auto-client-cert"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl-no-revoke")) { request.sslNoRevoke = config["ssl-no-revoke"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl-reqd")) { request.sslReqd = config["ssl-reqd"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl-revoke-best-effort")) { request.sslRevokeBestEffort = config["ssl-revoke-best-effort"]; } if (Object.prototype.hasOwnProperty.call(config, "ssl")) { request.ssl = config["ssl"]; } if (Object.prototype.hasOwnProperty.call(config, "sslv2")) { request.sslv2 = config["sslv2"]; } if (Object.prototype.hasOwnProperty.call(config, "sslv3")) { request.sslv3 = config["sslv3"]; } if (config["doh-url"]) { request.dohUrl = config["doh-url"]; } if (Object.prototype.hasOwnProperty.call(config, "doh-insecure")) { request.dohInsecure = config["doh-insecure"]; } if (Object.prototype.hasOwnProperty.call(config, "doh-cert-status")) { request.dohCertStatus = config["doh-cert-status"]; } if (config.proxy) { // https://github.com/curl/curl/blob/e498a9b1fe5964a18eb2a3a99dc52160d2768261/lib/url.c#L2388-L2390 request.proxy = config.proxy; if (request.proxyType && request.proxyType !== "http2") { delete request.proxyType; } if (config["proxy-user"]) { request.proxyAuth = config["proxy-user"]; } } if (Object.prototype.hasOwnProperty.call(config, "proxytunnel")) { request.proxytunnel = config.proxytunnel; } if (config.noproxy) { request.noproxy = config.noproxy; } if (config.preproxy) { request.preproxy = config.preproxy; } if (Object.prototype.hasOwnProperty.call(config, "proxy-anyauth")) { request.proxyAnyauth = config["proxy-anyauth"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-basic")) { request.proxyBasic = config["proxy-basic"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-digest")) { request.proxyDigest = config["proxy-digest"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-negotiate")) { request.proxyNegotiate = config["proxy-negotiate"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-ntlm")) { request.proxyNtlm = config["proxy-ntlm"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-ca-native")) { request.proxyCaNative = config["proxy-ca-native"]; } if (config["proxy-cacert"]) { request.proxyCacert = config["proxy-cacert"]; } if (config["proxy-capath"]) { request.proxyCapath = config["proxy-capath"]; } if (config["proxy-cert-type"]) { request.proxyCertType = config["proxy-cert-type"]; } if (config["proxy-cert"]) { request.proxyCert = config["proxy-cert"]; } if (config["proxy-ciphers"]) { request.proxyCiphers = config["proxy-ciphers"]; } if (config["proxy-crlfile"]) { request.proxyCrlfile = config["proxy-crlfile"]; } if (config["proxy-http2"]) { request.proxyType = "http2"; } if (config["proxy1.0"]) { request.proxy = config["proxy1.0"]; request.proxyType = "http1"; } if (Object.prototype.hasOwnProperty.call(config, "proxy-insecure")) { request.proxyInsecure = config["proxy-insecure"]; } if (config["proxy-key"]) { request.proxyKey = config["proxy-key"]; } if (config["proxy-key-type"]) { request.proxyKeyType = config["proxy-key-type"]; } if (config["proxy-pass"]) { request.proxyPass = config["proxy-pass"]; } if (config["proxy-pinnedpubkey"]) { request.proxyPinnedpubkey = config["proxy-pinnedpubkey"]; } if (config["proxy-pinnedpubkey"]) { request.proxyPinnedpubkey = config["proxy-pinnedpubkey"]; } if (config["proxy-service-name"]) { request.proxyServiceName = config["proxy-service-name"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-ssl-allow-beast")) { request.proxySslAllowBeast = config["proxy-ssl-allow-beast"]; } if ( Object.prototype.hasOwnProperty.call(config, "proxy-ssl-auto-client-cert") ) { request.proxySslAutoClientCert = config["proxy-ssl-auto-client-cert"]; } if (config["proxy-tls13-ciphers"]) { request.proxyTls13Ciphers = config["proxy-tls13-ciphers"]; } if (config["proxy-tlsauthtype"]) { request.proxyTlsauthtype = config["proxy-tlsauthtype"]; if ( request.proxyTlsauthtype.isString() && !eq(request.proxyTlsauthtype, "SRP") ) { warnf(global_, [ "proxy-tlsauthtype", "proxy-tlsauthtype is not supported: " + request.proxyTlsauthtype, ]); } } if (config["proxy-tlspassword"]) { request.proxyTlspassword = config["proxy-tlspassword"]; } if (config["proxy-tlsuser"]) { request.proxyTlsuser = config["proxy-tlsuser"]; } if (Object.prototype.hasOwnProperty.call(config, "proxy-tlsv1")) { request.proxyTlsv1 = config["proxy-tlsv1"]; } if (config["proxy-user"]) { request.proxyUser = config["proxy-user"]; } if (Object.prototype.hasOwnProperty.call(config, "proxytunnel")) { request.proxytunnel = config["proxytunnel"]; } if (config["socks4"]) { request.proxy = config["socks4"]; request.proxyType = "socks4"; } if (config["socks4a"]) { request.proxy = config["socks4a"]; request.proxyType = "socks4a"; } if (config["socks5"]) { request.proxy = config["socks5"]; request.proxyType = "socks5"; } if (config["socks5-hostname"]) { request.proxy = config["socks5-hostname"]; request.proxyType = "socks5-hostname"; } if (Object.prototype.hasOwnProperty.call(config, "socks5-basic")) { request.socks5Basic = config["socks5-basic"]; } if (Object.prototype.hasOwnProperty.call(config, "socks5-gssapi-nec")) { request.socks5GssapiNec = config["socks5-gssapi-nec"]; } if (config["socks5-gssapi-service"]) { request.socks5GssapiService = config["socks5-gssapi-service"]; } if (Object.prototype.hasOwnProperty.call(config, "socks5-gssapi")) { request.socks5Gssapi = config["socks5-gssapi"]; } if (config["haproxy-clientip"]) { request.haproxyClientIp = config["haproxy-clientip"]; } if (Object.prototype.hasOwnProperty.call(config, "haproxy-protocol")) { request.haproxyProtocol = config["haproxy-protocol"]; } if (config["max-time"]) { request.timeout = config["max-time"]; if ( config["max-time"].isString() && // TODO: parseFloat() like curl isNaN(parseFloat(config["max-time"].toString())) ) { warnf(global_, [ "max-time-not-number", "option --max-time: expected a proper numerical parameter: " + JSON.stringify(config["max-time"].toString()), ]); } } if (config["connect-timeout"]) { request.connectTimeout = config["connect-timeout"]; if ( config["connect-timeout"].isString() && isNaN(parseFloat(config["connect-timeout"].toString())) ) { warnf(global_, [ "connect-timeout-not-number", "option --connect-timeout: expected a proper numerical parameter: " + JSON.stringify(config["connect-timeout"].toString()), ]); } } if (config["expect100-timeout"]) { request.expect100Timeout = config["expect100-timeout"]; if ( config["expect100-timeout"].isString() && isNaN(parseFloat(config["expect100-timeout"].toString())) ) { warnf(global_, [ "expect100-timeout-not-number", "option --expect100-timeout: expected a proper numerical parameter: " + JSON.stringify(config["expect100-timeout"].toString()), ]); } } if (config["happy-eyeballs-timeout-ms"]) { request.happyEyeballsTimeoutMs = config["happy-eyeballs-timeout-ms"]; } if (config["speed-limit"]) { request.speedLimit = config["speed-limit"]; } if (config["speed-time"]) { request.speedTime = config["speed-time"]; } if (config["limit-rate"]) { request.limitRate = config["limit-rate"]; } if (config["max-filesize"]) { request.maxFilesize = config["max-filesize"]; } if (Object.prototype.hasOwnProperty.call(config, "keepalive")) { request.keepAlive = config.keepalive; } if (config["keepalive-time"]) { request.keepAliveTime = config["keepalive-time"]; } if (config["alt-svc"]) { request.altSvc = config["alt-svc"]; } if (Object.prototype.hasOwnProperty.call(config, "location")) { request.followRedirects = config.location; } if (config["location-trusted"]) { request.followRedirectsTrusted = config["location-trusted"]; } if (config["max-redirs"]) { request.maxRedirects = config["max-redirs"].trim(); if ( config["max-redirs"].isString() && !isInt(config["max-redirs"].toString()) ) { warnf(global_, [ "max-redirs-not-int", "option --max-redirs: expected a proper numerical parameter: " + JSON.stringify(config["max-redirs"].toString()), ]); } } if (Object.prototype.hasOwnProperty.call(config, "post301")) { request.post301 = config.post301; } if (Object.prototype.hasOwnProperty.call(config, "post302")) { request.post302 = config.post302; } if (Object.prototype.hasOwnProperty.call(config, "post303")) { request.post303 = config.post303; } if (config.fail) { request.fail = config.fail; } if (config.retry) { request.retry = config.retry; } if (config["retry-max-time"]) { request.retryMaxTime = config["retry-max-time"]; } if (Object.prototype.hasOwnProperty.call(config, "ftp-skip-pasv-ip")) { request.ftpSkipPasvIp = config["ftp-skip-pasv-ip"]; } if (config.httpVersion) { if ( config.httpVersion === "2" || config.httpVersion === "2-prior-knowledge" ) { request.http2 = true; } if (config.httpVersion === "3" || config.httpVersion === "3-only") { request.http3 = true; } request.httpVersion = config.httpVersion; } if (Object.prototype.hasOwnProperty.call(config, "http0.9")) { request.http0_9 = config["http0.9"]; } if (config.resolve && config.resolve.length) { request.resolve = config.resolve; } if (config["connect-to"] && config["connect-to"].length) { request.connectTo = config["connect-to"]; } if (config["unix-socket"]) { request.unixSocket = config["unix-socket"]; } if (config["abstract-unix-socket"]) { request.abstractUnixSocket = config["abstract-unix-socket"]; } if (config["netrc-optional"]) { request.netrc = "optional"; } else if (config.netrc || config["netrc-file"]) { request.netrc = "required"; } else if (config.netrc === false) { // TODO || config["netrc-optional"] === false ? request.netrc = "ignored"; } if (config["netrc-file"]) { request.netrcFile = config["netrc-file"]; } if (config["use-ascii"]) { request.useAscii = config["use-ascii"]; } if (config["continue-at"]) { request.continueAt = config["continue-at"]; } if (Object.prototype.hasOwnProperty.call(config, "crlf")) { request.crlf = config.crlf; } if (Object.prototype.hasOwnProperty.call(config, "clobber")) { request.clobber = config.clobber; } if (Object.prototype.hasOwnProperty.call(config, "remote-time")) { request.remoteTime = config["remote-time"]; } // Global options if (Object.prototype.hasOwnProperty.call(global_, "verbose")) { request.verbose = global_.verbose; } if (Object.prototype.hasOwnProperty.call(global_, "silent")) { request.silent = global_.silent; } return request; } export function buildRequests( global_: GlobalConfig, stdin?: Word, stdinFile?: Word, ): Request[] { if (!global_.configs.length) { // shouldn't happen warnf(global_, ["no-configs", "got empty config object"]); } return global_.configs.map((config) => buildRequest(global_, config, stdin, stdinFile), ); } export function getFirst( requests: Request[], warnings: Warnings, support?: Support, ): Request { if (requests.length > 1) { warnings.push([ "next", // TODO: better message, we might have two requests because of // --next or because of multiple curl commands or both "got " + requests.length + " curl requests, only converting the first one", ]); } const request = requests[0]; warnIfPartsIgnored(request, warnings, support); return request; }