npm-consider-tester
Version:
Check npm package dependencies, stats and impact on your package before installing it
418 lines (384 loc) • 14.1 kB
JavaScript
/**
* @file fetches package details from registry
*/
const fetch = require('node-fetch');
const semver = require('semver');
const getLicenseStr = require('./getLicenseStr');
const getLicenseType = require('./getLicenseType');
const url = require('url');
const printError = require('./printError');
const readline = require('readline');
let packageDetailsCache = {};
const npmConfig = require('rc')('npm', {
registry: `https://registry.npmjs.org/`
});
let licensingPath = '~/licensesfiles.txt'
const fs = require("fs");
const { query } = require('express');
/**
* @param {Response} r
* @return {Promise}
*/
function checkResponse(r) {
if (r.ok) {
return r.json();
}
console.log(`Error: Response is not ok ${r.status} ${r.statusText} ${r.url}`);
}
/**
* @param {Response} r
* @return {Promise}
*/
function checkResponse1(r) {
if (r.ok) {
console.log("WHAT ODES THIS DO? ", r.body)
return r.json();
}
console.log(`Error: Response is not ok ${r.status} ${r.statusText} ${r.url}`);
}
/**
* finds biggest matching version
* @param {string} versionLoose
* @param {string[]} versions
* @return {string}
*/
function getVersion(versionLoose, versions) {
let version;
for (let i = 0; i < versions.length; i += 1) {
let matchingVersion;
if (semver.satisfies(versions[i], versionLoose)) {
matchingVersion = versions[i];
}
if (matchingVersion) {
if (!version) {
version = matchingVersion;
} else if (semver.gt(matchingVersion, version)) {
version = matchingVersion;
}
}
}
return version;
}
const gitHubApiUrl = 'https://api.github.com/';
/**
* @param {string} owner
* @param {string} repo
* @returns {Object} details
* @returns {String} details.license
* @returns {Number} details.size
* @returns {String} details.modified
*/
function getSizeAndLicenseFromGitHub(owner, repo) {
const repoInfoUrl = `${gitHubApiUrl}repos/${owner}/${repo}`;// I believe size of downloaded repo will not depend on ref
readline.cursorTo(process.stdout, 0);
readline.clearLine(process.stdout, 1);
process.stdout.write(`GET ${repoInfoUrl}`);
return fetch(repoInfoUrl).then(checkResponse).then(({
size: sizeKb,
license: licenseObj,
updated_at: modified
}) => {
const size = sizeKb * 1024;
const license = licenseObj && licenseObj.spdx_id || 'Unknown';
return { size, license, modified };
});
}
/**
* @see https://developer.github.com/v3/repos/contents/
* @param {string} owner
* @param {string} repo
* @param {string} ref
* @returns {object} dependencies, name, version
*/
function getPackageJsonFromGitHub(owner, repo, ref) {
const packageJsonUrl = `${gitHubApiUrl}repos/${owner}/${repo}/contents/package.json?ref=${ref}`;
readline.cursorTo(process.stdout, 0);
readline.clearLine(process.stdout, 1);
process.stdout.write(`GET ${packageJsonUrl}`);
return fetch(packageJsonUrl).then(checkResponse).then(({ download_url: downloadUrl }) => {
readline.cursorTo(process.stdout, 0);
readline.clearLine(process.stdout, 1);
process.stdout.write(`GET ${downloadUrl}`);
return fetch(downloadUrl).then(checkResponse);
}).then((packageJson) => {
const dependencies = packageJson.dependencies || {};
const { name, version } = packageJson;
return { dependencies, name, version };
}, (e) => {
printError(`Cannot fetch package.json from GitHub for ${owner}/${repo}`);
throw e;
});
}
function addLicenseToFile( dataToAppend ){
let path1 = process.cwd();
let licensingPath = path1 + '/licensesfiles.txt'
try{
if (dataToAppend !== "" || dataToAppend !== undefined){
dataToAppend = dataToAppend.replace(/'/g, "'").replace(/>/g, ">").replace(/</g, "<");
fs.appendFileSync(licensingPath, dataToAppend);
console.log('Data has been written');
} else {
console.log("nothing to append, probably found an error");
}
} catch (e) {
console.log(`failed because of ${e}`);
}
}
function getLicenseFileFromGithubV1( owner, repo, ref ) {
console.log("STARTED THIS FUNCTION with the inputs: ", owner, " and ", repo);
const gitHubUrl = 'https://github.com/';
const packageJsonUrls = [`${gitHubUrl}${owner}/${repo}/blob/master/LICENSE`, `${gitHubUrl}${owner}/${repo}/blob/master/LICENSE.txt`, `${gitHubUrl}${owner}/${repo}/blob/master/LICENSE.md`]
for (jsonUrl of packageJsonUrls) {
fetch(jsonUrl).then(checkResponse, (e) => {
printError(`Cannot fetch from ${jsonURL}`);
}).then(addLicenseToFile);
// readline.cursorTo(process.stdout, 0);
// readline.clearLine(process.stdout, 1);
// process.stdout.write(`GET ${downloadUrl}`);
// return fetch(downloadUrl).then(checkResponse);
// }).then((packageJson) => {
// const dependencies = packageJson.dependencies || {};
// const { name, version } = packageJson;
// return { dependencies, name, version };
// }, (e) => {
// printError(`Cannot fetch package.json from GitHub for ${owner}/${repo}`);
// throw e;
// });
}
console.log("RETURNING FROM FUNCTION");
// const test = 'https://github.com/arora-r/npm-consider/blob/master/LICENSE'
return;
}
function preparse(data){
return data.split("<!DOCTYPE html>")[0];
}
async function workpls (jsonUrl) {
try {
var final = "";
var http = require('https')
var responseParts = [];
await http.get(`${jsonUrl}`, (res) => {
const { statusCode } = res;
const contentType = res.headers['content-type'];
res.setEncoding('utf8');
console.log("status code = ", statusCode);
console.log("content_type = ", contentType);
res.on("data", function(chunk) {
//add this chunk to the output to send
responseParts.push(chunk);
});
res.on("end", function(){
//now send your complete response
console.log('No more data in response');
final = responseParts.join('');
});
// res.on("data", function(chunk) {
// preparsed = preparse(chunk)
// parsed1 = preparse(String(chunk))
// parsed2 = preparse(chunk.toString())
// });
// console.log("THIS is PARSED:", preparsed, "\n\n\n")
// console.log("THIS is PA1:", parsed1)
// console.log("\n\n\nTHIS is PA2:", parsed2)
});
console.log("FINAL: ", final);
console.log("RESP: ", responseParts);
return final;
} catch (e){
console.log(e)
}
}
function workpls2 (jsonUrl) {
console.log("SHOULD NOW BE HERE WHY DIDNT IT DO ANYTHING", jsonUrl);
console.log("TYPE OF URL", typeof(jsonUrl));
// return new Promise(function(resolve, reject) {
// var superagent = require('superagent')
// const response = superagent.get(`${jsonUrl}`);
// return response.text;
// }).then((res) => {
// console.log("THIS SHOULD BE HOW IT WORKS: |", res + "|");
// return (res);
// })
var superagent = require('superagent')
// (async () => {
// var response = await superagent.get(`${jsonUrl}`);
// console.log(response.text);
// console.log("THIS IS THE TYPE: " + typeof(response.text));
// })();
// return response.text;
var parser = require('node-html-parser');
// import { parse } from 'node-html-parser';
return new Promise(() => {return superagent.get(`${jsonUrl}`).then((res) => {
console.log("THIS IS THE text IN THE FUNCTION", res.text)
var global = parser.parse(res.text)
// console.log(root.querySelectorAll('.blob-code.blob-code-inner.js-file-line'));
var file = [];
for (var query of global.querySelectorAll('.blob-code.blob-code-inner.js-file-line')){
// console.log("This is the inner html |" + query.innerHTML + "|");
// only grab the copyright information:
if (query.innerHTML != "\n")
file.push(query.innerHTML);
else
file.push("");
}
console.log("THIS IS THE LICENSE FILE:\n|" + file.join(' ') + "|");
addLicenseToFile(file.join('\n'));
}).catch((res) => {console.error("THIS IS THE error IN THE FUNCTION", res)}).text}).then((res) => {return res});
}
function getLicenseFileFromGithubV2(link) {
console.log("STARTED THIS FUNCTION with the inputs:|" + link + "|");
const packageJsonUrls = [`${link}/blob/master/LICENSE`]//, `${link}/blob/master/LICENSE.txt`, `${link}/blob/master/LICENSE.md`]
var preparsed, parsed1, parsed2 ;
for (jsonUrl of packageJsonUrls) {
// console.log("PLEASE SHOW ME IT WORKS |"+ typeof(workpls(jsonUrl))+"|\n\n\n\n\n\n");
workpls2(jsonUrl).then((data) => {console.log("THIS SHOULD BE HOW IT WORKS right???|" + data + "|")});
// fetch(jsonUrl).then(checkResponse1, (e) => {
// printError(`Cannot fetch from ${jsonURL}`);
// }).then((data) => {
// console.log("DOES JFDKSJKDSJFKDSJLKFJKLSDFJLFJSKLF FJDSFKLDSJFKLSDFKSDLLKJFKL");
// const jsdom = require("jsdom");
// var parser = new DOMParser();
// var doc = parser.parseFromString(data, "text/html");
// console.log("DATA IS: " + doc)
// // addLicenseToFile(stringify(data));
// // addLicenseToFile(PARSER_METHOD(stringify(data)));
// })
}
console.log("RETURNING FROM FUNCTION");
// const test = 'https://github.com/arora-r/npm-consider/blob/master/LICENSE'
return;
}
/**
* @param {object} urlObj
* @param {string} versionLoose
* @returns {object} details like dependencies, version, size, license, modified
*/
function getPackageDetailsFromGitHub({ host, path, hash, protocol }, versionLoose) {
console.log("DID IT AT LEAST MAKE IT HERE FDSJKFJDSKFJDKLSFJKLSDJFKL")
let owner;
let repo;
if (protocol === 'github:') {
owner = host;
repo = path.slice(1);
} else {
[owner, repo] = String(path).slice(1).replace(/\.git$/, '').split('/');
}
if (!owner || !repo) {
throw new Error(`Cannot parse github dependency url ${versionLoose}`);
}
let ref = 'master';
if (hash && hash.slice(1)) {
ref = hash.slice(1);
}
return Promise.all([
getSizeAndLicenseFromGitHub(owner, repo),
getPackageJsonFromGitHub(owner, repo, ref),
getLicenseFileFromGithubV1(owner, repo, ref)
]).then((detailsAr) => {
return Object.assign({}, ...detailsAr);
});
}
/**
* @param {string} name package name
* @param {string} versionLoose version selector
* @return {Promise}
*/
module.exports = function addLicenses(
name,
versionLoose
) {
packageDetailsCache = {}
console.log("started inputs are: ", name, versionLoose);
const versionUrlObj = url.parse(versionLoose);
if (versionUrlObj.protocol) {
console.log("Gets into this first if");
if (versionUrlObj.host === 'github.com' || versionUrlObj.protocol === 'github:') {
// TODO: cache result
console.log("FIRST MADE IT HERE")
return getPackageDetailsFromGitHub(versionUrlObj, versionLoose).then(({
dependencies, version, size, license, modified
}) => {
return {
name,
modified,
version,
license,
licenseType: getLicenseType(license),
dependencies,
versionLoose,
size
};
}, (e) => {
printError(e);
return Promise.resolve(null);
});
}
printError(`${
versionUrlObj.protocol
} is not supported by npm-consider, skipping ${
versionLoose
}`);
return Promise.resolve(null);
}
console.log("DONE WITH THE FIRST IF");
const key = `${name}@${versionLoose}`;
const scope = name[0] === '@' ? name.slice(0, name.indexOf('/')) : undefined;
let registryUrl = (scope && npmConfig[`${scope}:registry`]) || npmConfig.registry;
if (registryUrl.charAt(registryUrl.length - 1) !== `/`) {
registryUrl += `/`;
}
const infoUrl = `${registryUrl}${name.replace(`/`, `%2f`)}`;
if (!packageDetailsCache[key]) {
readline.cursorTo(process.stdout, 0);
readline.clearLine(process.stdout, 1);
console.log(`GETTING FROM ${infoUrl} jfkdjfakljsdkjaflj`)
process.stdout.write(`GET ${infoUrl}`);
packageDetailsCache[key] = fetch(infoUrl).then(checkResponse).then((packageInfo) => {
let version;
// console.log(`\n\n\nTHIS IS WHATS IN PACKAGE INFO AS STRING JSON${JSON.stringify(packageInfo)}`)
console.log(`PLEASE WORK: ${packageInfo[`repository`].url}`);
getLicenseFileFromGithubV2(packageInfo[`repository`].url.replace('git+','').replace('.git', ''));
if (!versionLoose) {
version = packageInfo[`dist-tags`].latest;
} else if (packageInfo[`dist-tags`][versionLoose]) {
version = packageInfo[`dist-tags`][versionLoose];
} else if (packageInfo.versions[versionLoose]) {
version = versionLoose;
} else {
version = getVersion(versionLoose, Object.keys(packageInfo.versions));
}
let versionDetails = packageInfo.versions[version];
if (!versionDetails) {
versionDetails = packageInfo.versions[packageInfo[`dist-tags`].latest];
}
let modified;
if (packageInfo.time) {
modified = packageInfo.time[version];
if (!modified) {
modified = packageInfo.time.modified;
}
}
return fetch(versionDetails.dist.tarball, { method: `HEAD` }).then((r) => {
console.log("DOES THIS FETCH");
const size = r.headers.get(`content-length`);
const license = getLicenseStr(
versionDetails.license || versionDetails.licenses || `Unknown`
);
const licenseType = getLicenseType(license);
return {
name,
modified,
version,
license,
licenseType,
dependencies: versionDetails.dependencies || {},
versionLoose,
size
};
});
});
}
console.log("SKIPS EVERYTHING DOES IT DO THIS");
// getLicenseFileFromGithub();
return packageDetailsCache[key];
};