ayakashi
Version:
The next generation web scraping framework
296 lines (286 loc) • 17.4 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.generateProject = void 0;
const mkdirp_1 = __importDefault(require("mkdirp"));
const path_1 = require("path");
const util_1 = require("util");
const fs_1 = require("fs");
const child_process_1 = require("child_process");
const opLog_1 = require("../../opLog/opLog");
const generateScraper_1 = require("./generateScraper");
const generateScript_1 = require("./generateScript");
const tsHelpers_1 = require("../tsHelpers");
const mkdirp = util_1.promisify(mkdirp_1.default);
const writeFile = util_1.promisify(fs_1.writeFile);
const readdir = util_1.promisify(fs_1.readdir);
function generateProject(projectDir, useCurrentFolder, ts) {
return __awaiter(this, void 0, void 0, function* () {
const opLog = opLog_1.getOpLog();
try {
const filesInDir = yield readdir(projectDir);
if (filesInDir.length > 0) {
opLog.error(`${projectDir} is not empty`);
return;
}
}
catch (_e) { }
if (!useCurrentFolder) {
yield mkdirp(projectDir);
}
process.chdir(projectDir);
opLog.info(`creating new project in ${projectDir}`);
if (ts)
yield mkdirp("src");
opLog.info("generating config file");
if (ts) {
const config = path_1.resolve(projectDir, "src", "ayakashi.config.ts");
yield writeFile(config, getConfigTS());
}
else {
yield writeFile("ayakashi.config.js", getConfig());
}
opLog.info("generating package.json");
yield writeFile("package.json", ts ? getpackageJsonTS() : getpackageJson());
if (ts) {
opLog.info("generating tsconfig.json");
yield writeFile("tsconfig.json", getTsConfig());
}
if (ts) {
const tsSource = path_1.resolve(projectDir, "src");
yield generateScraper_1.generateScraper(tsSource, "githubAbout", ts);
yield generateScript_1.generateScript(tsSource, "getPage", ts);
}
else {
yield generateScraper_1.generateScraper(projectDir, "githubAbout", ts);
yield generateScript_1.generateScript(projectDir, "getPage", ts);
}
yield writeFile(".gitignore", ts ? getGitIgnoreTS() : getGitIgnore());
let npm = "npm";
if (process.platform === "win32") {
npm = "npm.cmd";
}
yield runNpmInstall(npm);
if (ts) {
yield tsHelpers_1.buildTS();
}
opLog.messageBox([
"Your project is ready!",
"Get started:",
"https://ayakashi-io.github.io",
"",
"You may run your new project with:",
"ayakashi run"
]);
});
}
exports.generateProject = generateProject;
function getConfig() {
return (`/**
* @type {import("@ayakashi/types").Config}
*/
module.exports = {
config: {},
waterfall: [{
type: "script",
module: "getPage"
}, {
type: "scraper",
module: "githubAbout"
}, {
type: "script",
module: "printToConsole"
}]
};
`);
}
function getConfigTS() {
return (`import {Config} from "@ayakashi/types";
export default {
config: {},
waterfall: [{
type: "script",
module: "getPage"
}, {
type: "scraper",
module: "githubAbout"
}, {
type: "script",
module: "printToConsole"
}]
} as Config
`);
}
function getpackageJson() {
return (`{
"name": "my-project",
"version": "1.0.0",
"description": "An ayakashi project",
"homepage": "https://ayakashi-io.github.io",
"keywords": [
"ayakashi"
],
"author": "",
"license": "ISC",
"devDependencies": {
"@ayakashi/types": "^1.0.0-beta1"
}
}
`);
}
function getpackageJsonTS() {
return (`{
"name": "my-project",
"version": "1.0.0",
"description": "An ayakashi project",
"homepage": "https://ayakashi-io.github.io",
"keywords": [
"ayakashi"
],
"author": "",
"license": "ISC",
"devDependencies": {
"@ayakashi/types": "^1.0.0-beta1",
"typescript": "~4.8.4"
},
"scripts": {
"build": "tsc --build --clean && tsc --build",
"watch": "tsc --build --clean && tsc --watch"
}
}
`);
}
function getGitIgnore() {
return (`node_modules
`);
}
function getGitIgnoreTS() {
return (`node_modules
dist
`);
}
function getTsConfig() {
return (`{
"compilerOptions": {
/* Visit https://aka.ms/tsconfig to read more about this file */
/* Projects */
// "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */
// "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */
// "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */
// "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */
// "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */
// "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */
/* Language and Environment */
"target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
"lib": ["DOM", "ES2022"], /* Specify a set of bundled library declaration files that describe the target runtime environment. */
// "jsx": "preserve", /* Specify what JSX code is generated. */
// "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */
// "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */
// "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */
// "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */
// "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */
// "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */
// "noLib": true, /* Disable including any library files, including the default lib.d.ts. */
// "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */
// "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */
/* Modules */
"module": "commonjs", /* Specify what module code is generated. */
"rootDir": "./src", /* Specify the root folder within your source files. */
"moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */
// "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */
// "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */
// "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */
// "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */
// "types": [], /* Specify type package names to be included without being referenced in a source file. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
// "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */
// "resolveJsonModule": true, /* Enable importing .json files. */
// "noResolve": true, /* Disallow 'import's, 'require's or '<reference>'s from expanding the number of files TypeScript should add to a project. */
/* JavaScript Support */
"allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */
"checkJs": true, /* Enable error reporting in type-checked JavaScript files. */
// "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */
/* Emit */
// "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */
// "declarationMap": true, /* Create sourcemaps for d.ts files. */
// "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */
// "sourceMap": true, /* Create source map files for emitted JavaScript files. */
// "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */
"outDir": "./dist", /* Specify an output folder for all emitted files. */
// "removeComments": true, /* Disable emitting comments. */
// "noEmit": true, /* Disable emitting files from a compilation. */
// "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */
// "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */
// "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */
// "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */
// "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */
// "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */
// "newLine": "crlf", /* Set the newline character for emitting files. */
// "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */
// "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */
// "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */
// "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */
// "declarationDir": "./", /* Specify the output directory for generated declaration files. */
// "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */
/* Interop Constraints */
// "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */
"allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */
"esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */
// "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */
"forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */
/* Type Checking */
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */
// "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */
// "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */
// "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */
// "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */
// "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */
// "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */
// "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */
// "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */
// "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */
// "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */
// "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */
// "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */
// "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */
// "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */
// "allowUnusedLabels": true, /* Disable error reporting for unused labels. */
// "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */
/* Completeness */
// "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */
"skipLibCheck": true /* Skip type checking all .d.ts files. */
}
}
`);
}
function runNpmInstall(npm) {
const opLog = opLog_1.getOpLog();
const waiter = opLog.waiter("installing dependencies");
return new Promise(function (resolve) {
child_process_1.exec(`${npm} install`, function (err) {
if (err) {
waiter.fail("failed to run npm install, please run it manually");
console.error(err.message);
}
else {
waiter.succeed("dependencies installed");
}
resolve();
});
});
}