node-llama-cpp/templates/packed/node-typescript.json

Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level

{"files":[{"path":[".editorconfig"],"content":"root = true\n\n[*]\nindent_style = space\nindent_size = 4\n\n[{*.ts,*.tsx,*.js,*.jsx,*.css,*.scss}]\ninsert_final_newline = true\n\n[{package.json,package-lock.json,manifest.json}]\nindent_size = 2\n\n[*.yml]\nindent_size = 2\n"},{"path":[".gitignore"],"content":"/.idea\n/.vscode\nnode_modules\n.DS_Store\n\n/dist\n/models\n"},{"path":["README.md"],"content":"# Node + TypeScript + `node-llama-cpp`\nThis template provides a minimal setup to get Node working with TypeScript and `node-llama-cpp`, and some ESLint rules.\n\n## Get started\nInstall node modules and download the model files used by `node-llama-cpp`:\n```bash\nnpm install\n```\n\nStart the project:\n```bash\nnpm start\n```\n\n> Generated using `npm create node-llama-cpp@latest` ([learn more](https://node-llama-cpp.withcat.ai/guide/))\n"},{"path":["eslint.config.js"],"content":"// @ts-check\n\nimport importPlugin from \"eslint-plugin-import\";\nimport jsdoc from \"eslint-plugin-jsdoc\";\nimport n from \"eslint-plugin-n\";\nimport tseslint from \"typescript-eslint\";\nimport stylistic from \"@stylistic/eslint-plugin\";\n\n\nexport default tseslint.config({\n    ignores: [\"dist/\", \"models/\"]\n}, {\n    files: [\"**/**.{,c,m}{js,ts}\"],\n    extends: [\n        stylistic.configs[\"recommended-flat\"],\n        jsdoc.configs[\"flat/recommended\"],\n        importPlugin.flatConfigs.recommended\n    ],\n    plugins: {\n        n\n    },\n    languageOptions: {\n        globals: {\n            Atomics: \"readonly\",\n            SharedArrayBuffer: \"readonly\"\n        },\n\n        ecmaVersion: 2023,\n        sourceType: \"module\"\n    },\n    settings: {\n        \"import/resolver\": {\n            typescript: true,\n            node: true\n        },\n        jsdoc: {\n            exemptDestructuredRootsFromChecks: true,\n\n            tagNamePreference: {\n                hidden: \"hidden\"\n            }\n        }\n    },\n    rules: {\n        \"@stylistic/indent\": [\"off\"],\n        \"indent\": [\"warn\", 4, {\n            SwitchCase: 1,\n            FunctionDeclaration: {\n                parameters: \"first\"\n            },\n            ignoredNodes: [\n                // fix for indent warnings on function object return types when the function has no parameters\n                'FunctionExpression[params.length=0][returnType.type=\"TSTypeAnnotation\"]'\n            ]\n        }],\n        \"@stylistic/indent-binary-ops\": [\"off\"],\n        \"@stylistic/eqeqeq\": [\"off\"],\n        \"@stylistic/no-undef\": \"off\",\n        \"@stylistic/quotes\": [\"warn\", \"double\", {avoidEscape: true}],\n        \"no-unused-vars\": [\"warn\", {\n            args: \"none\",\n            ignoreRestSiblings: true,\n            varsIgnorePattern: \"^set\",\n            caughtErrors: \"none\"\n        }],\n        \"@stylistic/no-prototype-builtins\": [\"off\"],\n        \"@stylistic/object-curly-spacing\": [\"warn\", \"never\"],\n        \"@stylistic/semi\": [\"warn\", \"always\"],\n        \"@stylistic/no-undefined\": [\"off\"],\n        \"@stylistic/array-bracket-newline\": [\"error\", \"consistent\"],\n        \"@stylistic/brace-style\": [\"error\", \"1tbs\", {\n            allowSingleLine: false\n        }],\n        \"@stylistic/comma-spacing\": [\"error\", {\n            before: false,\n            after: true\n        }],\n        \"@stylistic/comma-style\": [\"error\", \"last\"],\n        \"@stylistic/comma-dangle\": [\"warn\", \"never\"],\n        \"no-var\": [\"error\"],\n        \"import/order\": [\"error\", {\n            groups: [\"builtin\", \"external\", \"internal\", \"parent\", \"sibling\", \"index\", \"type\", \"object\", \"unknown\"],\n            warnOnUnassignedImports: true\n        }],\n        \"n/file-extension-in-import\": [\"error\", \"always\"],\n        \"newline-per-chained-call\": [\"error\", {\n            ignoreChainWithDepth: 2\n        }],\n        \"no-confusing-arrow\": [\"error\"],\n        \"no-const-assign\": [\"error\"],\n        \"no-duplicate-imports\": [\"error\", {\n            includeExports: true\n        }],\n        camelcase: [\"warn\"],\n        \"@stylistic/jsx-quotes\": [\"warn\"],\n        yoda: [\"error\", \"never\", {\n            exceptRange: true\n        }],\n        \"no-eval\": [\"error\"],\n        \"array-callback-return\": [\"error\"],\n        \"no-empty\": [\"error\", {\n            allowEmptyCatch: true\n        }],\n        \"@stylistic/keyword-spacing\": [\"warn\"],\n        \"@stylistic/space-infix-ops\": [\"warn\"],\n        \"@stylistic/spaced-comment\": [\"warn\", \"always\", {\n            markers: [\"/\"]\n        }],\n        \"@stylistic/eol-last\": [\"warn\", \"always\"],\n        \"@stylistic/max-len\": [\"warn\", {\n            code: 140,\n            tabWidth: 4,\n            ignoreStrings: true\n        }],\n        \"@stylistic/quote-props\": [\"off\"],\n        \"@stylistic/arrow-parens\": [\"warn\", \"always\"],\n        \"@stylistic/no-multiple-empty-lines\": [\"off\"],\n        \"@stylistic/operator-linebreak\": [\"off\"],\n        \"@stylistic/block-spacing\": [\"warn\", \"never\"],\n        \"@stylistic/no-extra-parens\": [\"off\"],\n        \"@stylistic/padded-blocks\": [\"warn\"],\n        \"@stylistic/multiline-ternary\": [\"off\"],\n        \"@stylistic/lines-between-class-members\": [\"warn\", {\n            enforce: [\n                {blankLine: \"always\", prev: \"method\", next: \"*\"},\n                {blankLine: \"always\", prev: \"*\", next: \"method\"}\n            ]\n        }],\n        \"@stylistic/no-trailing-spaces\": [\"off\"],\n        \"@stylistic/no-multi-spaces\": [\"warn\"],\n        \"@stylistic/generator-star-spacing\": [\"off\"]\n    }\n}, {\n    files: [\"**/**.{,c,m}ts\"],\n    extends: [\n        jsdoc.configs[\"flat/recommended-typescript\"],\n        ...tseslint.configs.recommended\n    ],\n    settings: {\n        \"import/resolver\": {\n            typescript: true,\n            node: true\n        }\n    },\n    rules: {\n        \"no-constant-condition\": [\"warn\"],\n        \"import/named\": [\"off\"],\n        \"@typescript-eslint/explicit-module-boundary-types\": [\"off\"],\n        \"@typescript-eslint/ban-ts-comment\": [\"off\"],\n        \"@typescript-eslint/no-explicit-any\": [\"off\"],\n        \"@typescript-eslint/no-inferrable-types\": [\"off\"],\n        \"@typescript-eslint/no-unused-vars\": [\"warn\", {\n            args: \"none\",\n            ignoreRestSiblings: true,\n            varsIgnorePattern: \"^set\",\n            caughtErrors: \"none\"\n        }],\n        \"@typescript-eslint/no-empty-object-type\": [\"off\"],\n        \"@typescript-eslint/member-ordering\": [\"warn\", {\n            default: [\"field\", \"constructor\", \"method\", \"signature\"],\n            typeLiterals: []\n        }],\n        \"@typescript-eslint/parameter-properties\": [\"warn\", {\n            allow: []\n        }],\n        \"@typescript-eslint/explicit-member-accessibility\": [\"warn\"],\n        \"@stylistic/member-delimiter-style\": [\"warn\", {\n            multiline: {\n                delimiter: \"comma\",\n                requireLast: false\n            },\n            singleline: {\n                delimiter: \"comma\",\n                requireLast: false\n            },\n            multilineDetection: \"brackets\"\n        }],\n\n        \"jsdoc/require-param\": [\"off\"],\n        \"jsdoc/check-param-names\": [\"warn\", {\n            checkDestructured: false\n        }],\n        \"jsdoc/require-returns\": [\"off\"],\n        \"jsdoc/require-jsdoc\": [\"off\"],\n        \"jsdoc/require-yields\": [\"off\"],\n        \"jsdoc/require-param-description\": [\"off\"]\n    }\n});\n"},{"path":["package.json"],"content":"{\n  \"name\": \"node-llama-cpp-project\",\n  \"private\": true,\n  \"version\": \"0.0.0\",\n  \"main\": \"./dist/index.js\",\n  \"type\": \"module\",\n  \"types\": \"./dist/index.d.ts\",\n  \"files\": [\n    \"dist/\",\n    \"package.json\",\n    \"README.md\"\n  ],\n  \"exports\": {\n    \".\": {\n      \"import\": \"./dist/index.js\",\n      \"node\": \"./dist/index.js\",\n      \"types\": \"./dist/index.d.ts\",\n      \"default\": \"./dist/index.js\"\n    }\n  },\n  \"engines\": {\n    \"node\": \">=20.0.0\"\n  },\n  \"scripts\": {\n    \"postinstall\": \"npm run models:pull\",\n    \"models:pull\": \"node-llama-cpp pull --dir ./models \\\"{{modelUriOrUrl|escape|escape}}\\\"\",\n    \"start\": \"vite-node ./src/index.ts\",\n    \"start:build\": \"node ./dist/index.ts\",\n    \"prebuild\": \"rimraf ./dist ./tsconfig.tsbuildinfo\",\n    \"build\": \"tsc --build tsconfig.json --force\",\n    \"lint\": \"npm run lint:eslint\",\n    \"lint:eslint\": \"eslint --report-unused-disable-directives .\",\n    \"format\": \"npm run lint:eslint -- --fix\",\n    \"clean\": \"rm -rf ./node_modules ./dist ./tsconfig.tsbuildinfo ./models\"\n  },\n  \"dependencies\": {\n    \"chalk\": \"^5.4.1\",\n    \"node-llama-cpp\": \"^{{currentNodeLlamaCppModuleVersion|escape}}\"\n  },\n  \"devDependencies\": {\n    \"@stylistic/eslint-plugin\": \"^4.2.0\",\n    \"@types/node\": \"^22.13.11\",\n    \"eslint\": \"^9.23.0\",\n    \"eslint-import-resolver-typescript\": \"^4.2.2\",\n    \"eslint-plugin-import\": \"^2.31.0\",\n    \"eslint-plugin-jsdoc\": \"^50.6.8\",\n    \"eslint-plugin-n\": \"^17.16.2\",\n    \"rimraf\": \"^6.0.1\",\n    \"tslib\": \"^2.8.1\",\n    \"typescript\": \"^5.8.2\",\n    \"typescript-eslint\": \"^8.27.0\",\n    \"vite-node\": \"^3.0.9\"\n  }\n}"},{"path":["src","index.ts"],"content":"import {fileURLToPath} from \"url\";\nimport path from \"path\";\nimport chalk from \"chalk\";\nimport {getLlama, LlamaChatSession, resolveModelFile} from \"node-llama-cpp\";\n\nconst __dirname = path.dirname(fileURLToPath(import.meta.url));\nconst modelsDirectory = path.join(__dirname, \"..\", \"models\");\n\n\nconst llama = await getLlama();\n\nconsole.log(chalk.yellow(\"Resolving model file...\"));\nconst modelPath = await resolveModelFile(\n    \"{{modelUriOrFilename|escape}}\",\n    modelsDirectory\n);\n\nconsole.log(chalk.yellow(\"Loading model...\"));\nconst model = await llama.loadModel({modelPath});\n\nconsole.log(chalk.yellow(\"Creating context...\"));\nconst context = await model.createContext({\n    contextSize: {max: 8096} // omit this for a longer context size, but increased memory usage\n});\n\nconst session = new LlamaChatSession({\n    contextSequence: context.getSequence()\n});\nconsole.log();\n\n\nconst q1 = \"Hi there, how are you?\";\nconsole.log(chalk.yellow(\"User: \") + q1);\n\nprocess.stdout.write(chalk.yellow(\"AI: \"));\nconst a1 = await session.prompt(q1, {\n    // uncomment for a simpler response streaming, without segment information\n    // onTextChunk(chunk) {\n    //     // stream the response to the console as it's being generated\n    //     process.stdout.write(chunk);\n    // }\n    onResponseChunk(chunk) {\n        // stream the response to the console as it's being generated\n        // including segment information (like chain of thought)\n\n        if (chunk.type === \"segment\" && chunk.segmentStartTime != null)\n            process.stdout.write(chalk.bold(` [segment start: ${chunk.segmentType}] `));\n\n        process.stdout.write(chunk.text);\n\n        if (chunk.type === \"segment\" && chunk.segmentEndTime != null)\n            process.stdout.write(chalk.bold(` [segment end: ${chunk.segmentType}] `));\n    }\n});\nprocess.stdout.write(\"\\n\");\nconsole.log(chalk.yellow(\"Consolidated AI answer: \") + a1);\nconsole.log();\n\n\nconst q2 = \"Summarize what you said\";\nconsole.log(chalk.yellow(\"User: \") + q2);\n\nconst a2 = await session.prompt(q2);\nconsole.log(chalk.yellow(\"AI: \") + a2);\nconsole.log();\n\n\nconst q3 = \"What are the verbs in this sentence: 'The cat sat on the mat'\";\nconsole.log(chalk.yellow(\"User: \") + q3);\n\n// force the model to respond in accordance to the specified JSON schema format,\n// so we can parse it and use it programmatically\nconst responseGrammar = await llama.createGrammarForJsonSchema({\n    type: \"object\",\n    properties: {\n        verbs: {\n            type: \"array\",\n            items: {\n                type: \"string\"\n            }\n        }\n    }\n});\nconst a3 = await session.prompt(q3, {grammar: responseGrammar});\nconst parsedResponse = responseGrammar.parse(a3);\nconsole.log(chalk.yellow(\"AI:\"), parsedResponse.verbs);\nconsole.log();\n\nif (parsedResponse.verbs.length > 0) {\n    const q4 = `Define the verb \"${parsedResponse.verbs[0]}\"`;\n    console.log(chalk.yellow(\"User: \") + q4);\n\n    const a4 = await session.prompt(q4);\n    console.log(chalk.yellow(\"AI: \") + a4);\n    console.log();\n} else {\n    const q4 = \"Are you sure there are no verbs in the sentence?\";\n    console.log(chalk.yellow(\"User: \") + q4);\n\n    const a4 = await session.prompt(q4);\n    console.log(chalk.yellow(\"AI: \") + a4);\n    console.log();\n}\n"},{"path":["tsconfig.json"],"content":"{\n    \"compilerOptions\": {\n        \"lib\": [\"es2022\"],\n        \"module\": \"es2022\",\n        \"target\": \"es2022\",\n        \"esModuleInterop\": true,\n        \"noImplicitAny\": true,\n        \"noImplicitReturns\": true,\n        \"noImplicitThis\": true,\n        \"noImplicitOverride\": true,\n        \"removeComments\": false,\n        \"allowSyntheticDefaultImports\": true,\n        \"forceConsistentCasingInFileNames\": true,\n        \"noFallthroughCasesInSwitch\": true,\n        \"noUncheckedIndexedAccess\": true,\n        \"moduleDetection\": \"force\",\n        \"skipLibCheck\": true,\n        \"moduleResolution\": \"node\",\n        \"resolveJsonModule\": false,\n        \"strictNullChecks\": true,\n        \"isolatedModules\": true,\n        \"noEmit\": false,\n        \"outDir\": \"./dist\",\n        \"strict\": true,\n        \"sourceMap\": true,\n        \"composite\": false,\n        \"declaration\": true\n    },\n    \"files\": [\n        \"./src/index.ts\"\n    ],\n    \"include\": [\n        \"./src\"\n    ]\n}\n"}]}