@lobehub/chat/src/libs/langchain/loaders/code/index.ts

Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.

import {
  RecursiveCharacterTextSplitter,
  SupportedTextSplitterLanguage,
} from 'langchain/text_splitter';

import { loaderConfig } from '@/libs/langchain/loaders/config';

export const CodeLoader = async (text: string, language: string) => {
  const splitter = RecursiveCharacterTextSplitter.fromLanguage(
    language as SupportedTextSplitterLanguage,
    loaderConfig,
  );

  return await splitter.createDocuments([text]);
};