nextra-dify-plugin/dist/index.mjs

A Next.js webpack plugin for integrating Dify knowledge base with Nextra documentation

import axios from 'axios';
import { readFileSync } from 'node:fs';
import { relative, dirname, basename } from 'node:path';
import matter from 'gray-matter';
import { glob } from 'glob';

class DifyClient {
  client;
  knowledgeBaseId;
  documents = null;
  process_rule;
  constructor(apiToken, knowledgeBaseId, baseURL = "https://api.dify.ai/v1", process_rule) {
    this.knowledgeBaseId = knowledgeBaseId;
    this.client = axios.create({
      baseURL,
      headers: {
        "Authorization": `Bearer ${apiToken}`,
        "Content-Type": "application/json"
      }
    });
    this.process_rule = process_rule;
    this.initDocuments();
  }
  /**
   * 获取知识库中的所有文档
   */
  async initDocuments() {
    try {
      const response = await this.client.get(
        `/datasets/${this.knowledgeBaseId}/documents`
      );
      this.documents = response.data.data || [];
      return this.documents;
    } catch (error) {
      console.error("Failed to get documents:", error);
      return [];
    }
  }
  /**
   * 获取文档
   */
  async getDocuments() {
    if (!this.documents || this.documents.length) {
      await this.initDocuments();
    }
    return this.documents || [];
  }
  /**
   * 根据文档名称查找文档
   */
  async findDocumentByName(name) {
    const documents = await this.getDocuments();
    return documents.find((doc) => doc.name === name) || null;
  }
  async deleteModuleDocuments(moduleName) {
    const documents = await this.getDocuments();
    moduleName = moduleName.replace(/_\d+$/, "");
    const moduleDocuments = documents.filter((doc) => doc.name.startsWith(moduleName));
    for (const document of moduleDocuments) {
      await this.deleteDocument(document.id);
    }
  }
  /**
   * 删除文档
   */
  async deleteDocument(documentId) {
    try {
      await this.client.delete(`/datasets/${this.knowledgeBaseId}/documents/${documentId}`);
      if (this.documents) {
        this.documents = this.documents.filter((doc) => doc.id !== documentId);
      }
    } catch (error) {
      console.error("Failed to delete document:", error);
      throw error;
    }
  }
  /**
   * 创建新文档
   */
  async createDocument(name, content) {
    try {
      const { doc_form, doc_language, mode, rules } = this.process_rule || {};
      const formData = new FormData();
      const blob = new Blob([content], { type: "text/plain" });
      formData.append("data", JSON.stringify({
        indexing_technique: "high_quality",
        process_rule: {
          mode,
          rules
        },
        doc_form,
        doc_language
      }));
      formData.append("file", blob, `${name}.txt`);
      const response = await this.client.post(
        `/datasets/${this.knowledgeBaseId}/document/create-by-file`,
        formData,
        {
          headers: {
            "Content-Type": "multipart/form-data"
          }
        }
      );
      return {
        document_id: response.data.document.id,
        name,
        operation: "create"
      };
    } catch (error) {
      console.error("Failed to create document:", error);
      throw error;
    }
  }
  /**
   * 更新现有文档
   */
  async updateDocument(documentId, name, content) {
    try {
      const { doc_form, doc_language, mode, rules } = this.process_rule || {};
      const formData = new FormData();
      const blob = new Blob([content], { type: "text/plain" });
      formData.append("data", JSON.stringify({
        indexing_technique: "high_quality",
        process_rule: {
          mode,
          rules,
          doc_form,
          doc_language
        }
      }));
      formData.append("file", blob, `${name}.txt`);
      await this.client.post(
        `/datasets/${this.knowledgeBaseId}/documents/${documentId}/update-by-file`,
        formData,
        {
          headers: {
            "Content-Type": "multipart/form-data"
          }
        }
      );
      return {
        document_id: documentId,
        name,
        operation: "update"
      };
    } catch (error) {
      console.error("Failed to update document:", error);
      throw error;
    }
  }
  /**
   * 上传或更新文档
   */
  async uploadDocument(name, content) {
    const existingDoc = await this.findDocumentByName(name);
    if (existingDoc) {
      return this.updateDocument(existingDoc.id, name, content);
    } else {
      return this.createDocument(name, content);
    }
  }
}

class MDXParser {
  options;
  constructor(options) {
    this.options = options;
  }
  /**
   * 解析MDX文件
   */
  parseMDXFile(filePath, basePath) {
    const content = readFileSync(filePath, "utf-8");
    const { data: frontmatter, content: mdxContent } = matter(content);
    const relativePath = relative(basePath, filePath);
    const pathSegments = dirname(relativePath).split("/").filter(Boolean);
    const fileName = basename(filePath, ".mdx");
    const menuPath = [...pathSegments, fileName];
    const title = frontmatter.title || fileName;
    return {
      path: filePath,
      content: mdxContent,
      frontmatter,
      title,
      menuPath
    };
  }
  /**
   * 生成文档名称
   */
  generateDocumentName(mdxFile, chunkIndex) {
    const { projectName } = this.options;
    const menuPathStr = mdxFile.menuPath.join("_");
    const baseName = `${projectName}_${menuPathStr}`;
    if (chunkIndex !== void 0) {
      return `${baseName}_${chunkIndex + 1}`;
    }
    return baseName;
  }
  /**
   * 分割MDX内容
   */
  splitContent(content) {
    const { enableSplit, splitMarker = "---", maxChunkSize = 5e3, maxChunkSplitBeforeMarker } = this.options;
    if (!enableSplit) {
      return [content];
    }
    if (content.includes(splitMarker)) {
      return content.split(splitMarker).map((chunk) => chunk.trim()).filter((chunk) => chunk.length > 0);
    }
    const chunks = [];
    let currentChunk = "";
    const lines = content.split("\n");
    for (const line of lines) {
      const nextChunkLength = currentChunk.length + line.length + 1;
      if (nextChunkLength > maxChunkSize && currentChunk.length > 0) {
        if (maxChunkSplitBeforeMarker) {
          try {
            const markerRegex = new RegExp(maxChunkSplitBeforeMarker, "gm");
            const matches = Array.from(currentChunk.matchAll(markerRegex));
            if (matches.length > 0) {
              const lastMatch = matches[matches.length - 1];
              const splitPoint = lastMatch.index;
              chunks.push(currentChunk.substring(0, splitPoint).trim());
              currentChunk = `${currentChunk.substring(splitPoint)}
${line}`;
            } else {
              chunks.push(currentChunk.trim());
              currentChunk = line;
            }
          } catch {
            console.warn(`[MDXParser] Invalid regex pattern: ${maxChunkSplitBeforeMarker}, falling back to string matching`);
            if (currentChunk.includes(maxChunkSplitBeforeMarker)) {
              const lastMarkerIndex = currentChunk.lastIndexOf(maxChunkSplitBeforeMarker);
              if (lastMarkerIndex > 0) {
                chunks.push(currentChunk.substring(0, lastMarkerIndex).trim());
                currentChunk = `${currentChunk.substring(lastMarkerIndex)}
${line}`;
              } else {
                chunks.push(currentChunk.trim());
                currentChunk = line;
              }
            } else {
              chunks.push(currentChunk.trim());
              currentChunk = line;
            }
          }
        } else {
          chunks.push(currentChunk.trim());
          currentChunk = line;
        }
      } else {
        currentChunk += (currentChunk ? "\n" : "") + line;
      }
    }
    if (currentChunk.trim()) {
      chunks.push(currentChunk.trim());
    }
    return chunks.length > 0 ? chunks : [content];
  }
  /**
   * 将MDX文件转换为文档块
   */
  convertToDocumentChunks(mdxFile) {
    const chunks = this.splitContent(mdxFile.content);
    return chunks.map((content, index) => ({
      id: `${mdxFile.path}_${index}`,
      content,
      name: this.generateDocumentName(mdxFile, chunks.length > 1 ? index : void 0),
      index
    }));
  }
  /**
   * 清理和格式化内容
   */
  cleanContent(content) {
    return content.replace(/import\s+(?:\S.*?)??from\s+['"][^'"]*['"];?\s*/g, "").replace(/export\s+.*?;?\s*/g, "").replace(/\{\/\*[\s\S]*?\*\/\}/g, "").replace(/\n\s*\n\s*\n/g, "\n\n").trim();
  }
  /**
   * 处理MDX文件，返回准备上传的文档块
   */
  processMDXFile(filePath, basePath) {
    const mdxFile = this.parseMDXFile(filePath, basePath);
    const chunks = this.convertToDocumentChunks(mdxFile);
    return chunks.map((chunk) => ({
      ...chunk,
      content: this.cleanContent(chunk.content)
    }));
  }
}

class NextraDifyPlugin {
  options;
  difyClient;
  mdxParser;
  constructor(options) {
    this.options = {
      enableUpload: false,
      enableSplit: false,
      splitMarker: "---",
      include: ["**/*.mdx"],
      exclude: ["node_modules/**", ".next/**"],
      apiBaseUrl: "https://api.dify.ai/v1",
      maxChunkSize: 5e3,
      ...options
    };
    this.difyClient = new DifyClient(
      this.options.apiToken,
      this.options.knowledgeBaseId,
      this.options.apiBaseUrl,
      this.options.process_rule || {
        mode: "custom",
        indexing_technique: "high_quality",
        rules: {
          pre_processing_rules: [{ id: "remove_extra_spaces", enabled: true }],
          parent_mode: "full-doc",
          segmentation: {
            separator: "\n\n\n\n",
            max_tokens: 2e3
          },
          subchunk_segmentation: {
            separator: "\n\n",
            max_tokens: 400
          }
        },
        doc_form: "hierarchical_model",
        doc_language: "Chinese"
      }
    );
    this.mdxParser = new MDXParser(this.options);
  }
  /**
   * webpack插件应用方法
   */
  apply(compiler) {
    const pluginName = "NextraDifyPlugin";
    compiler.hooks.done.tapAsync(pluginName, async (stats, callback) => {
      if (!this.options.enableUpload) {
        console.log("[NextraDifyPlugin] Upload is disabled, skipping...");
        callback();
        return;
      }
      try {
        await this.processDocuments(compiler.context);
        console.log("[NextraDifyPlugin] Documents processed successfully");
      } catch (error) {
        console.error("[NextraDifyPlugin] Error processing documents:", error);
      } finally {
        callback();
      }
    });
  }
  /**
   * 处理所有文档
   */
  async processDocuments(context) {
    console.log("[NextraDifyPlugin] Starting document processing...");
    const files = await this.findMDXFiles(context);
    console.log(`[NextraDifyPlugin] Found ${files.length} MDX files`);
    for (const filePath of files) {
      try {
        await this.processFile(filePath, context);
      } catch (error) {
        console.error(`[NextraDifyPlugin] Error processing file ${filePath}:`, error);
      }
    }
  }
  /**
   * 查找所有MDX文件
   */
  async findMDXFiles(basePath) {
    const { include, exclude } = this.options;
    const allFiles = [];
    for (const pattern of include || ["**/*.mdx"]) {
      const files = await glob(pattern, {
        cwd: basePath,
        absolute: true,
        ignore: exclude
      });
      allFiles.push(...files);
    }
    return [...new Set(allFiles)];
  }
  /**
   * 处理单个文件
   */
  async processFile(filePath, basePath) {
    console.log(`[NextraDifyPlugin] Processing file: ${filePath}`);
    try {
      const documentChunks = this.mdxParser.processMDXFile(filePath, basePath);
      const [firstChunk] = documentChunks || [];
      if (firstChunk) {
        await this.difyClient.deleteModuleDocuments(firstChunk.name);
      }
      for (const chunk of documentChunks) {
        await this.uploadChunk(chunk);
      }
    } catch (error) {
      console.error(`[NextraDifyPlugin] Error processing file ${filePath}:`, error);
      throw error;
    }
  }
  /**
   * 上传文档块
   */
  async uploadChunk(chunk) {
    try {
      const result = await this.difyClient.uploadDocument(chunk.name, chunk.content);
      console.log(
        `[NextraDifyPlugin] \u2713  ${result.operation === "create" ? "Created" : "Updated"} document: ${chunk.name}`
      );
    } catch (error) {
      console.error(`[NextraDifyPlugin] Error uploading chunk ${chunk.name}:`, error);
      throw error;
    }
  }
  /**
   * 验证配置
   */
  static validateOptions(options) {
    if (!options.knowledgeBaseId) {
      throw new Error("NextraDifyPlugin: knowledgeBaseId is required");
    }
    if (!options.apiToken) {
      throw new Error("NextraDifyPlugin: apiToken is required");
    }
    if (!options.projectName) {
      throw new Error("NextraDifyPlugin: projectName is required");
    }
  }
}

export { DifyClient, MDXParser, NextraDifyPlugin };