@langchain/community/dist/document_loaders/web/confluence.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"confluence.cjs","names":["BaseDocumentLoader","options?: {\n    start?: number;\n    limit?: number;\n  }","url: string","initialHeaders: HeadersInit","page: ConfluencePage","codeBlocks: { language: string; code: string }[]","Document"],"sources":["../../../src/document_loaders/web/confluence.ts"],"sourcesContent":["import { htmlToText } from \"html-to-text\";\nimport { Document } from \"@langchain/core/documents\";\nimport { BaseDocumentLoader } from \"@langchain/core/document_loaders/base\";\n\n/**\n * Interface representing the parameters for configuring the\n * ConfluencePagesLoader.\n */\nexport interface ConfluencePagesLoaderParams {\n  baseUrl: string;\n  spaceKey: string;\n  username?: string;\n  accessToken?: string;\n  personalAccessToken?: string;\n  limit?: number;\n  expand?: string;\n  maxRetries?: number;\n}\n\n/**\n * Interface representing a Confluence page.\n */\nexport interface ConfluencePage {\n  id: string;\n  title: string;\n  type: string;\n  body: {\n    storage: {\n      value: string;\n    };\n  };\n  status: string;\n  version?: {\n    number: number;\n    when: string;\n    by: {\n      displayName: string;\n    };\n  };\n}\n\n/**\n * Interface representing the response from the Confluence API.\n */\nexport interface ConfluenceAPIResponse {\n  size: number;\n  results: ConfluencePage[];\n}\n\n/**\n * Class representing a document loader for loading pages from Confluence.\n * @example\n * ```typescript\n * const loader = new ConfluencePagesLoader({\n *   baseUrl: \"https:\n *   spaceKey: \"~EXAMPLE362906de5d343d49dcdbae5dEXAMPLE\",\n *   username: \"your-username\",\n *   accessToken: \"your-access-token\",\n * });\n * const documents = await loader.load();\n * console.log(documents);\n * ```\n */\nexport class ConfluencePagesLoader extends BaseDocumentLoader {\n  public readonly baseUrl: string;\n\n  public readonly spaceKey: string;\n\n  public readonly username?: string;\n\n  public readonly accessToken?: string;\n\n  public readonly limit: number;\n\n  public readonly maxRetries: number;\n\n  /**\n   * expand parameter for confluence rest api\n   * description can be found at https://developer.atlassian.com/server/confluence/expansions-in-the-rest-api/\n   */\n  public readonly expand?: string;\n\n  public readonly personalAccessToken?: string;\n\n  constructor({\n    baseUrl,\n    spaceKey,\n    username,\n    accessToken,\n    limit = 25,\n    expand = \"body.storage,version\",\n    personalAccessToken,\n    maxRetries = 5,\n  }: ConfluencePagesLoaderParams) {\n    super();\n    this.baseUrl = baseUrl;\n    this.spaceKey = spaceKey;\n    this.username = username;\n    this.accessToken = accessToken;\n    this.limit = limit;\n    this.expand = expand;\n    this.personalAccessToken = personalAccessToken;\n    this.maxRetries = maxRetries;\n  }\n\n  /**\n   * Returns the authorization header for the request.\n   * @returns The authorization header as a string, or undefined if no credentials were provided.\n   */\n  private get authorizationHeader(): string | undefined {\n    if (this.personalAccessToken) {\n      return `Bearer ${this.personalAccessToken}`;\n    } else if (this.username && this.accessToken) {\n      const authToken = Buffer.from(\n        `${this.username}:${this.accessToken}`\n      ).toString(\"base64\");\n      return `Basic ${authToken}`;\n    }\n\n    return undefined;\n  }\n\n  /**\n   * Fetches all the pages in the specified space and converts each page to\n   * a Document instance.\n   * @param options the extra options of the load function\n   * @param options.limit The limit parameter to overwrite the size to fetch pages.\n   * @param options.start The start parameter to set inital offset to fetch pages.\n   * @returns Promise resolving to an array of Document instances.\n   */\n  public async load(options?: {\n    start?: number;\n    limit?: number;\n  }): Promise<Document[]> {\n    try {\n      const pages = await this.fetchAllPagesInSpace(\n        options?.start,\n        options?.limit\n      );\n      return pages.map((page) => this.createDocumentFromPage(page));\n    } catch (error) {\n      console.error(\"Error:\", error);\n      return [];\n    }\n  }\n\n  /**\n   * Fetches data from the Confluence API using the provided URL.\n   * @param url The URL to fetch data from.\n   * @returns Promise resolving to the JSON response from the API.\n   */\n  protected async fetchConfluenceData(\n    url: string\n  ): Promise<ConfluenceAPIResponse> {\n    let retryCounter = 0;\n    while (true) {\n      retryCounter += 1;\n      try {\n        const initialHeaders: HeadersInit = {\n          \"Content-Type\": \"application/json\",\n          Accept: \"application/json\",\n        };\n\n        const authHeader = this.authorizationHeader;\n        if (authHeader) {\n          initialHeaders.Authorization = authHeader;\n        }\n\n        const response = await fetch(url, {\n          headers: initialHeaders,\n        });\n\n        if (!response.ok) {\n          throw new Error(\n            `Failed to fetch ${url} from Confluence: ${response.status}. Retrying...`\n          );\n        }\n\n        return await response.json();\n      } catch (error) {\n        if (retryCounter >= this.maxRetries)\n          throw new Error(\n            `Failed to fetch ${url} from Confluence (retry: ${retryCounter}): ${error}`\n          );\n      }\n    }\n  }\n\n  /**\n   * Recursively fetches all the pages in the specified space.\n   * @param start The start parameter to paginate through the results.\n   * @returns Promise resolving to an array of ConfluencePage objects.\n   */\n  private async fetchAllPagesInSpace(\n    start = 0,\n    limit = this.limit\n  ): Promise<ConfluencePage[]> {\n    const url = `${this.baseUrl}/rest/api/content?spaceKey=${this.spaceKey}&limit=${limit}&start=${start}&expand=${this.expand}`;\n    const data = await this.fetchConfluenceData(url);\n\n    if (data.size === 0) {\n      return [];\n    }\n\n    const nextPageStart = start + data.size;\n    const nextPageResults = await this.fetchAllPagesInSpace(\n      nextPageStart,\n      limit\n    );\n\n    return data.results.concat(nextPageResults);\n  }\n\n  /**\n   * Creates a Document instance from a ConfluencePage object.\n   * @param page The ConfluencePage object to convert.\n   * @returns A Document instance.\n   */\n  private createDocumentFromPage(page: ConfluencePage): Document {\n    const htmlContent = page.body.storage.value;\n\n    // Handle both self-closing and regular macros for attachments and view-file\n    const htmlWithoutOtherMacros = htmlContent.replace(\n      /<ac:structured-macro\\s+ac:name=\"(attachments|view-file)\"[^>]*(?:\\/?>|>.*?<\\/ac:structured-macro>)/gs,\n      \"[ATTACHMENT]\"\n    );\n\n    // Extract and preserve code blocks with unique placeholders\n    const codeBlocks: { language: string; code: string }[] = [];\n    const htmlWithPlaceholders = htmlWithoutOtherMacros.replace(\n      /<ac:structured-macro.*?<ac:parameter ac:name=\"language\">(.*?)<\\/ac:parameter>.*?<ac:plain-text-body><!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/ac:plain-text-body><\\/ac:structured-macro>/g,\n      (_, language, code) => {\n        const placeholder = `CODE_BLOCK_${codeBlocks.length}`;\n        codeBlocks.push({ language, code: code.trim() });\n        return `\\n${placeholder}\\n`;\n      }\n    );\n\n    // Convert the HTML content to plain text\n    let plainTextContent = htmlToText(htmlWithPlaceholders, {\n      wordwrap: false,\n      preserveNewlines: true,\n    });\n\n    // Reinsert code blocks with proper markdown formatting\n    codeBlocks.forEach(({ language, code }, index) => {\n      const placeholder = `CODE_BLOCK_${index}`;\n      plainTextContent = plainTextContent.replace(\n        placeholder,\n        `\\`\\`\\`${language}\\n${code}\\n\\`\\`\\``\n      );\n    });\n\n    // Remove empty lines\n    const textWithoutEmptyLines = plainTextContent.replace(/^\\s*[\\r\\n]/gm, \"\");\n\n    // Rest of the method remains the same...\n    return new Document({\n      pageContent: textWithoutEmptyLines,\n      metadata: {\n        id: page.id,\n        status: page.status,\n        title: page.title,\n        type: page.type,\n        url: `${this.baseUrl}/spaces/${this.spaceKey}/pages/${page.id}`,\n        version: page.version?.number,\n        updated_by: page.version?.by?.displayName,\n        updated_at: page.version?.when,\n      },\n    });\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AA+DA,IAAa,wBAAb,cAA2CA,0DAAmB;CAC5D,AAAgB;CAEhB,AAAgB;CAEhB,AAAgB;CAEhB,AAAgB;CAEhB,AAAgB;CAEhB,AAAgB;;;;;CAMhB,AAAgB;CAEhB,AAAgB;CAEhB,YAAY,EACV,SACA,UACA,UACA,aACA,QAAQ,IACR,SAAS,wBACT,qBACA,aAAa,GACe,EAAE;EAC9B,OAAO;EACP,KAAK,UAAU;EACf,KAAK,WAAW;EAChB,KAAK,WAAW;EAChB,KAAK,cAAc;EACnB,KAAK,QAAQ;EACb,KAAK,SAAS;EACd,KAAK,sBAAsB;EAC3B,KAAK,aAAa;CACnB;;;;;CAMD,IAAY,sBAA0C;AACpD,MAAI,KAAK,oBACP,QAAO,CAAC,OAAO,EAAE,KAAK,qBAAqB;WAClC,KAAK,YAAY,KAAK,aAAa;GAC5C,MAAM,YAAY,OAAO,KACvB,GAAG,KAAK,SAAS,CAAC,EAAE,KAAK,aAAa,CACvC,CAAC,SAAS,SAAS;AACpB,UAAO,CAAC,MAAM,EAAE,WAAW;EAC5B;AAED,SAAO;CACR;;;;;;;;;CAUD,MAAa,KAAKC,SAGM;AACtB,MAAI;GACF,MAAM,QAAQ,MAAM,KAAK,qBACvB,SAAS,OACT,SAAS,MACV;AACD,UAAO,MAAM,IAAI,CAAC,SAAS,KAAK,uBAAuB,KAAK,CAAC;EAC9D,SAAQ,OAAO;GACd,QAAQ,MAAM,UAAU,MAAM;AAC9B,UAAO,CAAE;EACV;CACF;;;;;;CAOD,MAAgB,oBACdC,KACgC;EAChC,IAAI,eAAe;AACnB,SAAO,MAAM;GACX,gBAAgB;AAChB,OAAI;IACF,MAAMC,iBAA8B;KAClC,gBAAgB;KAChB,QAAQ;IACT;IAED,MAAM,aAAa,KAAK;AACxB,QAAI,YACF,eAAe,gBAAgB;IAGjC,MAAM,WAAW,MAAM,MAAM,KAAK,EAChC,SAAS,eACV,EAAC;AAEF,QAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MACR,CAAC,gBAAgB,EAAE,IAAI,kBAAkB,EAAE,SAAS,OAAO,aAAa,CAAC;AAI7E,WAAO,MAAM,SAAS,MAAM;GAC7B,SAAQ,OAAO;AACd,QAAI,gBAAgB,KAAK,WACvB,OAAM,IAAI,MACR,CAAC,gBAAgB,EAAE,IAAI,yBAAyB,EAAE,aAAa,GAAG,EAAE,OAAO;GAEhF;EACF;CACF;;;;;;CAOD,MAAc,qBACZ,QAAQ,GACR,QAAQ,KAAK,OACc;EAC3B,MAAM,MAAM,GAAG,KAAK,QAAQ,2BAA2B,EAAE,KAAK,SAAS,OAAO,EAAE,MAAM,OAAO,EAAE,MAAM,QAAQ,EAAE,KAAK,QAAQ;EAC5H,MAAM,OAAO,MAAM,KAAK,oBAAoB,IAAI;AAEhD,MAAI,KAAK,SAAS,EAChB,QAAO,CAAE;EAGX,MAAM,gBAAgB,QAAQ,KAAK;EACnC,MAAM,kBAAkB,MAAM,KAAK,qBACjC,eACA,MACD;AAED,SAAO,KAAK,QAAQ,OAAO,gBAAgB;CAC5C;;;;;;CAOD,AAAQ,uBAAuBC,MAAgC;EAC7D,MAAM,cAAc,KAAK,KAAK,QAAQ;EAGtC,MAAM,yBAAyB,YAAY,QACzC,uGACA,eACD;EAGD,MAAMC,aAAmD,CAAE;EAC3D,MAAM,uBAAuB,uBAAuB,QAClD,gLACA,CAAC,GAAG,UAAU,SAAS;GACrB,MAAM,cAAc,CAAC,WAAW,EAAE,WAAW,QAAQ;GACrD,WAAW,KAAK;IAAE;IAAU,MAAM,KAAK,MAAM;GAAE,EAAC;AAChD,UAAO,CAAC,EAAE,EAAE,YAAY,EAAE,CAAC;EAC5B,EACF;EAGD,IAAI,gDAA8B,sBAAsB;GACtD,UAAU;GACV,kBAAkB;EACnB,EAAC;EAGF,WAAW,QAAQ,CAAC,EAAE,UAAU,MAAM,EAAE,UAAU;GAChD,MAAM,cAAc,CAAC,WAAW,EAAE,OAAO;GACzC,mBAAmB,iBAAiB,QAClC,aACA,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,KAAK,QAAQ,CAAC,CACrC;EACF,EAAC;EAGF,MAAM,wBAAwB,iBAAiB,QAAQ,gBAAgB,GAAG;AAG1E,SAAO,IAAIC,oCAAS;GAClB,aAAa;GACb,UAAU;IACR,IAAI,KAAK;IACT,QAAQ,KAAK;IACb,OAAO,KAAK;IACZ,MAAM,KAAK;IACX,KAAK,GAAG,KAAK,QAAQ,QAAQ,EAAE,KAAK,SAAS,OAAO,EAAE,KAAK,IAAI;IAC/D,SAAS,KAAK,SAAS;IACvB,YAAY,KAAK,SAAS,IAAI;IAC9B,YAAY,KAAK,SAAS;GAC3B;EACF;CACF;AACF"}