use langchain

packages/markdown-token-splitter/package.json

@@ -34,10 +34,7 @@
     "react-dom": "^18.2.0"
   },
   "dependencies": {
-    "js-tiktoken": "^1.0.14",
-    "remark-parse": "^10.0.0",
-    "remark-stringify": "^10.0.0",
-    "tiktoken": "^1.0.16",
-    "unified": "^10.1.2"
+    "@langchain/core": "^0.3.5",
+    "langchain": "^0.3.2"
   }
 }

packages/markdown-token-splitter/src/services/markdown-token-splitter.ts

@@ -1,89 +1,24 @@
-import type { TiktokenModel } from 'js-tiktoken';
-import { encodingForModel } from 'js-tiktoken';
-import type { Root, Content } from 'mdast';
-import remarkParse from 'remark-parse';
-import remarkStringify from 'remark-stringify';
-import { unified } from 'unified';
-
-export function splitMarkdownByTokens(
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+
+/**
+ * Function to recursively split a markdown string by header sections (and within subsections if they exceed the specified max token count).
+ *
+ * @param markdownString - The input markdown string
+ * @param chunkSize - The chunk size for splitting (default is 1000)
+ * @returns An array of split markdown sections
+ */
+export async function splitMarkdownByTokens(
     markdownString: string,
-    maxTokens: number,
-    modelName: TiktokenModel,
-): string[] {
-  // Parse the markdown into an AST
-  const tree = unified().use(remarkParse).parse(markdownString) as Root;
-  const encoding = encodingForModel(modelName);
-
-  function countTokens(text: string): number {
-    const tokens = encoding.encode(text);
-    return tokens.length;
-  }
-
-  // Recursively split sections
-  function splitSectionRecursively(nodes: Content[]): Content[][] {
-    const sections: Content[][] = [];
-    let currentSection: Content[] = [];
-
-    for (let i = 0; i < nodes.length; i++) {
-      const node = nodes[i];
-      currentSection.push(node);
-
-      const markdown = unified()
-        .use(remarkStringify)
-        .stringify({ type: 'root', children: currentSection });
-      const tokenCount = countTokens(markdown);
-
-      if (tokenCount > maxTokens) {
-        // If the token count exceeds the limit, treat the nodes up to the previous one as a section
-        currentSection.pop();
-        if (currentSection.length > 0) {
-          sections.push([...currentSection]);
-        }
-        // Start a new section from the current node
-        currentSection = [node];
-
-        // If a single node exceeds maxTokens, add it as its own section
-        const singleNodeMarkdown = unified()
-          .use(remarkStringify)
-          .stringify({ type: 'root', children: [node] });
-        const singleNodeTokenCount = countTokens(singleNodeMarkdown);
-        if (singleNodeTokenCount > maxTokens) {
-          sections.push([node]);
-          currentSection = [];
-        }
-      }
-
-      // If it's the last node, add the section
-      if (i === nodes.length - 1 && currentSection.length > 0) {
-        sections.push([...currentSection]);
-      }
-    }
-
-    // Recursively split each section
-    const recursivelySplitSections: Content[][] = [];
-    for (const section of sections) {
-      // If the section contains child headings, split further
-      const hasHeading = section.some(node => node.type === 'heading');
-      if (hasHeading && countTokens(unified().use(remarkStringify).stringify({ type: 'root', children: section })) > maxTokens) {
-        // Recursively split child nodes
-        const childSections = splitSectionRecursively(section);
-        recursivelySplitSections.push(...childSections);
-      }
-      else {
-        recursivelySplitSections.push(section);
-      }
-    }
-
-    return recursivelySplitSections;
-  }
+    chunkSize = 1000, // Default chunk size set to 1000
+): Promise<Document[]> {
+  const validMarkdownString = markdownString || '';
 
-  // Recursively split the AST's child nodes
-  const splitSections = splitSectionRecursively(tree.children);
+  const mdSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize, // Use the provided chunkSize
+    chunkOverlap: 0,
+  });
 
-  // Convert the split sections back into markdown strings
-  const markdownSections = splitSections.map(sectionNodes => unified()
-    .use(remarkStringify)
-    .stringify({ type: 'root', children: sectionNodes }));
+  const mdDocs = await mdSplitter.createDocuments([validMarkdownString]);
 
-  return markdownSections;
+  return mdDocs;
 }

packages/markdown-token-splitter/test/index.test.js

@@ -1,142 +1,104 @@
-import { encodingForModel } from 'js-tiktoken';
-import test from 'tape';
+import { describe, it, expect } from 'vitest';
 
 import { splitMarkdownByTokens } from '../src/services/markdown-token-splitter';
 
-const modelName = 'gpt-3.5-turbo'; // Replace with the appropriate model name if needed
+describe('splitMarkdownByTokens', () => {
 
-test('should split markdown into sections under the max token limit', (t) => {
-  const markdown = `
+  it('should split markdown into sections using the specified chunk size', async() => {
+    const markdown = `
 # Heading 1
-Content under heading 1.
+This is some content under heading 1.
 
 # Heading 2
-Content under heading 2.
-
-## Subheading 2.1
-Content under subheading 2.1.
+This is some content under heading 2.
 
 # Heading 3
-Content under heading 3.
-  `;
-  const maxTokens = 50;
+This is some content under heading 3.
 
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.ok(sections.length > 1, 'Sections should be more than one');
-  sections.forEach((section) => {
-    const tokens = encodingForModel(modelName).encode(section);
-    t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
+# Heading 4
+This is some content under heading 4.
+`;
+    const chunkSize = 60;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
+
+    // Expect the result to have more than one section due to chunkSize limitations
+    expect(result.length).toBeGreaterThan(1);
+    for (const section of result) {
+      expect(section.pageContent.length).toBeLessThanOrEqual(chunkSize);
+    }
   });
 
-  t.end();
-});
-
-test('should handle markdown without headings', (t) => {
-  const markdown = 'This is a markdown without any headings. It should be returned as a single section unless it exceeds the max token limit.';
-  const maxTokens = 100;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.equal(sections.length, 1, 'Sections should be exactly one');
-  const tokens = encodingForModel(modelName).encode(sections[0]);
-  t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
+  it('should handle markdown without headers', async() => {
+    const markdown = `
+This is some content without any headers. It should not be split unless it exceeds the chunk size.
+`;
+    const chunkSize = 100;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
 
-  t.end();
-});
+    // Since the content is short, expect no splits
+    expect(result.length).toBe(1);
+    expect(result[0].pageContent.length).toBeLessThanOrEqual(chunkSize);
+  });
 
-test('should split large content under a single heading', (t) => {
-  const markdown = `
+  it('should split large content under a single heading', async() => {
+    const markdown = `
 # Large Heading
-${'Long content. '.repeat(500)}
-  `;
-  const maxTokens = 100;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.ok(sections.length > 1, 'Sections should be more than one');
-  sections.forEach((section) => {
-    const tokens = encodingForModel(modelName).encode(section);
-    t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
+${'This is some repetitive content. '.repeat(50)}
+`;
+    const chunkSize = 100;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
+
+    expect(result.length).toBeGreaterThan(1);
+    for (const section of result) {
+      expect(section.pageContent.length).toBeLessThanOrEqual(chunkSize);
+    }
   });
 
-  t.end();
-});
+  it('should handle empty markdown input', async() => {
+    const markdown = '';
+    const chunkSize = 10;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
 
-test('should handle empty markdown input', (t) => {
-  const markdown = '';
-  const maxTokens = 50;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.equal(sections.length, 0, 'Sections should be zero');
-
-  t.end();
-});
-
-test('should handle markdown where a single node exceeds maxTokens', (t) => {
-  const markdown = `
-# Heading with Large Content
-${'Very long content. '.repeat(1000)}
-  `;
-  const maxTokens = 10;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.ok(sections.length > 1, 'Sections should be more than one');
-  sections.forEach((section) => {
-    const tokens = encodingForModel(modelName).encode(section);
-    t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
+    // Expect an empty result for empty markdown input
+    expect(result.length).toBe(0);
   });
 
-  t.end();
-});
-
-test('should correctly split nested headings', (t) => {
-  const markdown = `
+  it('should correctly split nested headings', async() => {
+    const markdown = `
 # Heading 1
 Content under heading 1.
 
 ## Subheading 1.1
 Content under subheading 1.1.
 
-### Sub-subheading 1.1.1
-Content under sub-subheading 1.1.1.
-
 # Heading 2
 Content under heading 2.
-  `;
-  const maxTokens = 30;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.ok(sections.length > 1, 'Sections should be more than one');
-  sections.forEach((section) => {
-    const tokens = encodingForModel(modelName).encode(section);
-    t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
+`;
+    const chunkSize = 50;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
+
+    // Expect multiple sections
+    expect(result.length).toBeGreaterThan(1);
+    for (const section of result) {
+      expect(section.pageContent.length).toBeLessThanOrEqual(chunkSize);
+    }
   });
 
-  t.end();
-});
-
-test('should not split sections unnecessarily', (t) => {
-  const markdown = `
-# Short Heading
+  it('should not split if content fits within chunk size', async() => {
+    const markdown = `
+# Heading
 Short content.
-  `;
-  const maxTokens = 100;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
+`;
+    const chunkSize = 100;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
 
-  t.equal(sections.length, 1, 'Sections should be exactly one');
-  const tokens = encodingForModel(modelName).encode(sections[0]);
-  t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
-
-  t.end();
-});
+    // Expect the result to be a single section since the content is small
+    expect(result.length).toBe(1);
+    expect(result[0].pageContent.length).toBeLessThanOrEqual(chunkSize);
+  });
 
-test('should handle multiple consecutive headings', (t) => {
-  const markdown = `
+  it('should handle multiple consecutive headers', async() => {
+    const markdown = `
 # Heading 1
 
 # Heading 2
@@ -144,16 +106,14 @@ test('should handle multiple consecutive headings', (t) => {
 # Heading 3
 
 # Heading 4
-  `;
-  const maxTokens = 10;
-
-  const sections = splitMarkdownByTokens(markdown, maxTokens, modelName);
-
-  t.equal(sections.length, 4, 'Sections should be four');
-  sections.forEach((section) => {
-    const tokens = encodingForModel(modelName).encode(section);
-    t.ok(tokens.length <= maxTokens, 'Section tokens should be less than or equal to maxTokens');
+`;
+    const chunkSize = 50;
+    const result = await splitMarkdownByTokens(markdown, chunkSize);
+
+    // Expect each heading to be treated as a separate section
+    expect(result.length).toBeGreaterThan(1);
+    for (const section of result) {
+      expect(section.pageContent.length).toBeLessThanOrEqual(chunkSize);
+    }
   });
-
-  t.end();
 });

yarn.lock

@@ -2182,11 +2182,8 @@
 "@growi/markdown-token-splitter@link:packages/markdown-token-splitter":
   version "1.0.0"
   dependencies:
-    js-tiktoken "^1.0.14"
-    remark-parse "^10.0.0"
-    remark-stringify "^10.0.0"
-    tiktoken "^1.0.16"
-    unified "^10.1.2"
+    "@langchain/core" "^0.3.5"
+    langchain "^0.3.2"
 
 "@growi/pluginkit@link:packages/pluginkit":
   version "1.0.1"
@@ -2599,6 +2596,40 @@
   dependencies:
     "@types/react" ">=16.0.0"
 
+"@langchain/core@^0.3.5":
+  version "0.3.5"
+  resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.3.5.tgz#e5229dc5d65422ea53eb029c91e6fc3423bec7e9"
+  integrity sha512-Zr71YP9k30mShviYLcb7TKOYTbSyBu68xncfwVuU5bAVPXCYMJWLY+OQbOELnGuIDJ3oWTEH3TzQYYNUF8kEnw==
+  dependencies:
+    ansi-styles "^5.0.0"
+    camelcase "6"
+    decamelize "1.2.0"
+    js-tiktoken "^1.0.12"
+    langsmith "^0.1.56"
+    mustache "^4.2.0"
+    p-queue "^6.6.2"
+    p-retry "4"
+    uuid "^10.0.0"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
+"@langchain/openai@>=0.1.0 <0.4.0":
+  version "0.3.4"
+  resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.3.4.tgz#fa9db2679b97d543023dea4835e1b52675dc23cf"
+  integrity sha512-XHmtRIvDPMnk+c/FUTqL93WEORCFgCLWakebSRovuvx7UAQS+cnW3al47GEkG433XTyEdZynJE0ilTgAesbKsw==
+  dependencies:
+    js-tiktoken "^1.0.12"
+    openai "^4.57.3"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
+"@langchain/textsplitters@>=0.0.0 <0.2.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-0.1.0.tgz#f37620992192df09ecda3dfbd545b36a6bcbae46"
+  integrity sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==
+  dependencies:
+    js-tiktoken "^1.0.12"
+
 "@ldapjs/asn1@2.0.0":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@ldapjs/asn1/-/asn1-2.0.0.tgz#e25fa38fcf0b4310275d6a5a05fe4603efef5eb4"
@@ -4607,6 +4638,14 @@
     "@types/node" "*"
     form-data "^4.0.0"
 
+"@types/node-fetch@^2.6.4":
+  version "2.6.11"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.11.tgz#9b39b78665dae0e82a08f02f4967d62c66f95d24"
+  integrity sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==
+  dependencies:
+    "@types/node" "*"
+    form-data "^4.0.0"
+
 "@types/node@*", "@types/node@>=10.0.0", "@types/node@>=12", "@types/node@>=12.0.0", "@types/node@>=8.9.0", "@types/node@^20.14.0":
   version "20.14.0"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.0.tgz#49ceec7b34f8621470cff44677fa9d461a477f17"
@@ -4624,6 +4663,13 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.5.tgz#0dd636fe7b2c6055cbed0d4ca3b7fb540f130a96"
   integrity sha512-LMy+vDDcQR48EZdEx5wRX1q/sEl6NdGuHXPnfeL8ixkwCOSZ2qnIyIZmcCbdX0MeRqHhAcHmX+haCbrS8Run+A==
 
+"@types/node@^18.11.18":
+  version "18.19.54"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.54.tgz#f1048dc083f81b242640f04f18fb3e4ccf13fcdb"
+  integrity sha512-+BRgt0G5gYjTvdLac9sIeE0iZcJxi4Jc4PV5EUzqi+88jmQLr+fRZdv2tCTV7IHKSGxM6SaLoOXQWWUiLUItMw==
+  dependencies:
+    undici-types "~5.26.4"
+
 "@types/normalize-package-data@^2.4.0":
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz#e486d0d97396d79beedd0a6e33f4534ff6b4973e"
@@ -4710,7 +4756,7 @@
     "@types/scheduler" "*"
     csstype "^3.0.2"
 
-"@types/retry@^0.12.0":
+"@types/retry@0.12.0", "@types/retry@^0.12.0":
   version "0.12.0"
   resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d"
   integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==
@@ -4827,6 +4873,11 @@
   resolved "https://registry.yarnpkg.com/@types/url-join/-/url-join-4.0.2.tgz#e8774924c7f492626ee3309baf6697f80e1414df"
   integrity sha512-uv54MkAtQ4B5Qm20LmMN7tAdczqRenu1K6Sf7PHCygqylVJlRwjpUE5OGofqxdXGH3QJUu+qvDZzPadz5EOjxA==
 
+"@types/uuid@^10.0.0":
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-10.0.0.tgz#e9c07fe50da0f53dc24970cca94d619ff03f6f6d"
+  integrity sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==
+
 "@types/warning@^3.0.0":
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/@types/warning/-/warning-3.0.0.tgz#0d2501268ad8f9962b740d387c4654f5f8e23e52"
@@ -6368,6 +6419,11 @@ camelcase-keys@^6.2.2:
     map-obj "^4.0.0"
     quick-lru "^4.0.1"
 
+camelcase@6, camelcase@^6.2.0, camelcase@^6.3.0:
+  version "6.3.0"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
+  integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
+
 camelcase@^2.0.0:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f"
@@ -6378,11 +6434,6 @@ camelcase@^5.0.0, camelcase@^5.3.1:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320"
   integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==
 
-camelcase@^6.2.0, camelcase@^6.3.0:
-  version "6.3.0"
-  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a"
-  integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==
-
 camelize-ts@^1.0.8:
   version "1.0.9"
   resolved "https://registry.yarnpkg.com/camelize-ts/-/camelize-ts-1.0.9.tgz#6ac46fbe660d18e093568ef0d56c836141b700f4"
@@ -6855,7 +6906,7 @@ commander@9.2.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-9.2.0.tgz#6e21014b2ed90d8b7c9647230d8b7a94a4a419a9"
   integrity sha512-e2i4wANQiSXgnrBlIatyHtP1odfUp0BbV5Y5nEGbxtIrStkEOAAzCUirvLBNXHLr7kwLvJl6V+4V3XV9x7Wd9w==
 
-commander@^10.0.0:
+commander@^10.0.0, commander@^10.0.1:
   version "10.0.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06"
   integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==
@@ -7758,7 +7809,7 @@ decamelize-keys@^1.1.0:
     decamelize "^1.1.0"
     map-obj "^1.0.0"
 
-decamelize@^1.1.0, decamelize@^1.1.2, decamelize@^1.2.0:
+decamelize@1.2.0, decamelize@^1.1.0, decamelize@^1.1.2, decamelize@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
 
@@ -9391,6 +9442,11 @@ forever-agent@~0.6.1:
   resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
   integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=
 
+form-data-encoder@1.7.2:
+  version "1.7.2"
+  resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040"
+  integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==
+
 form-data@^2.5.0:
   version "2.5.1"
   resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4"
@@ -9423,6 +9479,14 @@ format@^0.2.0:
   resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b"
   integrity sha1-1hcBB+nv3E7TDJ3DkBbflCtctYs=
 
+formdata-node@^4.3.2:
+  version "4.4.1"
+  resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2"
+  integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==
+  dependencies:
+    node-domexception "1.0.0"
+    web-streams-polyfill "4.0.0-beta.3"
+
 forwarded@0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811"
@@ -11569,7 +11633,7 @@ js-sha256@^0.9.0:
   resolved "https://registry.yarnpkg.com/js-sha256/-/js-sha256-0.9.0.tgz#0b89ac166583e91ef9123644bd3c5334ce9d0966"
   integrity sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==
 
-js-tiktoken@^1.0.14:
+js-tiktoken@^1.0.12:
   version "1.0.14"
   resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.14.tgz#756f353262d559da16b58b5bcecfd93330076da2"
   integrity sha512-Pk3l3WOgM9joguZY2k52+jH82RtABRgB5RdGFZNUGbOKGMVlNmafcPA3b0ITcCZPu1L9UclP1tne6aw7ZI4Myg==
@@ -11701,6 +11765,11 @@ jsonparse@^1.2.0:
   resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"
   integrity sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=
 
+jsonpointer@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559"
+  integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==
+
 jsonwebtoken@^8.5.1:
   version "8.5.1"
   resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d"
@@ -11877,6 +11946,36 @@ kruptein@^3.0.0:
   dependencies:
     asn1.js "^5.4.1"
 
+langchain@^0.3.2:
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.3.2.tgz#aec3e679d3d6c36f469448380affa475c92fbd86"
+  integrity sha512-kd2kz1cS/PIVrLEDFlrZsAasQfPLbY1UqCZbRKa3/QcpB33/n6xPDvXSMfBuKhvNj0bjW6MXDR9HZTduXjJBgg==
+  dependencies:
+    "@langchain/openai" ">=0.1.0 <0.4.0"
+    "@langchain/textsplitters" ">=0.0.0 <0.2.0"
+    js-tiktoken "^1.0.12"
+    js-yaml "^4.1.0"
+    jsonpointer "^5.0.1"
+    langsmith "^0.1.56-rc.1"
+    openapi-types "^12.1.3"
+    p-retry "4"
+    uuid "^10.0.0"
+    yaml "^2.2.1"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
+langsmith@^0.1.56, langsmith@^0.1.56-rc.1:
+  version "0.1.61"
+  resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.1.61.tgz#4c3d2677fe79d9f936198918c3af6c48e54f79d7"
+  integrity sha512-XQE4KPScwPmdaT0mWDzhNxj9gvqXUR+C7urLA0QFi27XeoQdm17eYpudenn4wxC0gIyUJutQCyuYJpfwlT5JnQ==
+  dependencies:
+    "@types/uuid" "^10.0.0"
+    commander "^10.0.1"
+    p-queue "^6.6.2"
+    p-retry "4"
+    semver "^7.6.3"
+    uuid "^10.0.0"
+
 language-subtag-registry@~0.3.2:
   version "0.3.21"
   resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a"
@@ -13772,6 +13871,11 @@ node-cron@^3.0.2:
   dependencies:
     uuid "8.3.2"
 
+node-domexception@1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
+  integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
+
 node-emoji@^1.11.0:
   version "1.11.0"
   resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"
@@ -14184,6 +14288,24 @@ open@^8.0.0, open@^8.4.0:
     is-docker "^2.1.1"
     is-wsl "^2.2.0"
 
+openai@^4.57.3:
+  version "4.66.1"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-4.66.1.tgz#07d228cb52d58471e512bad8669e7d001e97791d"
+  integrity sha512-+sSyV6VtGHerPb6Kfi76hrEOVt+wayvuw7GX/ky7rAR11kN6JVs4dgbtoaxzDXvAxc5dcQLuXeco54mBfOMoQQ==
+  dependencies:
+    "@types/node" "^18.11.18"
+    "@types/node-fetch" "^2.6.4"
+    abort-controller "^3.0.0"
+    agentkeepalive "^4.2.1"
+    form-data-encoder "1.7.2"
+    formdata-node "^4.3.2"
+    node-fetch "^2.6.7"
+
+openapi-types@^12.1.3:
+  version "12.1.3"
+  resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3"
+  integrity sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==
+
 opener@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598"
@@ -14310,7 +14432,7 @@ p-map@^4.0.0:
   dependencies:
     aggregate-error "^3.0.0"
 
-p-queue@^6.6.1:
+p-queue@^6.6.1, p-queue@^6.6.2:
   version "6.6.2"
   resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426"
   integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==
@@ -14318,6 +14440,14 @@ p-queue@^6.6.1:
     eventemitter3 "^4.0.4"
     p-timeout "^3.2.0"
 
+p-retry@4:
+  version "4.6.2"
+  resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16"
+  integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==
+  dependencies:
+    "@types/retry" "0.12.0"
+    retry "^0.13.1"
+
 p-retry@^4.0.0:
   version "4.2.0"
   resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.2.0.tgz#ea9066c6b44f23cab4cd42f6147cdbbc6604da5d"
@@ -16050,7 +16180,7 @@ retry-request@^4.2.2:
     debug "^4.1.1"
     extend "^3.0.2"
 
-retry@0.13.1:
+retry@0.13.1, retry@^0.13.1:
   version "0.13.1"
   resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658"
   integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==
@@ -16278,7 +16408,7 @@ secure-json-parse@^2.4.0:
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
 
-semver@>=7.3.2, semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.3, semver@^7.5.4:
+semver@>=7.3.2, semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3:
   version "7.6.3"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
   integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
@@ -17672,11 +17802,6 @@ through2@^2.0.0, through2@~2.0.0:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
 
-tiktoken@^1.0.16:
-  version "1.0.16"
-  resolved "https://registry.yarnpkg.com/tiktoken/-/tiktoken-1.0.16.tgz#efa91e6bb653dfbe81455865ea90ee845de915ad"
-  integrity sha512-hRcORIGF2YlAgWx3nzrGJOrKSJwLoc81HpXmMQk89632XAgURc7IeV2FgQ2iXo9z/J96fCvpsHg2kWoHcbj9fg==
-
 tiny-commit-walker@^1.1.2:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/tiny-commit-walker/-/tiny-commit-walker-1.2.1.tgz#937bfba0abe1d520ae636f4095de62e239ba5498"
@@ -18560,6 +18685,11 @@ uuid@>=8.1.0, uuid@^9.0.0:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
   integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
 
+uuid@^10.0.0:
+  version "10.0.0"
+  resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294"
+  integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==
+
 uuid@^3.1.0, uuid@^3.3.2:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@@ -18784,6 +18914,11 @@ web-namespaces@^2.0.0:
   resolved "https://registry.yarnpkg.com/web-namespaces/-/web-namespaces-2.0.1.tgz#1010ff7c650eccb2592cebeeaf9a1b253fd40692"
   integrity sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==
 
+web-streams-polyfill@4.0.0-beta.3:
+  version "4.0.0-beta.3"
+  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
+  integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
+
 web-worker@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/web-worker/-/web-worker-1.2.0.tgz#5d85a04a7fbc1e7db58f66595d7a3ac7c9c180da"
@@ -19195,6 +19330,11 @@ yaml@^2.0.0:
   resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.4.tgz#53fc1d514be80aabf386dc6001eb29bf3b7523b2"
   integrity sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==
 
+yaml@^2.2.1:
+  version "2.5.1"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.5.1.tgz#c9772aacf62cb7494a95b0c4f1fb065b563db130"
+  integrity sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==
+
 yargs-parser@^18.1.2, yargs-parser@^18.1.3:
   version "18.1.3"
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-18.1.3.tgz#be68c4975c6b2abf469236b0c870362fab09a7b0"
@@ -19338,6 +19478,16 @@ zip-stream@^4.1.0:
     compress-commons "^4.1.0"
     readable-stream "^3.6.0"
 
+zod-to-json-schema@^3.22.3:
+  version "3.23.3"
+  resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.3.tgz#56cf4e0bd5c4096ab46e63159e20998ec7b19c39"
+  integrity sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==
+
+zod@^3.22.4:
+  version "3.23.8"
+  resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
+  integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==
+
 zwitch@^1.0.0:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/zwitch/-/zwitch-1.0.5.tgz#d11d7381ffed16b742f6af7b3f223d5cd9fe9920"