markdown-splitter.d.ts 588 B

123456789101112131415
  1. import { TiktokenModel } from 'js-tiktoken';
  2. export type MarkdownFragment = {
  3. label: string;
  4. type: string;
  5. text: string;
  6. tokenCount: number;
  7. };
  8. /**
  9. * Splits Markdown text into labeled markdownFragments using remark-parse and remark-stringify,
  10. * processing each content node separately and labeling them as 1-content-1, 1-content-2, etc.
  11. * @param markdownText - The input Markdown string.
  12. * @returns An array of labeled markdownFragments.
  13. */
  14. export declare function splitMarkdownIntoFragments(markdownText: string, model: TiktokenModel): Promise<MarkdownFragment[]>;