weseek__growi/packages/markdown-splitter/dist/services/markdown-splitter.cjs

"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const q=require("js-tiktoken"),M=require("js-yaml"),f=require("remark-frontmatter"),y=require("remark-gfm"),S=require("remark-parse"),C=require("remark-stringify"),d=require("unified");function F(t,n){if(n>t.length)for(;t.length<n;)t.push(1);else n===t.length||t.splice(n),t[n-1]++;return t.join("-")}async function w(t,n){const s=[],g=[];let i="";const c={};if(typeof t!="string"||t.trim()==="")return s;const u=q.encodingForModel(n),p=d.unified().use(S).use(f,["yaml"]).use(y),m={bullet:"-",rule:"-"},a=d.unified().use(f,["yaml"]).use(y).use(C,m),k=p.parse(t);for(const o of k.children)if(o.type==="yaml"){const r=M.load(o.value),e=JSON.stringify(r,null,2),l=u.encode(e).length;s.push({label:"frontmatter",type:"yaml",text:e,tokenCount:l})}else if(o.type==="heading"){const r=o.depth;i=F(g,r);const e=a.stringify(o).trim(),l=u.encode(e).length;s.push({label:`${i}-heading`,type:o.type,text:e,tokenCount:l})}else{const r=a.stringify(o).trim();if(r!==""){const e=i||"0";c[e]?c[e]++:c[e]=1;const l=i!==""?`${i}-content-${c[e]}`:`0-content-${c[e]}`,h=u.encode(r).length;s.push({label:l,type:o.type,text:r,tokenCount:h})}}return s}exports.splitMarkdownIntoFragments=w;
//# sourceMappingURL=markdown-splitter.cjs.map