grix-connector/dist/core/text-segmentation/safe-markdown-stream-segmenter.js

Connect local AI coding agents (Claude, Codex, Gemini, Qwen, DeepSeek, Cursor, OpenCode, Pi, OpenHuman, Reasonix) to the Grix scheduling platform. Also serves as an OpenClaw plugin for Grix channel transport.

const R=3e3,y=4500;class H{enabled;targetChars;hardLimitChars;segmentChars=0;state={inFence:!1,fenceMarker:"```"};constructor(e){this.enabled=e?.enabled??!0,this.targetChars=Math.max(256,Number(e?.targetChars??3e3)),this.hardLimitChars=Math.max(this.targetChars+128,Number(e?.hardLimitChars??4500))}push(e){if(!e)return[];if(!this.enabled)return this.segmentChars+=e.length,this.state=m(e,this.state),[{text:e,closeAfter:!1}];const s=[];let t=e;for(;t.length>0;){const r=this.targetChars-this.segmentChars,i=this.segmentChars>=this.hardLimitChars;if(r>0&&t.length<=r){this.segmentChars+=t.length,this.state=m(t,this.state),s.push({text:t,closeAfter:!1});break}const h=T(t,this.state),a=Math.max(64,Math.floor(this.targetChars*.75)-this.segmentChars),o=Math.max(128,this.hardLimitChars-this.segmentChars),c=0,l=Math.max(0,Math.floor(this.targetChars*.3)-this.segmentChars),g=Math.max(32,Math.floor(this.targetChars*.5)-this.segmentChars),u=F(h,a,o,c,l,g);if(u>0&&u<t.length){const f=t.slice(0,u),A=m(f,this.state);this.segmentChars+=f.length,s.push({text:f,closeAfter:!0,reason:"threshold"}),t=t.slice(u),this.segmentChars=0,this.state=A;continue}if(!i&&this.segmentChars+t.length<this.hardLimitChars){this.segmentChars+=t.length,this.state=m(t,this.state),s.push({text:t,closeAfter:!1});break}const x=Math.max(128,this.hardLimitChars-this.segmentChars),p=Math.min(t.length,x);if(p>=t.length){this.segmentChars+=t.length,this.state=m(t,this.state),this.state.inFence?s.push({text:t,closeAfter:!1}):(s.push({text:t,closeAfter:!0,reason:"hard_limit"}),this.segmentChars=0);break}const M=this.state.inFence?p:L(t,p);let C=t.slice(0,M),d=t.slice(M);if(this.state.inFence){const f=this.state.fenceMarker||"```";C=`${C}
${f}`,d=`${f}
${d}`}s.push({text:C,closeAfter:!0,reason:"hard_limit"}),t=d,this.segmentChars=0,this.state={inFence:!1,fenceMarker:this.state.fenceMarker||"```"}}return k(s)}finish(){return[]}reset(){this.segmentChars=0,this.state={inFence:!1,fenceMarker:"```"}}}function L(n,e){const s=Math.max(0,e-300),r=n.slice(s,e).lastIndexOf(`
`);if(r>=0)return s+r+1;const i=n.indexOf(`
`,e);return i>=0&&i<e+300?i+1:e}function k(n){const e=[];for(const s of n){if(!s.text)continue;const t=e[e.length-1];if(t&&!t.closeAfter&&!s.closeAfter&&!t.reason&&!s.reason){t.text+=s.text;continue}e.push(s)}return e}function F(n,e,s,t,r,i){const h=[{priorities:[6,5],min:t},{priorities:[4,3],min:r},{priorities:[2],min:i},{priorities:[1,0],min:e}];for(const a of h){let o=-1;for(const c of n)a.priorities.includes(c.priority)&&c.pos>=a.min&&c.pos<=s&&(o=c.pos);if(o>=0)return o}return-1}function _(n){const e=n.match(/^\s*(#{1,6})(?:\s|$)/);return e?e[1].length:0}function S(n){return/^\s*([-*_])\s*\1(?:\s*\1){1,}\s*$/.test(n)}function b(n){return/^\s*(?:[-*+]|\d+[.)]) /.test(n)}function T(n,e){const s=[];let t={...e};const r=n.split(`
`);let i=0;for(let h=0;h<r.length;h++){const a=r[h]??"",o=h<r.length-1?`${a}
`:a;if(t=m(o,t),i+=o.length,t.inFence||h>=r.length-1)continue;const c=r[h+1]??"",l=_(c);if(l>=1&&l<=6){const g=l===1?6:l===2?5:l===3?4:3;s.push({pos:i,priority:g});continue}if(S(a)){s.push({pos:i,priority:3});continue}if(a.trim()===""){s.push({pos:i,priority:2});continue}if(b(a)){s.push({pos:i,priority:1});continue}/[。！？.!?]\s*$/.test(a)&&s.push({pos:i,priority:0})}return s}function m(n,e){let s=e.inFence,t=e.fenceMarker;const r=n.split(`
`);for(const i of r){const a=(i??"").trimStart();if(a.startsWith("```")||a.startsWith("~~~")){const o=a.startsWith("~~~")?"~~~":"```";s?o===t&&(s=!1):(s=!0,t=o)}}return{inFence:s,fenceMarker:t}}export{H as SafeMarkdownStreamSegmenter};