[{"data":1,"prerenderedAt":325},["ShallowReactive",2],{"project-43652968":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":35,"aspects":36,"strip":43,"post_text":45,"what":46,"said":82,"maker_in_thread":184,"shape":195,"grid":199,"attempts":287,"same_week":288,"maker_shows":38,"maker_hidden":318,"reader_version":319,"card":320},"43652968","Chonky","a neural approach for text semantic chunking","title","Show HN: Chonky – a neural approach for text semantic chunking","https:\u002F\u002Fgithub.com\u002Fmirth\u002Fchonky","github.com","hessdalenlight",1744373919,2025,169,35,416,3837,11,"notable","ai","text chunking model","library",[24,25,26,27,28],"semantic chunking","text splitter","rag","distilbert","paragraph segmentation","debated",-0.03,{"positive":32,"neutral":33,"mixed":34,"critical":32},4,8,0,16,{"idea":37,"performance":39,"docs":41},{"positive":38,"critical":34},2,{"positive":40,"critical":40},1,{"positive":34,"critical":42},3,[34,42,44,34,34,34,34,34,34,34,40,40,34,34,40,34,34,34,40,42,44,42,34,44,34,42,34,42,34,34,42,34,42,44,42],5,"TLDR: I’ve made a transformer model and a wrapper library that segments text into meaningful semantic chunks.\n\nThe current text splitting approaches rely on heuristics (although one can use neural embedder to group semantically related sentences).\n\nI propose a fully neural approach to semantic chunking.\n\nI took the base distilbert model and trained it on a bookcorpus to split concatenated text paragraphs into original paragraphs. Basically it’s a token classification task. Model fine-tuning took day and a half on a 2x1080ti.\n\nThe library could be used as a text splitter module in a RAG system ",{"problem":47,"audience":53,"pricing":57,"motivation":58,"time_to_build":62,"stack":69,"openness":79,"stage":80,"team_size":81},{"value":48,"source":49,"evidence":50},"Heuristic text splitting is weak for RAG; a neural model splits concatenated text into original paragraphs","maker",[51],{"id":52},"post",{"value":54,"source":49,"evidence":55},"Developers building RAG systems or processing transcripts",[56],{"id":52},null,{"value":59,"source":49,"evidence":60},"Improve text splitting over heuristics for RAG pipelines",[61],{"id":52},{"value":63,"source":49,"evidence":64},"Model fine-tuning took a day and a half on 2x1080ti",[65,66],{"id":52},{"id":67,"by":11,"text":68},"43672420","Actually day and a half :). I'm all for it but unfortunately I have pretty old hardware.",[70,75],{"value":71,"source":72,"evidence":73},"Python","inferred",[74],{"id":52},{"value":76,"source":49,"evidence":77},"DistilBERT (fine-tuned token classification)",[78],{"id":52},"open_source","launched","unknown",{"praise":83,"objections":111,"requests":138,"questions":158,"alternatives":176},[84,100],{"value":85,"source":86,"evidence":87},"Interesting and impressive approach","commenter",[88,92,96],{"id":89,"by":90,"text":91},"43673634","rekovacs","Really amazing and impressive work!",{"id":93,"by":94,"text":95},"43673599","legel","Very cool! The training objective is clever. The 50+ filters at Ecodash.ai for 90,000 plants came from a custom RAG model on top of 800,000 raw web pages. Because LLM’s are…",{"id":97,"by":98,"text":99},"43673635","kamranjon","Interesting! I worked previously for a company that did automatic generation of short video clips from long videos. I fine-tuned a t5 model by taking many Wikipedia articles and…",{"value":101,"source":86,"evidence":102},"Useful for noisy transcripts and OCR'd PDFs",[103,107],{"id":104,"by":105,"text":106},"43673879","vunderba","This is the library that I use, mainly around very noisy IRC chat transcripts and it works pretty well. OP I'd love to see a paragraph matching comparison benchmark against…",{"id":108,"by":109,"text":110},"43672993","cckolon","This would be useful when chunking PDFs scanned with OCR. I've done that before and paragraph breaks were detected pretty inconsistently.",[112,123],{"value":113,"source":86,"evidence":114},"No benchmarks showing performance against alternatives",[115,119],{"id":116,"by":117,"text":118},"43671099","mentalgear","I applaud the FOSS initiative but as with anything ml: benchmarks please so we can see what test cases are covered and how well they align with a project's needs.",{"id":120,"by":121,"text":122},"43671018","jaggirs","Did you evaluate it on a RAG benchmark?",{"value":124,"source":86,"evidence":125},"README lacked example output",[126,130,134],{"id":127,"by":128,"text":129},"43671086","suddenlybananas","I feel you could improve your README.md considerably just by showing the actual output of the little snippet you show.",{"id":131,"by":132,"text":133},"43671131","HeavyStorm","Came here to write exactly that. The author includes a large sentence in the sample, so it should show us the output.",{"id":135,"by":136,"text":137},"43671108","petesergeant","Love that people are trying to improve chunkers, but just some examples of how it chunked some input text in the README would go a long way here!",[139,143,154],{"value":140,"source":86,"evidence":141},"Benchmark against wtpsplit",[142],{"id":104,"by":105,"text":106},{"value":144,"source":86,"evidence":145},"Support for other languages",[146,150],{"id":147,"by":148,"text":149},"43672799","olavfosse","Does it work on other languages?",{"id":151,"by":152,"text":153},"43673143","fareesh","The non english space in these fields is so far behind in terms of accuracy and reliability, it's crazy",{"value":155,"source":86,"evidence":156},"Evaluation on a RAG benchmark",[157],{"id":120,"by":121,"text":122},[159,166,169],{"value":160,"source":86,"evidence":161},"Is chunking deterministic?",[162],{"id":163,"by":164,"text":165},"43674901","rybosome","Interesting idea - is the chunking deterministic? It would have to be to be useful, but I’m wondering how that interacts with the neural net.",{"value":149,"source":86,"evidence":167},[168],{"id":147,"by":148,"text":149},{"value":170,"source":86,"evidence":171},"Can it index fiction books in a vector DB for RAG?",[172],{"id":173,"by":174,"text":175},"43671855","sushidev","So I could use this to index i.e. a fiction book in a vector db, right? And the semantic chunking will possibly provide better results at query time for rag, did I understand…",[177],{"value":178,"source":86,"evidence":179},"wtpsplit",[180],{"id":181,"by":182,"text":183},"43671297","mathis-l","You might want to take a look at https:\u002F\u002Fgithub.com\u002Fsegment-any-text\u002Fwtpsplit It uses a similar approach but the focus is on sentence\u002Fparagraph segmentation generally and not…",{"comments":44,"answered":32,"key":185},[186,189,192],{"id":187,"text":188},"43671209","Just fixed it.",{"id":190,"text":191},"43672057","No it shouldn't but since it neural net there is a small chance though.",{"id":193,"text":194},"43672039","Yes and yes you are correct!",{"commenters":196,"max_depth":42,"maker_replies":44,"first_comment_mins":197,"lifespan_hours":198},28,2632,66,[200,202,204,206,209,212,215,217,220,224,226,228,229,231,234,235,237,240,243,246,250,252,257,259,261,263,264,267,271,272,277,279,281,283,285],[97,34,34,201,98,34],"other\u002Fneutral\u002Fidea",[181,40,42,203,182,34],"project\u002Fneutral\u002Fidea",[104,40,44,205,105,34],"project\u002Fpositive\u002Fperformance",[207,38,34,201,208,34],"43672718","michaelmarkell",[210,38,34,201,211,34],"43672966","serjester",[213,38,34,201,214,34],"43673019","lmeyerov",[216,38,34,201,208,34],"43673160",[218,38,34,201,219,34],"43673912","DeveloperErrata",[221,38,34,222,223,34],"43674441","commenter\u002Fcritical\u002Fnone","J_Shelby_J",[225,38,34,201,223,34],"43674419",[127,42,40,227,128,34],"project\u002Fcritical\u002Fdocs",[131,42,40,227,132,34],[187,42,34,230,11,40],"maker\u002Fneutral\u002Fdocs",[232,32,34,201,233,34],"43672904","andai",[135,44,40,227,136,34],[93,236,34,201,94,34],6,[238,236,34,201,239,34],"43674017","throwaway7783",[241,236,34,201,242,34],"43678641","searchguy",[116,244,40,245,117,34],7,"project\u002Fcritical\u002Fperformance",[247,33,42,248,249,34],"43672628","maker\u002Fneutral\u002Fidea","dmos62",[108,33,44,251,109,34],"project\u002Fpositive\u002Fidea",[253,254,42,255,256,34],"43671492",9,"project\u002Fneutral\u002Fperformance","oezi",[190,254,34,258,11,40],"maker\u002Fneutral\u002Fperformance",[173,260,44,251,174,34],10,[193,260,34,262,11,40],"maker\u002Fpositive\u002Fidea",[163,18,42,255,164,34],[151,265,34,266,152,34],12,"other\u002Fcritical\u002Fnone",[268,269,42,255,270,34],"43672388",13,"acstorage",[67,269,34,258,11,40],[273,274,34,275,276,34],"43672657",14,"other\u002Fneutral\u002Fnone","cmenge",[120,278,42,255,121,34],15,[280,278,34,258,11,40],"43671053",[282,278,42,258,121,34],"43671287",[89,35,44,284,90,34],"maker\u002Fpositive\u002Fnone",[147,286,42,203,148,34],17,[],[289,294,299,304,309,313],{"id":290,"name":291,"points":292,"category":293},"43690289","Unsure Calculator",930,"dev_tools",{"id":295,"name":296,"points":297,"category":298},"43675248","SignalBloom",292,"finance_crypto",{"id":300,"name":301,"points":302,"category":303},"43667061","memEx",194,"productivity",{"id":305,"name":306,"points":307,"category":308},"43677909","Infocom Unix Z-Machine (pez)",157,"languages",{"id":310,"name":311,"points":312,"category":308},"43682547","protobuf-ts-types",138,{"id":314,"name":315,"points":316,"category":317},"43689178","MCP-Shield",134,"security_privacy",false,"v2",{"id":321,"name":5,"line":6,"ai":318,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":30,"top":18,"kw":322,"maker":11,"domain":10,"tier":19,"attempts":34,"strip":323,"url":9,"mix":324,"praise":85,"objection":113,"sub":21},43652968,[24,25,26,27,28],"03500000001100100013530503030030353",[32,33,34,32],1794036672716]