[{"data":1,"prerenderedAt":183},["ShallowReactive",2],{"project-45703196":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":35,"aspects":36,"strip":40,"post_text":42,"what":43,"said":71,"maker_in_thread":125,"shape":127,"grid":130,"attempts":145,"same_week":146,"maker_shows":32,"maker_hidden":175,"reader_version":176,"card":177},"45703196","Chonky","a neural text semantic chunking goes multilingual","title","Show HN: Chonky – a neural text semantic chunking goes multilingual","https:\u002F\u002Fhuggingface.co\u002Fmirth\u002Fchonky_mmbert_small_multilingual_1","huggingface.co","hessdalenlight",1761393704,2025,43,6,1370,3837,36,"shown","ai","text chunking model","library",[24,25,26,27,28],"semantic chunking","multilingual nlp","rag chunking","mmbert","text splitting","debated",-0.067,{"positive":32,"neutral":33,"mixed":34,"critical":32},2,1,0,5,{"idea":37,"docs":38,"performance":39},{"positive":33,"critical":33},{"positive":33,"critical":34},{"positive":34,"critical":33},[35,34,41,35,33,33],3,"TLDR: I’m expanding the family of text-splitting Chonky models with new multilingual model.\n\nYou can learn more about this neural approach in a previous post: https:\u002F\u002Fnews.ycombinator.com\u002Fitem?id=43652968\n\nSince the release of the first distilbert-based model I’ve released two more models based on a ModernBERT. All these models were pre-trained and fine-tuned primary on English texts.\n\nBut recently mmBERT(https:\u002F\u002Fhuggingface.co\u002Fblog\u002Fmmbert) has been released. This model pre-trained on massive dataset that contains 1833 languages. So I had an idea of fine-tuning a new multilingual Chonky model.",{"problem":44,"audience":50,"pricing":50,"motivation":51,"time_to_build":50,"stack":55,"openness":68,"stage":69,"team_size":70},{"value":45,"source":46,"evidence":47},"Splitting text into semantic chunks, especially for real-world OCR, transcripts and notes","maker",[48],{"id":49},"post",null,{"value":52,"source":46,"evidence":53},"Extend the Chonky text-splitting family to multilingual text using mmBERT",[54],{"id":49},[56,60,64],{"value":57,"source":46,"evidence":58},"ModernBERT",[59],{"id":49},{"value":61,"source":46,"evidence":62},"mmBERT",[63],{"id":49},{"value":65,"source":46,"evidence":66},"DistilBERT",[67],{"id":49},"open_source","launched","unknown",{"praise":72,"objections":88,"requests":103,"questions":108,"alternatives":116},[73,81],{"value":74,"source":75,"evidence":76},"Interesting approach; useful for unstructured text from speech-to-text","commenter",[77],{"id":78,"by":79,"text":80},"45709447","kamranjon","This is interesting! I once trained a t5 model by removing newlines from Wikipedia text and it worked surprisingly well \u002F at the time the context length was the biggest issue.…",{"value":82,"source":75,"evidence":83},"Documentation and evaluation comparison against rules-based chunking are useful",[84],{"id":85,"by":86,"text":87},"45736266","jimmySixDOF","you can read the labels this (-y) uses modernBERT and even has an eval comparison to the (-ie) in it's GitHub so you can see the improvement as tested -- although if you want to…",[89,96],{"value":90,"source":75,"evidence":91},"Example output looks not useful",[92],{"id":93,"by":94,"text":95},"45710798","TZubiri","That example looks terribly useless. Maybe there's an actually useful application you had in mind? I don't know say Chonk(\"Hey I forgot my password, this is Tom from X Company\")…",{"value":97,"source":75,"evidence":98},"Splits related texts and keeps unrelated ones together in the demo",[99],{"id":100,"by":101,"text":102},"45723029","freakynit","This is absolutely useless. Tried a few examples yesterday using hf demo. Fcking retarded af. It literally splitted the text in-between of related texts while at the same time…",[104],{"value":105,"source":75,"evidence":106},"Show a practical application for the splitting",[107],{"id":93,"by":94,"text":95},[109],{"value":110,"source":75,"evidence":111},"How does it compare to Chonkie's semantic chunking approach",[112],{"id":113,"by":114,"text":115},"45710091","CjHuber","Took me a minute to realize this is not about Chonkie. I would be interested in how this compares to the other's semantic chunking approach",[117,121],{"value":118,"source":75,"evidence":119},"Chonkie semantic chunking",[120],{"id":113,"by":114,"text":115},{"value":122,"source":75,"evidence":123},"Rules-based chunking",[124],{"id":85,"by":86,"text":87},{"comments":34,"answered":34,"key":126},[],{"commenters":15,"max_depth":32,"maker_replies":34,"first_comment_mins":128,"lifespan_hours":129},1067,78,[131,133,137,139,141,143],[78,34,35,132,79,34],"project\u002Fpositive\u002Fidea",[134,34,34,135,136,34],"45713136","other\u002Fneutral\u002Fnone","nvdnadj92",[113,33,41,138,114,34],"project\u002Fneutral\u002Fnone",[85,33,35,140,86,34],"project\u002Fpositive\u002Fdocs",[93,32,33,142,94,34],"project\u002Fcritical\u002Fidea",[100,41,33,144,101,34],"project\u002Fcritical\u002Fperformance",[],[147,152,157,162,167,171],{"id":148,"name":149,"points":150,"category":151},"45706792","oxdraw",266,"dev_tools",{"id":153,"name":154,"points":155,"category":156},"45715055","MyraOS",247,"infrastructure",{"id":158,"name":159,"points":160,"category":161},"45731366","Bash Screensavers",246,"creative",{"id":163,"name":164,"points":165,"category":166},"45684689","Deadstack",188,"productivity",{"id":168,"name":169,"points":170,"category":161},"45721352","ISS in Real Time",165,{"id":172,"name":173,"points":174,"category":151},"45717724","vite-plugin-use-golang",158,false,"v2",{"id":178,"name":5,"line":6,"ai":175,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":179,"top":18,"kw":180,"maker":11,"domain":10,"tier":19,"attempts":34,"strip":181,"url":9,"mix":182,"praise":74,"objection":90,"sub":21},45703196,-0.07,[24,25,26,27,28],"503511",[32,33,34,32],1794036684706]