[{"data":1,"prerenderedAt":288},["ShallowReactive",2],{"project-22035961":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":35,"aspects":36,"strip":42,"post_text":44,"what":45,"said":78,"maker_in_thread":121,"shape":124,"grid":128,"attempts":252,"same_week":253,"maker_shows":186,"maker_hidden":280,"reader_version":281,"card":282},"22035961","HuggingFace","Fast tokenization library for deep-learning NLP pipelines","title","Show HN: HuggingFace – Fast tokenization library for deep-learning NLP pipelines","https:\u002F\u002Fgithub.com\u002Fhuggingface\u002Ftokenizers","github.com","julien_c",1578933626,2020,168,42,354,3028,12,"notable","languages","nlp tokenization library","library",[24,25,26,27,28],"tokenization","nlp","bpe","wordpiece","sentencepiece","curious",0.207,{"positive":32,"neutral":33,"mixed":34,"critical":34},3,10,0,13,{"idea":37,"performance":38,"docs":40,"execution":41},{"positive":34,"critical":34},{"positive":39,"critical":34},1,{"positive":34,"critical":34},{"positive":34,"critical":34},[34,43,34,34,43,34,34,32,32,34,34,34,34,34,34,32,43,32,32,34,34,34,34,34,34,34,34,34,34,32,34,34,34,32,34,34,32,32,32,34],5,null,{"problem":46,"audience":53,"pricing":44,"motivation":44,"time_to_build":44,"stack":62,"openness":75,"stage":76,"team_size":77},{"value":47,"source":48,"evidence":49},"Converting strings into model input tensors for NLP neural nets","maker",[50],{"id":51,"by":11,"text":52},"22036317","TL;DR: Hugging Face, the NLP research company known for its transformers library (DISCLAIMER: I work at Hugging Face), has just released a new open-source library for ultra-fast…",{"value":54,"source":55,"evidence":56},"NLP and deep-learning practitioners","inferred",[57,58],{"id":51,"by":11,"text":52},{"id":59,"by":60,"text":61},"22036746","wyldfire","The README [1] shows a great example: The sentence \"Hello, y'all! How are you ?\" is tokenized into words. Those words are then encoded into integers representative of the words'…",[63,67,71],{"value":64,"source":48,"evidence":65},"Rust",[66],{"id":51,"by":11,"text":52},{"value":68,"source":48,"evidence":69},"Python bindings",[70],{"id":51,"by":11,"text":52},{"value":72,"source":48,"evidence":73},"Node.js bindings",[74],{"id":51,"by":11,"text":52},"open_source","launched","company",{"praise":79,"objections":88,"requests":96,"questions":104,"alternatives":116},[80],{"value":81,"source":82,"evidence":83},"Very fast tokenization, a 6-7x speedup in one quick test","commenter",[84],{"id":85,"by":86,"text":87},"22036750","hnaccy","Great! Just did a quick test and got a 6-7x speedup on tokenization.",[89],{"value":90,"source":82,"evidence":91},"Repos lack tests",[92],{"id":93,"by":94,"text":95},"22042568","m0zg","Question for HuggingFace folks. Your repos do not contain any tests. Why is that? How do you ensure your stuff actually works after you make a change?",[97],{"value":98,"source":82,"evidence":99},"Comparison with spaCy's tokenizer",[100],{"id":101,"by":102,"text":103},"22036751","ZeroCool2u","We use both SpaCy and HuggingFace at work. Is there a comparison of this vs SpaCy's tokenizer[1]? 1. https:\u002F\u002Fspacy.io\u002Fusage\u002Flinguistic-features#tokenization",[105,112],{"value":106,"source":82,"evidence":107},"What does tokenization do?",[108],{"id":109,"by":110,"text":111},"22036657","rsp1984","What does tokenization (of strings, I guess) do?",{"value":113,"source":82,"evidence":114},"Why are repos without tests?",[115],{"id":93,"by":94,"text":95},[117],{"value":118,"source":82,"evidence":119},"spaCy tokenizer",[120],{"id":101,"by":102,"text":103},{"comments":39,"answered":34,"key":122},[123],{"id":51,"text":52},{"commenters":125,"max_depth":126,"maker_replies":39,"first_comment_mins":43,"lifespan_hours":127},34,6,23,[129,131,135,139,142,146,150,152,153,158,162,164,167,170,173,175,178,180,184,189,192,194,196,199,201,204,207,211,215,218,221,224,227,230,232,235,238,241,244,245,246,249],[51,34,34,130,11,39],"project\u002Fneutral\u002Fnone",[132,39,43,133,134,34],"22037178","maker\u002Fpositive\u002Fnone","mark_l_watson",[136,39,34,137,138,34],"22039625","other\u002Fneutral\u002Fnone","dunefox",[140,39,34,137,141,34],"22041645","dzink",[143,144,43,133,145,34],"22037492",2,"screye",[147,144,34,148,149,34],"22038506","commenter\u002Fneutral\u002Fnone","manojlds",[151,144,34,137,145,34],"22038752",[101,32,32,130,102,34],[154,155,32,156,157,34],"22036857",4,"project\u002Fneutral\u002Fidea","LunaSea",[159,155,34,160,161,34],"22037194","other\u002Fneutral\u002Fidea","brockf",[163,155,34,137,157,34],"22037565",[165,155,34,137,166,34],"22037718","visarga",[168,155,34,137,169,34],"22037804","PeterisP",[171,155,34,137,172,34],"22043043","Erlich_Bachman",[174,155,34,137,169,34],"22045532",[176,43,32,156,177,34],"22037238","useful",[85,126,43,179,86,34],"project\u002Fpositive\u002Fperformance",[181,126,32,182,183,34],"22039072","maker\u002Fneutral\u002Fperformance","clmnt",[185,186,32,187,188,34],"22037268",7,"project\u002Fneutral\u002Fdocs","orestis",[190,186,34,160,191,34],"22037549","ogrisel",[193,186,34,160,191,34],"22037579",[195,186,34,137,188,34],"22037960",[197,186,34,137,198,34],"22039581","nestorD",[200,186,34,148,188,34],"22041219",[202,186,34,148,203,34],"22041841","oddnearfuture",[205,186,34,160,206,34],"22038795","rococode",[208,186,34,209,210,34],"22041615","other\u002Fpositive\u002Fidea","samcodes",[212,213,34,137,214,34],"22037114",8,"echelon",[216,213,34,137,217,34],"22037221","mraison",[219,213,34,148,220,34],"22037873","brokensegue",[222,213,32,156,223,34],"22037246","starpilot",[225,213,34,137,226,34],"22037491","Isn0gud",[228,213,34,137,229,34],"22037500","crawdog",[231,213,34,137,166,34],"22037785",[93,233,32,234,94,34],9,"maker\u002Fneutral\u002Fexecution",[236,33,34,137,237,34],"22036033","virtuous_signal",[239,33,34,137,240,34],"22040803","rjmorris",[242,243,32,187,149,34],"22038515",11,[109,18,32,187,110,34],[59,18,32,187,60,34],[247,35,34,137,248,34],"22038219","tarr11",[250,35,34,137,251,34],"22038537","itronitron",[],[254,259,264,269,273,276],{"id":255,"name":256,"points":257,"category":258},"22045053","Diary Email",515,"productivity",{"id":260,"name":261,"points":262,"category":263},"22027177","Podnami",340,"consumer_social",{"id":265,"name":266,"points":267,"category":268},"22044198","Cantools",220,"dev_tools",{"id":270,"name":271,"points":272,"category":258},"21998632","Ryeboard",185,{"id":274,"name":275,"points":272,"category":268},"22044368","Gter",{"id":277,"name":278,"points":279,"category":268},"22005495","Lucidity",184,false,"v2",{"id":283,"name":5,"line":6,"ai":280,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":284,"top":18,"kw":285,"maker":11,"domain":10,"tier":19,"attempts":34,"strip":286,"url":9,"mix":287,"praise":81,"objection":90,"sub":21},22035961,0.21,[24,25,26,27,28],"0500500330000003533000000000030003003330",[32,33,34,34],1793991001919]