[{"data":1,"prerenderedAt":217},["ShallowReactive",2],{"project-36044755":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":14,"rank_in_year":15,"shown_in_year":16,"top":17,"tier":18,"category":19,"subcategory":20,"format":21,"keywords":22,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":33,"aspects":34,"strip":39,"post_text":41,"what":42,"said":85,"maker_in_thread":146,"shape":153,"grid":157,"attempts":181,"same_week":182,"maker_shows":40,"maker_hidden":209,"reader_version":210,"card":211},"36044755","Willow Inference Server","Optimized ASR\u002FTTS\u002FLLM for Willow\u002FWebRTC\u002FREST","title","Show HN: Willow Inference Server: Optimized ASR\u002FTTS\u002FLLM for Willow\u002FWebRTC\u002FREST","https:\u002F\u002Fgithub.com\u002Ftoverainc\u002Fwillow-inference-server","github.com","kkielhofner",1684851740,2023,13,2112,3109,68,"shown","ai","speech recognition inference server","api",[23,24,25,26,27],"speech recognition","whisper","self-hosted voice assistant","text to speech","cuda inference","curious",0.319,{"positive":31,"neutral":31,"mixed":32,"critical":32},3,0,6,{"idea":35,"performance":37},{"positive":36,"critical":32},2,{"positive":38,"critical":32},1,[40,32,32,32,31,32,31,32,40,32,40,32,31],5,"Hey HN!\n\nWillow Inference Server (WIS) is a focused and highly optimized language inference server implementation. Our goal is to \"automagically\" enable performant, cost-effective self-hosting of released state of the art\u002Fbest of breed models to enable speech and language tasks:\n\nPrimarily targeting CUDA (works on CPU too) with support for low-end (cheap) devices such as the Tesla P4, GTX 1060, and up. Don't worry - it screams on an RTX 4090 too! (See benchmarks on Github).\n\nMemory optimized - all three default Whisper (base, medium, large-v2) models loaded simultaneously with TTS support insi",{"problem":43,"audience":49,"pricing":60,"motivation":61,"time_to_build":60,"stack":66,"openness":83,"stage":84,"team_size":83},{"value":44,"source":45,"evidence":46},"Self-hosting fast, cost-effective speech recognition, TTS and LLM inference for voice assistants","maker",[47],{"id":48},"post",{"value":50,"source":51,"evidence":52},"Self-hosters building private voice assistants, such as Willow users","inferred",[53,57],{"id":54,"by":55,"text":56},"36048226","kunwon1","Thanks for this, I've been looking forward to it. I used Amazon Echo devices during their first 6 months of public availability before I got sufficiently creeped out to pull the…",{"id":58,"by":11,"text":59},"36045657","Thanks! We believe we have a practical, viable approach to provide an Alexa equivalent (or better) experience with self-hosted privacy and control. Not to mention all of the…",null,{"value":62,"source":45,"evidence":63},"Enable private, self-hosted Alexa-like voice assistance without a black-box cloud",[64,65],{"id":48},{"id":58,"by":11,"text":59},[67,71,75,79],{"value":68,"source":45,"evidence":69},"CTranslate2",[70],{"id":48},{"value":72,"source":45,"evidence":73},"AutoGPTQ",[74],{"id":48},{"value":76,"source":45,"evidence":77},"Whisper",[78],{"id":48},{"value":80,"source":45,"evidence":81},"Vicuna",[82],{"id":48},"unknown","beta",{"praise":86,"objections":105,"requests":110,"questions":125,"alternatives":136},[87,97],{"value":88,"source":89,"evidence":90},"Fast response times on low-cost GPUs","commenter",[91,94],{"id":92,"by":11,"text":93},"36045282","Thanks! Yes, \"realtime multiple\" is audio\u002Fspeech length divided by actual inference time. You got it! The demo video is showing the slowest response times because it is using the…",{"id":95,"by":11,"text":96},"36056949","Thank you! Unfortunately the oldest GPUs we support are Pascal and your GTX 960 is Maxwell. We have this cutoff for GPU hardware support for two reasons: 1) Nvidia doesn't…",{"value":98,"source":89,"evidence":99},"Privacy-respecting alternative to Amazon Echo",[100,101],{"id":54,"by":55,"text":56},{"id":102,"by":103,"text":104},"36045427","morehamish","Brilliant. Willow looks like a game-changer to me, thanks a million!",[106],{"value":107,"source":45,"evidence":108},"Older GPUs such as Maxwell are unsupported",[109],{"id":95,"by":11,"text":96},[111,118],{"value":112,"source":89,"evidence":113},"Support for older GPUs like the GTX 960",[114],{"id":115,"by":116,"text":117},"36055191","unverbraucht","It looks amazing, I'll definitely try it out! Quick question: would a GTX 960 also work for inference? I happen to have one lying around and could whip up a system with it.…",{"value":119,"source":89,"evidence":120},"A Willow Application Server",[121,122],{"id":54,"by":55,"text":56},{"id":123,"by":11,"text":124},"36048412","Thanks! One note - if you're going to self-host WIS (YOU SHOULD) I suggest making good use of the more powerful hardware otherwise and putting HA on it. HA on Raspberry Pi (while…",[126,132],{"value":127,"source":89,"evidence":128},"Whether realtime multiple means speed relative to real time",[129],{"id":130,"by":103,"text":131},"36045165","Looks fantastic! A question re. benchmarks: the \"realtime multiple\" means \"how many times faster the model is processing the audio compared to real time\" here? (I.e. the opposite…",{"value":133,"source":89,"evidence":134},"Whether larger beam sizes improve quality",[135],{"id":130,"by":103,"text":131},[137,141],{"value":138,"source":89,"evidence":139},"Amazon Echo",[140],{"id":54,"by":55,"text":56},{"value":142,"source":89,"evidence":143},"Home Assistant on Raspberry Pi",[144,145],{"id":54,"by":55,"text":56},{"id":123,"by":11,"text":124},{"comments":33,"answered":31,"key":147},[148,149,152],{"id":123,"text":124},{"id":150,"text":151},"36049240","WIS has been tested with WSL (if you're running Windows) and supports anything from a GTX 1060 3GB up to H100 so you can certainly throw it on your gaming desktop to start.",{"id":92,"text":93},{"commenters":154,"max_depth":33,"maker_replies":33,"first_comment_mins":155,"lifespan_hours":156},4,30,41,[158,160,162,165,167,168,170,172,174,175,176,177,178],[54,32,40,159,55,32],"project\u002Fpositive\u002Fidea",[123,32,32,161,11,38],"maker\u002Fneutral\u002Fperformance",[163,32,32,164,55,32],"36048494","commenter\u002Fneutral\u002Fnone",[150,32,32,166,11,38],"project\u002Fneutral\u002Fperformance",[130,38,31,166,103,32],[92,38,32,169,11,38],"project\u002Fpositive\u002Fperformance",[171,38,31,161,103,32],"36045312",[173,38,32,169,11,38],"36045396",[102,38,40,159,103,32],[58,38,32,159,11,38],[115,36,40,169,116,32],[95,36,32,166,11,38],[179,36,31,180,116,32],"36067947","maker\u002Fneutral\u002Fnone",[],[183,188,192,197,201,205],{"id":184,"name":185,"points":186,"category":187},"36009774","YouTube Full Text Search",540,"dev_tools",{"id":189,"name":190,"points":191,"category":19},"36064639","Pixian",383,{"id":193,"name":194,"points":195,"category":196},"36047143","Open sourcing Harmonic",305,"consumer_social",{"id":198,"name":199,"points":200,"category":187},"36020717","Trogon",248,{"id":202,"name":203,"points":204,"category":187},"36020001","SpaceBadgers",199,{"id":206,"name":207,"points":208,"category":19},"36037419","Blotter",166,false,"v2",{"id":212,"name":5,"line":6,"ai":209,"cat":19,"year":13,"points":14,"comments":14,"rec":28,"s":213,"top":17,"kw":214,"maker":11,"domain":10,"tier":18,"attempts":32,"strip":215,"url":9,"mix":216,"praise":88,"objection":107,"sub":20},36044755,0.32,[23,24,25,26,27],"5000303050503",[31,31,32,32],1793991066348]