[{"data":1,"prerenderedAt":379},["ShallowReactive",2],{"project-41743327":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":35,"aspects":36,"strip":43,"post_text":46,"what":47,"said":98,"maker_in_thread":169,"shape":181,"grid":186,"attempts":338,"same_week":343,"maker_shows":38,"maker_hidden":371,"reader_version":372,"card":373},"41743327","LiveKit Agents","Open source framework OpenAI uses for Advanced Voice","title","Show HN: Open source framework OpenAI uses for Advanced Voice","https:\u002F\u002Fgithub.com\u002Flivekit\u002Fagents","github.com","russ",1728061264,2024,266,61,271,3515,8,"legend","ai","voice agent framework","library",[24,25,26,27,28],"voice agent","webrtc","realtime api","gpt-4o","speech to speech","curious",0.035,{"positive":32,"neutral":33,"mixed":34,"critical":32},4,16,2,26,{"idea":37,"pricing":39,"execution":40,"performance":42},{"positive":38,"critical":34},1,{"positive":38,"critical":38},{"positive":41,"critical":41},0,{"positive":41,"critical":38},[44,38,41,41,41,38,41,38,41,41,41,41,41,45,45,45,45,45,45,38,44,41,44,41,41,41,45,41,41,41,41,41,41,45,45,41,45,41,45,44],5,3,"Hey HN, we've been working with OpenAI for the past few months on the new Realtime API.\n\nThe goal is to give everyone access to the same stack that underpins Advanced Voice in the ChatGPT app.\n\nUnder the hood it works like this: - A user's speech is captured by a LiveKit client SDK in the ChatGPT app - Their speech is streamed using WebRTC to OpenAI’s voice agent - The agent relays the speech prompt over websocket to GPT-4o - GPT-4o runs inference and streams speech packets (over websocket) back to the agent - The agent relays generated speech using WebRTC back to the user’s device\n\nThe Realti",{"problem":48,"audience":54,"pricing":59,"motivation":69,"time_to_build":73,"stack":74,"openness":95,"stage":96,"team_size":97},{"value":49,"source":50,"evidence":51},"Bridges WebRTC audio from users to the OpenAI Realtime websocket API for reliable voice conversations over lossy networks","maker",[52],{"id":53},"post",{"value":55,"source":56,"evidence":57},"Developers building realtime voice AI applications","inferred",[58],{"id":53},{"value":60,"source":50,"evidence":61},"Usage costs roughly $0.15 per minute of conversation; LiveKit Cloud charges about $0.0005 per participant minute",[62,65],{"id":63,"by":11,"text":64},"41746982","50% human speaking at $0.06\u002Fminute of tokens 50% AI speaking at $0.24\u002Fminute of tokens we (LiveKit Cloud) charge ~$0.0005\u002Fminute for each participant (in this case there would be…",{"id":66,"by":67,"text":68},"41746958","shayps","It shakes out to around $0.15 per minute for an average conversation. If history is a guide though, this will get a lot cheaper pretty quickly.",{"value":70,"source":50,"evidence":71},"Give everyone access to the same stack that powers Advanced Voice in ChatGPT, and avoid websocket packet loss problems",[72],{"id":53},null,[75,79,83,87],{"value":76,"source":50,"evidence":77},"WebRTC",[78],{"id":53},{"value":80,"source":50,"evidence":81},"WebSocket",[82],{"id":53},{"value":84,"source":50,"evidence":85},"GPT-4o via OpenAI Realtime API",[86],{"id":53},{"value":88,"source":89,"evidence":90},"Silero VAD","commenter",[91],{"id":92,"by":93,"text":94},"41751872","davidz","currently we are using silero VAD to detect speech: https:\u002F\u002Fgithub.com\u002Flivekit\u002Fagents\u002Fblob\u002Fmain\u002Flivekit-plugins\u002Flivekit-plugins-silero\u002Flivekit\u002Fplugins\u002Fsilero\u002Fvad.py it works well…","open_source","launched","company",{"praise":99,"objections":111,"requests":129,"questions":141,"alternatives":155},[100],{"value":101,"source":89,"evidence":102},"Useful for voice AI and the ChatGPT-style voice experience",[103,107],{"id":104,"by":105,"text":106},"41748560","spuz","Is there anyone besides OpenAI working on a speech to speech model? I find it incredibly useful and it's the sole reason that I pay for their service but I do find it very…",{"id":108,"by":109,"text":110},"41745747","FanaHOVA","Olivier, Michelle, and Romain gave you guys a shoutout like 3 times in our DevDay recap podcast if you need more testimonial quotes :) https:\u002F\u002Fwww.latent.space\u002Fp\u002Fdevday-2024",[112,122],{"value":113,"source":89,"evidence":114},"Audio is sent to the model continuously, so costs are high until turn detection is added",[115,119],{"id":116,"by":117,"text":118},"41746066","pj_mukh","Super cool! Didn't realize OpenAI is just using LiveKit. Does the pricing breakdown to be the same as having a OpenAI Advanced Voice socket open the whole time? It's like $9\u002Fhr!…",{"id":120,"by":93,"text":121},"41746382","Currently it does: all audio is sent to the model. However, we are working on turn detection within the framework, so you won't have to send silence to the model when the user…",{"value":123,"source":89,"evidence":124},"Users report choppy audio in ChatGPT voice mode, possibly due to load",[125],{"id":126,"by":127,"text":128},"41752165","gastonmorixe","80% of the times I’m experiencing choppy audio on my iPhone 15 Pro Max (18.1b) on Voice Mode (Standard and Advanced). My internet connection is FTTH and WiFi 7 state of the art…",[130,137],{"value":131,"source":89,"evidence":132},"Voice activity detection so silence is not sent to the model",[133,134],{"id":120,"by":93,"text":121},{"id":135,"by":117,"text":136},"41752247","Can I currently put a VAD module in the pipeline and only send audio when there is an active conversation? Feel like just that would solve the problem?",{"value":138,"source":89,"evidence":139},"Better end-of-turn detection",[140],{"id":92,"by":93,"text":94},[142,149],{"value":143,"source":89,"evidence":144},"Whether WebRTC is needed for backend-only applications",[145],{"id":146,"by":147,"text":148},"41748157","lolpanda","so the WebRTC helps with the unreliable network between the mobile clients and the server side. if the application is backend only, would it make sense to use WebRTC or should I…",{"value":150,"source":89,"evidence":151},"Whether the Realtime API uses the same model as Advanced Voice",[152],{"id":153,"by":127,"text":154},"41745617","Nice they have many partners on this. I see Azure as well. There is a common consensus that the new Realtime API is not actually using the same Advanced Voice model \u002F engine - or…",[156,163],{"value":157,"source":89,"evidence":158},"Moshi by Kyutai",[159],{"id":160,"by":161,"text":162},"41748593","Ey7NFZ3P0nzAe","Yes. Kyutai released an opened model called moshi : https:\u002F\u002Fgithub.com\u002Fkyutai-labs\u002Fmoshi There's also llama-omni and a few others. None of them are even close to 4o from an LLM…",{"value":164,"source":89,"evidence":165},"Ultravox by Fixie",[166],{"id":167,"by":11,"text":168},"41750286","There’s Ultravox as well (from one of the creators of WebRTC): https:\u002F\u002Fgithub.com\u002Ffixie-ai\u002Fultravox Their model builds a speech-to-speech layer into Llama. Last I checked they…",{"comments":170,"answered":45,"key":171},11,[172,175,178],{"id":173,"text":174},"41750340","Which components feel ad hoc? In most real applications, the agent has additional logic (function calling, RAG, etc) than simply relaying a stream to the model server. In those…",{"id":176,"text":177},"41752467","You really don’t need to know about WebRTC at all when you use LiveKit. That’s largely thanks to the SDKs abstracting away all the complexity. Having good SDKs that work across…",{"id":179,"text":180},"41753341","Heh, actually I'm pretty sure I've come across your X profile before. :) You're definitely in a small minority of folks with a deep(er) understanding of WebRTC.",{"commenters":182,"max_depth":183,"maker_replies":170,"first_comment_mins":184,"lifespan_hours":185},35,7,249,62,[187,191,195,199,202,204,207,209,212,215,218,222,226,228,230,231,233,234,236,238,240,241,245,248,252,253,254,257,260,262,264,267,269,271,273,274,275,278,281,282,287,289,291,294,297,299,302,304,306,309,312,314,315,317,319,321,326,328,331,332,336],[188,41,44,189,190,41],"41749490","project\u002Fpositive\u002Fidea","racecar789",[192,41,38,193,194,41],"41749520","project\u002Fcritical\u002Fidea","TZubiri",[196,41,41,197,198,41],"41749603","other\u002Fneutral\u002Fnone","ensignavenger",[200,41,41,197,201,41],"41750062","daveguy",[203,41,41,197,198,41],"41751249",[205,41,41,197,206,41],"41750088","andrew_eu",[208,41,41,197,198,41],"41751263",[210,41,41,197,211,41],"41763566","android521",[213,41,38,193,214,41],"41751960","beeboobaa3",[216,41,41,197,217,41],"41753499","fosheezy",[219,38,38,220,221,41],"41749131","project\u002Fcritical\u002Fpricing","throw14082020",[223,38,45,224,225,41],"41750106","project\u002Fmixed\u002Fexecution","fidotron",[173,38,41,227,11,38],"commenter\u002Fneutral\u002Fnone",[229,38,41,227,225,41],"41751588",[176,38,41,227,11,38],[232,38,41,227,225,41],"41752953",[179,38,41,227,11,38],[235,38,41,227,11,38],"41751599",[237,38,41,227,221,41],"41754039",[116,34,45,239,117,41],"project\u002Fneutral\u002Fpricing",[120,34,45,239,93,41],[242,34,45,243,244,41],"41747890","project\u002Fneutral\u002Fexecution","rukuu001",[246,34,45,243,247,41],"41748273","balloob",[249,34,45,250,251,41],"41750188","project\u002Fneutral\u002Fidea","ValentinA23",[92,34,45,243,93,41],[135,34,45,250,117,41],[255,34,45,239,256,41],"41747550","npace12",[258,45,45,250,259,41],"41746906","solarkraft",[261,45,41,243,11,38],"41747366",[126,45,38,263,127,41],"project\u002Fcritical\u002Fperformance",[265,45,41,266,11,38],"41752432","project\u002Fneutral\u002Fperformance",[108,32,44,268,109,41],"maker\u002Fpositive\u002Fnone",[270,32,41,268,11,38],"41746424",[272,32,44,268,67,41],"41746658",[104,44,41,197,105,41],[160,44,41,197,161,41],[276,44,41,197,277,41],"41749249","0x1ceb00da",[279,44,41,197,280,41],"41749652","mbrock",[167,44,41,197,11,38],[283,284,45,285,286,41],"41745615",6,"project\u002Fneutral\u002Fnone","mycall",[288,284,45,285,93,41],"41746393",[290,183,41,197,277,41],"41749245",[292,183,41,227,293,41],"41749286","meiraleal",[295,183,41,227,296,41],"41749366","hshshshsvsv",[298,183,41,227,194,41],"41749523",[300,183,41,227,301,41],"41749586","tempaccount420",[303,183,41,227,194,41],"41755119",[305,183,41,227,280,41],"41749682",[307,183,41,227,308,41],"41752274","nialse",[310,183,41,227,311,41],"41750044","ada1981",[313,183,45,285,280,41],"41749638",[153,18,45,285,127,41],[316,18,41,285,11,38],"41746421",[318,18,41,227,127,41],"41752088",[146,320,45,250,147,41],9,[322,323,45,324,325,41],"41746198",10,"project\u002Fmixed\u002Fpricing","willsmith72",[327,323,41,239,11,38],"41746455",[329,323,45,239,330,41],"41746925","wordpad25",[66,323,45,239,67,41],[333,323,44,334,335,41],"41747367","project\u002Fpositive\u002Fpricing","cdolan",[63,323,41,337,11,38],"maker\u002Fneutral\u002Fpricing",[339,342],{"id":340,"t":341,"points":34,"comments":41},"39060924",1705697317,{"id":4,"t":12,"points":14,"comments":15},[344,349,354,358,362,366],{"id":345,"name":346,"points":347,"category":348},"41742278","One",506,"dev_tools",{"id":350,"name":351,"points":352,"category":353},"41776878","Kotlin Money",427,"languages",{"id":355,"name":356,"points":357,"category":348},"41780297","Player.style",392,{"id":359,"name":360,"points":361,"category":348},"41740568","Chebyshev approximation calculator",278,{"id":363,"name":364,"points":365,"category":348},"41764635","ARIA DevTools",272,{"id":367,"name":368,"points":369,"category":370},"41780255","URL expander",213,"other",false,"v2",{"id":374,"name":5,"line":6,"ai":371,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":375,"top":18,"kw":376,"maker":11,"domain":10,"tier":19,"attempts":34,"strip":377,"url":9,"mix":378,"praise":101,"objection":113,"sub":21},41743327,0.04,[24,25,26,27,28],"5100010100000333333150500030000003303035",[32,33,34,32],1794036662309]