[{"data":1,"prerenderedAt":160},["ShallowReactive",2],{"project-39279998":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":9,"maker":10,"posted_at":11,"year":12,"points":13,"comments":14,"rank_in_year":15,"shown_in_year":16,"top":17,"tier":18,"category":19,"subcategory":20,"format":21,"keywords":22,"reception":27,"sentiment":9,"stance_mix":28,"scored_comments":32,"aspects":33,"strip":36,"post_text":38,"what":39,"said":62,"maker_in_thread":90,"shape":101,"grid":105,"attempts":122,"same_week":123,"maker_shows":102,"maker_hidden":152,"reader_version":153,"card":154},"39279998","Unify","Dynamic LLM Benchmarks and SSO for Multi-Vendor Deployment","title","Show HN: Unify – Dynamic LLM Benchmarks and SSO for Multi-Vendor Deployment",null,"danlenton",1707250884,2024,91,8,900,3515,26,"notable","ai","llm api and benchmarking","web_app",[23,24,25,26],"llm benchmarks","llm endpoints","unified api","model hub","quiet",{"positive":29,"neutral":30,"mixed":31,"critical":31},3,1,0,4,{"pricing":34,"design":35},{"positive":31,"critical":31},{"positive":30,"critical":31},[29,31,37,31,37,31,37,31],5,"Hey HN! I’m the founder of Unify, and we’ve just released our Model Hub, which provides a collection of LLM endpoints with live runtime benchmarks all plotted across time: https:\u002F\u002Funify.ai\u002Fhub\n\nA key finding is that static tabular runtime benchmarks for LLMs simply do not work. It’s necessary to take a time-series perspective, and plot the variations through time.\n\nWe currently have 21 models provided by: Anyscale, Perplexity AI, Replicate, Together AI, OctoAI,  Mistral AI and OpenAI, with more on the roadmap.\n\nWe test across different regions (Asia, US, Europe), with varied concurrency and se",{"problem":40,"audience":46,"pricing":51,"motivation":55,"time_to_build":9,"stack":59,"openness":60,"stage":61,"team_size":60},{"value":41,"source":42,"evidence":43},"Static LLM runtime benchmarks miss variation; comparing multi-vendor endpoints and managing several accounts is hard","maker",[44],{"id":45},"post",{"value":47,"source":48,"evidence":49},"Developers deploying LLMs across multiple providers","inferred",[50],{"id":45},{"value":52,"source":42,"evidence":53},"Free promo credits of $5 per week via HACKERNEWS code; otherwise unknown",[54],{"id":45},{"value":56,"source":42,"evidence":57},"Maker found static benchmarks inadequate and wanted time-series comparison across providers",[58],{"id":45},[],"unknown","launched",{"praise":63,"objections":79,"requests":80,"questions":81,"alternatives":89},[64,72],{"value":65,"source":66,"evidence":67},"Cross-region stats are helpful","commenter",[68],{"id":69,"by":70,"text":71},"39288733","fudoshin2596","The cross region stats are actually quite helpful, this is cool!",{"value":73,"source":66,"evidence":74},"Looks awesome",[75],{"id":76,"by":77,"text":78},"39289185","nchudleigh","This looks awesome, going to be exploring today.",[],[],[82],{"value":83,"source":66,"evidence":84},"Why some providers are much cheaper for the same model",[85],{"id":86,"by":87,"text":88},"39294616","araghuvanshi","The cost comparisons for the same model are interesting. I'm curious about why certain providers are a lot cheaper than others - for example, mistral-8-7b on OctoAI costs $0.2\u002F1m…",[],{"comments":32,"answered":30,"key":91},[92,95,98],{"id":93,"text":94},"39300075","I think different providers are just trying to provide value at different areas in the market. OctoAI are consistently the most cost effective, but typically not as fast, while…",{"id":96,"text":97},"39290033","Glad it's useful! Let us know if there are any other features you'd like to see :)",{"id":99,"text":100},"39290024","Great! Let us know what you think good and bad. We'll be iterating very quickly.",{"commenters":37,"max_depth":102,"maker_replies":32,"first_comment_mins":103,"lifespan_hours":104},2,1067,37,[106,108,109,111,113,115,116,120],[86,31,29,107,87,31],"project\u002Fneutral\u002Fpricing",[93,31,31,107,10,30],[69,30,37,110,70,31],"project\u002Fpositive\u002Fdesign",[96,30,31,112,10,30],"maker\u002Fneutral\u002Fnone",[76,102,37,114,77,31],"project\u002Fpositive\u002Fnone",[99,102,31,112,10,30],[117,29,37,118,119,31],"39289523","maker\u002Fpositive\u002Fnone","jfong",[121,29,31,118,10,30],"39290018",[],[124,129,134,138,143,148],{"id":125,"name":126,"points":127,"category":128},"39288039","One Minute Focus",741,"consumer_social",{"id":130,"name":131,"points":132,"category":133},"39263854","Atopile",595,"dev_tools",{"id":135,"name":136,"points":137,"category":19},"39261486","Natural-SQL-7B",362,{"id":139,"name":140,"points":141,"category":142},"39214743","filippo.io\u002Fmlkem768",315,"languages",{"id":144,"name":145,"points":146,"category":147},"39232572","Minesweeper Twist",293,"games",{"id":149,"name":150,"points":151,"category":128},"39260760","Random Hacker News",267,false,"v2",{"id":155,"name":5,"line":6,"ai":152,"cat":19,"year":12,"points":13,"comments":14,"rec":27,"s":9,"top":17,"kw":156,"maker":10,"domain":157,"tier":18,"attempts":31,"strip":158,"url":-1,"mix":159,"praise":65,"objection":-1,"sub":20},39279998,[23,24,25,26],"","30505050",[29,30,31,31],1794036649313]