[{"data":1,"prerenderedAt":164},["ShallowReactive",2],{"project-44648120":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":31,"aspects":33,"strip":38,"post_text":29,"what":39,"said":63,"maker_in_thread":112,"shape":115,"grid":116,"attempts":127,"same_week":128,"maker_shows":31,"maker_hidden":157,"reader_version":158,"card":159},"44648120","superglue","We let agents use APIs to find out if they can actually...do things?","title","Show HN: We let agents use APIs to find out if they can actually...do things?","https:\u002F\u002Fsuperglue.ai\u002Fapi-ranking\u002F","superglue.ai","adinagoerres",1753197455,2025,11,5,2710,3837,71,"shown","ai","llm benchmark","other",[24,25,26,27],"llm api benchmark","agent api integration","api ranking","llm evaluation","quiet",null,{"positive":31,"neutral":32,"mixed":32,"critical":32},3,0,{"performance":34,"idea":36},{"positive":35,"critical":32},1,{"positive":37,"critical":32},2,[15,32,15,32,15],{"problem":40,"audience":47,"pricing":29,"motivation":51,"time_to_build":29,"stack":59,"openness":60,"stage":61,"team_size":62},{"value":41,"source":42,"evidence":43},"LLMs often fail when writing code to call APIs","maker",[44],{"id":45,"by":11,"text":46},"44648121","Hi HN! Adina here from superglue. Today I’d like to share a new benchmark we’ve just open sourced: an Agent-API Benchmark, in which we test how well LLMs handle APIs. tl;dr: LLMs…",{"value":48,"source":42,"evidence":49},"Teams building agents that need reliable API access",[50],{"id":45,"by":11,"text":46},{"value":52,"source":42,"evidence":53},"Automating API integrations is their own problem, and existing LLM benchmarks seem disconnected from it",[54,55],{"id":45,"by":11,"text":46},{"id":56,"by":57,"text":58},"44648597","sfaist","The reason we think this would be interesting to share here is that these llm benchmarks seem increasingly disconnected from reality. idc if the llm can solve a PhD math question…",[],"open_source","launched","unknown",{"praise":64,"objections":80,"requests":85,"questions":102,"alternatives":111},[65,73],{"value":66,"source":67,"evidence":68},"Benchmark results are useful and surprisingly good for a first version","commenter",[69],{"id":70,"by":71,"text":72},"44650595","nimar","v interesting benchmark, looking forward to see it evolve over time. actually surprisingly good results already. maybe add a couple harder APIs (or more complex queries) as well…",{"value":74,"source":67,"evidence":75},"Benchmarks like this are needed, since API performance varies a lot for agents",[76],{"id":77,"by":78,"text":79},"44649263","kyleledbetter","Super cool that you all published these benchmarks. We've seen similar, where some APIs work REALLY well with agents, but convoluted ones just product churn in the agent tool…",[81],{"value":82,"source":67,"evidence":83},"Benchmark results are not that surprising given experience with LLM API calls",[84],{"id":56,"by":57,"text":58},[86,90,98],{"value":87,"source":67,"evidence":88},"Add harder APIs or more complex queries where current models fail",[89],{"id":70,"by":71,"text":72},{"value":91,"source":67,"evidence":92},"Test OpenAI models such as o3 and the model superglue uses",[93,94],{"id":70,"by":71,"text":72},{"id":95,"by":96,"text":97},"44648649","michael-fuest","Love hearing Sam Altman talking about feeling the AGI and seeing that million dollar reasoning models can't execute simple API calls despite having a lot of docs and the entire…",{"value":99,"source":67,"evidence":100},"Benchmark Supabase APIs",[101],{"id":77,"by":78,"text":79},[103,107],{"value":104,"source":67,"evidence":105},"How well does OpenAI's agent mode handle API tasks",[106],{"id":95,"by":96,"text":97},{"value":108,"source":67,"evidence":109},"Which model superglue used in the benchmark",[110],{"id":70,"by":71,"text":72},[],{"comments":35,"answered":32,"key":113},[114],{"id":45,"text":46},{"commenters":15,"max_depth":35,"maker_replies":35,"first_comment_mins":32,"lifespan_hours":37},[117,119,121,123,125],[77,32,15,118,78,32],"project\u002Fpositive\u002Fperformance",[95,35,32,120,96,32],"other\u002Fneutral\u002Fnone",[56,37,15,122,57,32],"project\u002Fpositive\u002Fidea",[45,31,32,124,11,35],"project\u002Fpositive\u002Fexecution",[70,126,15,122,71,32],4,[],[129,134,139,144,148,153],{"id":130,"name":131,"points":132,"category":133},"44664873","Tender",1030,"consumer_social",{"id":135,"name":136,"points":137,"category":138},"44594584","Conductor",228,"dev_tools",{"id":140,"name":141,"points":142,"category":143},"44630648","connmap",205,"infrastructure",{"id":145,"name":146,"points":147,"category":143},"44661673","Apple Health MCP",199,{"id":149,"name":150,"points":151,"category":152},"44613438","Compass CNC",176,"hardware",{"id":154,"name":155,"points":156,"category":143},"44662497","TheProtector",167,false,"v2",{"id":160,"name":5,"line":6,"ai":157,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":161,"maker":11,"domain":10,"tier":19,"attempts":32,"strip":162,"url":9,"mix":163,"praise":66,"objection":82,"sub":21},44648120,[24,25,26,27],"50505",[31,32,32,32],1794036678961]