[{"data":1,"prerenderedAt":154},["ShallowReactive",2],{"project-35807020":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":31,"aspects":33,"strip":35,"post_text":29,"what":36,"said":57,"maker_in_thread":98,"shape":104,"grid":106,"attempts":116,"same_week":117,"maker_shows":146,"maker_hidden":147,"reader_version":148,"card":149},"35807020","Promptfoo","CLI for testing & improving LLM prompt quality","title","Show HN: Promptfoo – CLI for testing & improving LLM prompt quality","https:\u002F\u002Fgithub.com\u002Ftyppo\u002Fpromptfoo","github.com","typpo",1683144194,2023,14,5,2012,3109,65,"shown","dev_tools","prompt testing cli","cli",[24,25,26,27],"llm testing","prompt evaluation","prompt engineering","model comparison","quiet",null,{"positive":31,"neutral":32,"mixed":32,"critical":32},1,0,{"design":34},{"positive":31,"critical":32},[32,32,32,15,32],{"problem":37,"audience":44,"pricing":29,"motivation":51,"time_to_build":29,"stack":55,"openness":56,"stage":56,"team_size":56},{"value":38,"source":39,"evidence":40},"No systematic way to compare prompt and model outputs across many examples","maker",[41],{"id":42,"by":11,"text":43},"35807023","Hi HN, I built this because I'm tuning a bunch of prompts and don't have a great way to do this systematically. This CLI tool helps you pick the best prompt and model by allowing…",{"value":45,"source":39,"evidence":46},"Developers tuning LLM prompts who run tests in bulk",[47,48],{"id":42,"by":11,"text":43},{"id":49,"by":11,"text":50},"35811410","Looks like the playground is mainly for comparison between models, not prompts, and doesn't support templating? Vercel's is similar but not free and open-source. I'm running…",{"value":52,"source":39,"evidence":53},"Maker was tuning many prompts and lacked a systematic method",[54],{"id":42,"by":11,"text":43},[],"unknown",{"praise":58,"objections":67,"requests":68,"questions":80,"alternatives":88},[59],{"value":60,"source":61,"evidence":62},"Good idea; CLI suits running many use cases","commenter",[63],{"id":64,"by":65,"text":66},"35808556","Oras","This is a good idea, I used gradio and streamlit to list outputs from different models to check the output manually. But using CLI makes more sense for running multiple use cases…",[],[69,73],{"value":70,"source":61,"evidence":71},"Add a config file (yaml or json) for prompts, variables and models",[72],{"id":64,"by":65,"text":66},{"value":74,"source":61,"evidence":75},"Add an init command to scaffold files",[76,77],{"id":64,"by":65,"text":66},{"id":78,"by":11,"text":79},"35809743","Thanks for the suggestion! I've added a `promptfoo init` command so the initial scaffolding is much easier.",[81],{"value":82,"source":61,"evidence":83},"How does it compare to OpenPlayground?",[84],{"id":85,"by":86,"text":87},"35810423","brianjking","https:\u002F\u002Fgithub.com\u002Fnat\u002Fopenplayground How does this compare to Nat's OpenPlayground? Vercel recently launched a playground too.",[89,93],{"value":90,"source":61,"evidence":91},"OpenPlayground by Nat",[92],{"id":85,"by":86,"text":87},{"value":94,"source":61,"evidence":95},"Vercel playground",[96,97],{"id":85,"by":86,"text":87},{"id":49,"by":11,"text":50},{"comments":99,"answered":31,"key":100},3,[101,102,103],{"id":42,"text":43},{"id":49,"text":50},{"id":78,"text":79},{"commenters":99,"max_depth":99,"maker_replies":99,"first_comment_mins":32,"lifespan_hours":105},9,[107,109,111,112,114],[42,32,32,108,11,31],"project\u002Fneutral\u002Fidea",[85,32,32,110,86,32],"commenter\u002Fneutral\u002Fnone",[49,32,32,110,11,31],[64,31,15,113,65,32],"project\u002Fpositive\u002Fdesign",[78,31,32,115,11,31],"maker\u002Fpositive\u002Fdesign",[],[118,123,127,132,137,142],{"id":119,"name":120,"points":121,"category":122},"35781959","webfft",244,"creative",{"id":124,"name":125,"points":126,"category":20},"35762175","Frogmouth",240,{"id":128,"name":129,"points":130,"category":131},"35764355","EVA",237,"data",{"id":133,"name":134,"points":135,"category":136},"35745719","Polish Pixels",216,"games",{"id":138,"name":139,"points":140,"category":141},"35786242","Currl",123,"consumer_social",{"id":143,"name":144,"points":145,"category":20},"35764635","PySaaS",121,10,false,"v2",{"id":150,"name":5,"line":6,"ai":147,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":151,"maker":11,"domain":10,"tier":19,"attempts":32,"strip":152,"url":9,"mix":153,"praise":60,"objection":-1,"sub":21},35807020,[24,25,26,27],"00050",[31,32,32,32],1794036634953]