[{"data":1,"prerenderedAt":166},["ShallowReactive",2],{"project-41695549":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":34,"aspects":35,"strip":37,"post_text":39,"what":40,"said":66,"maker_in_thread":107,"shape":111,"grid":114,"attempts":130,"same_week":131,"maker_shows":158,"maker_hidden":159,"reader_version":160,"card":161},"41695549","Pipet","CLI tool for scraping and extracting data online, with pipes","title","Show HN: Pipet – CLI tool for scraping and extracting data online, with pipes","https:\u002F\u002Fgithub.com\u002Fbjesus\u002Fpipet","github.com","yoavm",1727692050,2024,49,7,1307,3515,38,"shown","dev_tools","web scraping cli","cli",[24,22,25,26,27],"web scraping","data extraction","unix pipes","curl","quiet",null,{"positive":31,"neutral":32,"mixed":32,"critical":33},2,0,1,3,{"performance":36},{"positive":32,"critical":33},[38,32,38,33,32,32,32],5,"I often find myself in situations where I need to extract some data from a website, either as one time thing or periodically (and then watch for changes). Maybe I'm tracking stocks, or a delivery, or I want to know when tickets become available to the local sauna; maybe a friend called and asked \"can you get me all data from that website into a spreadsheet?\" (this happens surprisingly often)\n\nI used to write one-off scripts for that, often with Python or JavaScript, but I noticed I spend about one minute getting the right CSS selectors, and then another 10 minutes setting the rest of the scrip",{"problem":41,"audience":29,"pricing":29,"motivation":47,"time_to_build":29,"stack":51,"openness":65,"stage":65,"team_size":65},{"value":42,"source":43,"evidence":44},"Writing one-off scrapers and boilerplate to extract data from websites","maker",[45],{"id":46},"post",{"value":48,"source":43,"evidence":49},"Maker kept writing one-off scraping scripts and wanted to stop writing boilerplate",[50],{"id":46},[52,58,61],{"value":53,"source":43,"evidence":54},"Go",[55],{"id":56,"by":11,"text":57},"41724223","Just as an example: if you have go installed on your laptop, you can extract all the comments from this post as JSON by creating a file with ``` curl…",{"value":27,"source":43,"evidence":59},[60],{"id":46},{"value":62,"source":43,"evidence":63},"Playwright",[64],{"id":46},"unknown",{"praise":67,"objections":83,"requests":91,"questions":98,"alternatives":106},[68,76],{"value":69,"source":70,"evidence":71},"Useful for people who keep one-off scraping scripts","commenter",[72],{"id":73,"by":74,"text":75},"41718732","itsbrex","Thus is great. Thanks for sharing. Sitting on a cache of one-off scripts as well. Looking forward to checking this out further.",{"value":77,"source":70,"evidence":78},"Positive reaction to the project",[79],{"id":80,"by":81,"text":82},"41695582","rotemtam","Well done!",[84],{"value":85,"source":70,"evidence":86},"Scraping needs advanced bot evasion such as real browsers and GPU-based human emulation",[87],{"id":88,"by":89,"text":90},"41723911","cynicalsecurity","That's all nonsense. If it's not imitating a real user by launching a real browser on a GPU and moving the mouse cursor with AI like a real human would, you could throw this…",[92],{"value":93,"source":43,"evidence":94},"Example of a site needing AI browser emulation to scrape",[95],{"id":96,"by":11,"text":97},"41724021","The idea is that you run this from your personal computer, so I don't see any issues with the network really? Regarding the AI emulation browser in GPU - do you have an example…",[99],{"value":100,"source":70,"evidence":101},"Whether sites have advanced bot protection that the tool must handle",[102],{"id":103,"by":104,"text":105},"41723961","moltar","Because every website has such an advanced protection?",[],{"comments":31,"answered":32,"key":108},[109,110],{"id":56,"text":57},{"id":96,"text":97},{"commenters":112,"max_depth":31,"maker_replies":31,"first_comment_mins":15,"lifespan_hours":113},6,57,[115,117,119,121,123,125,127],[73,32,38,116,74,32],"project\u002Fpositive\u002Fnone",[56,33,32,118,11,33],"project\u002Fpositive\u002Fdocs",[80,31,38,120,81,32],"maker\u002Fpositive\u002Fnone",[88,34,33,122,89,32],"project\u002Fcritical\u002Fperformance",[103,34,32,124,104,32],"commenter\u002Fneutral\u002Fnone",[96,34,32,126,11,33],"maker\u002Fneutral\u002Fprivacy",[128,34,32,124,129,32],"41724136","mariocesar",[],[132,137,141,145,149,153],{"id":133,"name":134,"points":135,"category":136},"41710227","Tavus",455,"ai",{"id":138,"name":139,"points":140,"category":20},"41701644","qrframe",401,{"id":142,"name":143,"points":144,"category":20},"41711032","Sourcebot",259,{"id":146,"name":147,"points":148,"category":20},"41695756","CrystalClear Audio",223,{"id":150,"name":151,"points":152,"category":20},"41683144","htmgo",166,{"id":154,"name":155,"points":156,"category":157},"41654723","Cronexpr",160,"languages",4,false,"v2",{"id":162,"name":5,"line":6,"ai":159,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":163,"maker":11,"domain":10,"tier":19,"attempts":32,"strip":164,"url":9,"mix":165,"praise":69,"objection":85,"sub":21},41695549,[24,22,25,26,27],"5051000",[31,32,32,33],1794036661917]