[{"data":1,"prerenderedAt":500},["ShallowReactive",2],{"project-32561127":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":36,"aspects":37,"strip":47,"post_text":49,"what":50,"said":111,"maker_in_thread":239,"shape":244,"grid":248,"attempts":462,"same_week":463,"maker_shows":333,"maker_hidden":492,"reader_version":493,"card":494},"32561127","Crawlee","Web scraping and browser automation library for Node.js","title","Show HN: Crawlee – Web scraping and browser automation library for Node.js","https:\u002F\u002Fcrawlee.dev\u002F","crawlee.dev","jancurn",1661235939,2022,282,80,153,2570,6,"legend","dev_tools","web scraping library","library",[24,25,26,27,28],"web scraping","browser automation","node.js","puppeteer","playwright","loved",0.267,{"positive":32,"neutral":33,"mixed":34,"critical":35},19,37,1,3,60,{"design":38,"execution":41,"privacy":42,"idea":43,"performance":44,"docs":45,"pricing":46},{"positive":39,"critical":40},2,0,{"positive":35,"critical":34},{"positive":40,"critical":40},{"positive":34,"critical":34},{"positive":34,"critical":40},{"positive":39,"critical":34},{"positive":40,"critical":40},[48,40,40,40,35,48,35,35,35,34,35,48,35,35,48,35,40,35,48,35,35,35,48,48,48,48,48,35,34,35,48,48,48,48,48,35,48,35,35,35],5,"Hey HN,\n\nThis is Jan, founder of Apify, a web scraping and automation platform. Drawing on our team's years of experience, today we're launching Crawlee [1], the web scraping and browser automation library for Node.js that's designed for the fastest development and maximum reliability in production.\n\nFor details, see the short video [2] or read the announcement blog post [3].\n\nMain features:\n\n-  Supports headless browsers with Playwright or Puppeteer\n\n-  Supports raw HTTP crawling with Cheerio or JSDOM\n\n-  Automated parallelization and scaling of crawlers for best performance\n\n-  Avoids blocki",{"problem":51,"audience":57,"pricing":66,"motivation":74,"time_to_build":82,"stack":83,"openness":108,"stage":109,"team_size":110},{"value":52,"source":53,"evidence":54},"Building reliable web scrapers and crawlers that avoid blocking","maker",[55],{"id":56},"post",{"value":58,"source":59,"evidence":60},"Developers building web scrapers in Node.js","inferred",[61,62],{"id":56},{"id":63,"by":64,"text":65},"32561840","franga2000","Looks like you took the good ideas from Scrapy's crawling engine and combined it with a great scraping API, which is all I ever wanted in a bot framework! I'm especially excited…",{"value":67,"source":68,"evidence":69},"Free and always will be free","commenter",[70],{"id":71,"by":72,"text":73},"32561862","mnmkng","Hey! Crawlee uses the libraries from our fingerprint suite internally. https:\u002F\u002Fgithub.com\u002Fapify\u002Ffingerprint-suite#performance It has an A rating in the BotD (fingerprint.js)…",{"value":75,"source":68,"evidence":76},"Improve the developer experience for Apify customers and used internally in their projects",[77,81],{"id":78,"by":79,"text":80},"32561756","B4nan","Hi there! We dont have any benchmarks for Crawlee just yet, but we are working on those as we speak. We care deeply about bot detection, one of the features of Crawlee is…",{"id":71,"by":72,"text":73},null,[84,88,92,96,100,104],{"value":85,"source":53,"evidence":86},"Node.js",[87],{"id":56},{"value":89,"source":53,"evidence":90},"TypeScript",[91],{"id":56},{"value":93,"source":53,"evidence":94},"Playwright",[95],{"id":56},{"value":97,"source":53,"evidence":98},"Puppeteer",[99],{"id":56},{"value":101,"source":53,"evidence":102},"Cheerio",[103],{"id":56},{"value":105,"source":53,"evidence":106},"JSDOM",[107],{"id":56},"open_source","launched","company",{"praise":112,"objections":132,"requests":165,"questions":193,"alternatives":219},[113,121,128],{"value":114,"source":68,"evidence":115},"Unified API for headless and HTTP scraping",[116,117],{"id":63,"by":64,"text":65},{"id":118,"by":119,"text":120},"32561621","samwillis","This looks really neat, I love the idea of a single api for both traditional and headless scraping. From my experience headless scraping is in the order of 10-100x slower and…",{"value":122,"source":68,"evidence":123},"Solid production experience and responsive maintainers",[124],{"id":125,"by":126,"text":127},"32562550","corford","We do a lot of web scraping (hundreds of millions of requests, multiple terabytes of data per month) and have been using Crawlee - previously known as Apify SDK - since its v0.20…",{"value":129,"source":68,"evidence":130},"Good design, cleanly built on Scrapy-like ideas",[131],{"id":63,"by":64,"text":65},[133,147,154],{"value":134,"source":68,"evidence":135},"Node.js and JavaScript are disliked as scraping languages",[136,140,144],{"id":137,"by":138,"text":139},"32561914","bilekas","> I'm not particularly happy to see JavaScript begin taking over another field as it truly is an awful language Sorry but this irked me. What exactly are you hangups with JS ?…",{"id":141,"by":142,"text":143},"32563156","saiya-jin","node.js dependency hell that often makes java's transitive dependencies on bigger projects look like hello world? Exception handling? Performance? No real multithreading afaik? I…",{"id":145,"by":64,"text":146},"32577211","Nothing against dynamically typed languages, Python is one as well. Maybe \"awful\" was a bit of a strong word, but to quickly summarise my issues with JS: - all the syntax and…",{"value":148,"source":68,"evidence":149},"Rate limiting and proxy use may disrespect sites",[150],{"id":151,"by":152,"text":153},"32567025","boesboes","I see you basically recommend bypassing rate limits by using proxies etc? Why not just respect rate limits if set properly? A little bit of consideration for what\u002Fwhomever is on…",{"value":155,"source":68,"evidence":156},"Name Crawlee is awkward without an article",[157,161],{"id":158,"by":159,"text":160},"32568318","michaelmior","This looks very cool, but am I the only one who has an aversion to any product\u002Flibrary calling itself the X instead of an X?",{"id":162,"by":163,"text":164},"32575344","httbs","Yes but on Github, the place that people sensitive to the definite article will see the most, it says \"The web scraping and browser automation library.\" I opened a PR to change…",[166,172,186],{"value":167,"source":68,"evidence":168},"Captcha solver plugin",[169],{"id":170,"by":72,"text":171},"32563597","You can use any captcha solving service with Crawlee, but we plan to add a plugin to make its use much easier. It's on our roadmap.",{"value":173,"source":68,"evidence":174},"Better documentation, including login and cookie examples",[175,179,182],{"id":176,"by":177,"text":178},"32565174","pqdbr","This looks really great. However, I can't find examples of how to handle scraping behind a login or a paywall, without having to 'type' credentials every time.",{"id":180,"by":177,"text":181},"32565250","Just found this on the Apify documentation: https:\u002F\u002Fdocs.apify.com\u002Ftutorials\u002Flog-into-a-website-using-puppeteer#save-and-reuse-cookies Is there a similar guide for Crawlee?",{"id":183,"by":184,"text":185},"32562035","alvis","In a way, I hate you, but at the same time I love you. It's because I'm working on something similar to get data for my product. Seems like I'm going to use Ampify instead to…",{"value":187,"source":68,"evidence":188},"Python version",[189],{"id":190,"by":191,"text":192},"32569492","dennisy","Oh this looks lovely, congratulations! I would really like this but running in Python.",[194,205,212],{"value":195,"source":68,"evidence":196},"How does stealth and bot detection compare to other tools",[197,201],{"id":198,"by":199,"text":200},"32561435","OtmaneBenazzou","Hi! It looks really REALLY cool! Is there any kind of detection\u002Fstealthiness benchmark compared to libraries such as puppeteer-stealth or fakebrowser? Honestly no matter how…",{"id":202,"by":203,"text":204},"32566045","thekeyper","Hi! Very cool project. Just out of curiosity, what trips up Crawlee on CreepJS? I haven't heard of anyone actually using it in production (actually don't think it's meant for…",{"value":206,"source":68,"evidence":207},"Can it use Apify proxies without paid hosting",[208],{"id":209,"by":210,"text":211},"32566243","nerdix","Is there a way to use Apify's paid proxies without paying for the hosting? That doesn't look like option on the website.",{"value":213,"source":68,"evidence":214},"Does it run in AWS Lambda",[215],{"id":216,"by":217,"text":218},"32567953","cebert","Can Crawlee run in AWS Lambdas?",[220,228,236],{"value":221,"source":68,"evidence":222},"Scrapy",[223,224],{"id":63,"by":64,"text":65},{"id":225,"by":226,"text":227},"32585288","mdaniel","In that they're trying to be crawling frameworks, and for sure Scrapy allows headed crawling via Splash, it's just not something I've needed or advocate for Scrapy also has a…",{"value":229,"source":68,"evidence":230},"Puppeteer-extra stealth plugin",[231,232],{"id":198,"by":199,"text":200},{"id":233,"by":234,"text":235},"32561935","jonatron","I'm not aware of a benchmark, but puppeteer-extra-plugin-stealth can be detected:…",{"value":93,"source":68,"evidence":237},[238],{"id":183,"by":184,"text":185},{"comments":34,"answered":34,"key":240},[241],{"id":242,"text":243},"32568413","Probably not :)",{"commenters":245,"max_depth":18,"maker_replies":34,"first_comment_mins":246,"lifespan_hours":247},46,40,57,[249,251,254,255,256,259,260,263,265,267,269,271,273,275,276,277,279,281,285,287,289,292,295,298,301,304,307,310,313,316,320,322,325,327,330,332,335,338,343,346,351,354,356,357,361,364,368,370,375,378,382,384,388,392,396,398,400,402,404,407,409,413,415,419,421,425,427,429,431,433,437,439,441,442,445,447,452,454,457,460],[63,40,48,250,64,40],"project\u002Fpositive\u002Fdesign",[252,40,40,253,72,40],"32561896","commenter\u002Fneutral\u002Fnone",[137,40,40,253,138,40],[141,40,40,253,142,40],[257,40,40,253,258,40],"32575590","LunaSea",[145,40,40,253,64,40],[261,40,40,253,262,40],"32566905","nsonha",[264,40,40,253,64,40],"32577126",[266,40,40,253,262,40],"32577981",[198,34,35,268,199,40],"project\u002Fneutral\u002Fnone",[270,34,40,253,64,40],"32561745",[125,34,48,272,126,40],"project\u002Fpositive\u002Fexecution",[78,34,35,274,79,40],"maker\u002Fneutral\u002Fnone",[71,34,35,274,72,40],[202,34,35,268,203,40],[278,34,35,274,72,40],"32566262",[233,34,35,280,234,40],"project\u002Fneutral\u002Fprivacy",[282,39,35,283,284,40],"32570745","project\u002Fneutral\u002Fexecution","mfrye0",[286,39,40,253,72,40],"32571236",[151,35,34,288,152,40],"project\u002Fcritical\u002Fidea",[290,35,35,291,72,40],"32567218","maker\u002Fneutral\u002Fidea",[293,35,40,253,294,40],"32567318","uwuemu",[118,296,48,297,119,40],4,"project\u002Fpositive\u002Fperformance",[299,296,48,300,72,40],"32561720","maker\u002Fpositive\u002Fnone",[302,296,35,268,303,40],"32568860","leonhard",[305,296,40,306,72,40],"32569210","other\u002Fneutral\u002Fnone",[308,48,35,283,309,40],"32562887","nodoodles",[311,48,35,312,72,40],"32563219","maker\u002Fneutral\u002Fexecution",[314,48,40,306,315,40],"32567893","wumpus",[317,18,48,318,319,40],"32563071","project\u002Fpositive\u002Fidea","quickthrower2",[321,18,48,300,72,40],"32563632",[158,323,34,324,159,40],7,"project\u002Fcritical\u002Fdocs",[242,323,40,326,11,34],"commenter\u002Fneutral\u002Fdocs",[328,323,40,326,329,40],"32568432","MattyMc",[162,323,35,331,163,40],"project\u002Fneutral\u002Fdocs",[209,333,35,334,210,40],8,"project\u002Fneutral\u002Fpricing",[336,333,35,337,72,40],"32566363","maker\u002Fneutral\u002Fpricing",[339,340,48,341,342,40],"32566021",9,"project\u002Fpositive\u002Fnone","aaronyarborough",[344,340,35,345,72,40],"32566140","maker\u002Fneutral\u002Fdocs",[347,348,35,349,350,40],"32563337",10,"project\u002Fneutral\u002Fidea","bobajeff",[352,348,35,349,353,40],"32563491","willmorrison",[355,348,40,253,72,40],"32563602",[170,348,35,291,72,40],[358,359,35,268,360,40],"32561804",11,"monkeydust",[362,359,35,268,363,40],"32561984","metalwarrior665",[365,366,48,341,367,40],"32562400",12,"apienx",[369,366,48,300,72,40],"32562553",[371,372,35,373,374,40],"32564004",13,"project\u002Fneutral\u002Fdesign","lloydatkinson",[376,372,35,377,72,40],"32565249","maker\u002Fneutral\u002Fperformance",[379,380,48,341,381,40],"32564711",14,"conradfr",[383,380,40,253,72,40],"32564913",[385,386,48,341,387,40],"32561982",15,"OlikEten",[389,386,48,390,391,40],"32562544","maker\u002Fpositive\u002Fdesign","mvolfik",[393,394,48,341,395,40],"32583079",16,"saasxyz",[190,397,35,268,191,40],17,[399,397,40,253,226,40],"32570837",[401,397,35,349,191,40],"32582822",[225,397,34,403,226,40],"project\u002Fcritical\u002Fexecution",[405,397,35,268,406,40],"32589994","riekus",[408,397,40,253,226,40],"32594736",[410,411,48,300,412,40],"32562238",18,"daolf",[414,411,35,274,72,40],"32562246",[416,411,48,417,418,40],"32562477","maker\u002Fpositive\u002Fexecution","tpaksoy",[420,411,35,274,79,40],"32562484",[422,32,48,423,424,40],"32564084","project\u002Fpositive\u002Fdocs","ketanip",[216,426,35,268,217,40],20,[428,426,35,268,72,40],"32568022",[430,426,48,341,217,40],"32568032",[432,426,48,417,72,40],"32568635",[434,435,35,349,436,40],"32562216",21,"wigster",[438,435,35,349,72,40],"32562268",[176,440,35,331,177,40],22,[180,440,35,331,177,40],[443,440,48,444,72,40],"32565328","maker\u002Fpositive\u002Fdocs",[446,440,35,268,72,40],"32565278",[448,449,35,450,451,40],"32562129",23,"project\u002Fneutral\u002Fperformance","elevenoh",[453,449,35,450,72,40],"32562737",[455,449,40,456,363,40],"32562817","other\u002Fneutral\u002Fidea",[183,458,35,459,184,40],24,"project\u002Fmixed\u002Fdocs",[461,458,35,331,363,40],"32562358",[],[464,469,474,479,483,488],{"id":465,"name":466,"points":467,"category":468},"32572770","Pornpen.ai",711,"ai",{"id":470,"name":471,"points":472,"category":473},"32568423","react-design-editor",232,"creative",{"id":475,"name":476,"points":477,"category":478},"32521395","SineRider",220,"games",{"id":480,"name":481,"points":482,"category":20},"32578311","Fuzz Map",205,{"id":484,"name":485,"points":486,"category":487},"32551862","Digs.fm",131,"consumer_social",{"id":489,"name":490,"points":491,"category":487},"32534907","Brew.fm",128,false,"v2",{"id":495,"name":5,"line":6,"ai":492,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":496,"top":18,"kw":497,"maker":11,"domain":10,"tier":19,"attempts":40,"strip":498,"url":9,"mix":499,"praise":114,"objection":134,"sub":21},32561127,0.27,[24,25,26,27,28],"5000353331353353035333555553135555535333",[32,33,34,35],1793991050603]