[{"data":1,"prerenderedAt":277},["ShallowReactive",2],{"project-16437082":3},{"id":4,"name":5,"line":5,"line_source":6,"title":7,"url":8,"domain":9,"maker":10,"posted_at":11,"year":12,"points":13,"comments":14,"rank_in_year":15,"shown_in_year":16,"top":17,"tier":18,"category":19,"subcategory":20,"format":21,"keywords":22,"reception":26,"sentiment":27,"stance_mix":28,"scored_comments":33,"aspects":34,"strip":39,"post_text":42,"what":43,"said":58,"maker_in_thread":146,"shape":148,"grid":151,"attempts":239,"same_week":240,"maker_shows":29,"maker_hidden":269,"reader_version":270,"card":271},"16437082","Headless Chrome Crawler","title","Show HN: Headless Chrome Crawler","https:\u002F\u002Fgithub.com\u002Fyujiosaka\u002Fheadless-chrome-crawler","github.com","yujiosaka",1519302692,2018,172,33,221,2155,11,"notable","dev_tools","web crawler","library",[23,20,24,25],"headless chrome","puppeteer","scraping","curious",-0.062,{"positive":29,"neutral":30,"mixed":31,"critical":32},1,14,0,4,19,{"docs":35,"execution":36,"idea":37,"performance":38},{"positive":31,"critical":31},{"positive":29,"critical":32},{"positive":31,"critical":31},{"positive":31,"critical":31},[31,31,40,31,31,40,31,31,31,31,31,31,31,29,40,29,40,40,41,31,40,40,40,40,40,29,31,40,31,40,40,29,40],3,5,null,{"problem":42,"audience":42,"pricing":42,"motivation":42,"time_to_build":42,"stack":44,"openness":57,"stage":57,"team_size":57},[45,53],{"value":46,"source":47,"evidence":48},"Puppeteer","commenter",[49],{"id":50,"by":51,"text":52},"16438977","itsjustme2","Looks like this is actually built on top of puppeteer. See the \"Note\" under \"Installation\": https:\u002F\u002Fgithub.com\u002Fyujiosaka\u002Fheadless-chrome-crawler\u002Fblob\u002Fmaster\u002FREADME.md#installation",{"value":54,"source":47,"evidence":55},"Headless Chrome",[56],{"id":50,"by":51,"text":52},"unknown",{"praise":59,"objections":67,"requests":93,"questions":108,"alternatives":125},[60],{"value":61,"source":47,"evidence":62},"Pretty cool; useful for custom crawlers",[63],{"id":64,"by":65,"text":66},"16437752","ptasker","Pretty cool, but I recommend anyone wanting to do this kind of thing to check out the source Puppeteer library. You can do some really powerful stuff and make a custom crawler…",[68,79,86],{"value":69,"source":47,"evidence":70},"Headless browsers use high resources and are hard to supervise",[71,75],{"id":72,"by":73,"text":74},"16439311","codedokode","This has been possible for a long time with any browser using Selenium for example. It has APIs and client libraries for many languages. Also using a real browser brings a lot of…",{"id":76,"by":77,"text":78},"16440311","tegansnyder","There are a lot of folks reevaluating their crawling engines lately now that Chrome headless is maturing. To me there are some important considerations in terms of CPU\u002Fmemory…",{"value":80,"source":47,"evidence":81},"Crawlers without rate limits burden servers and need banning",[82],{"id":83,"by":84,"text":85},"16439049","tesin","While as a developer I find this super interesting, as a system administrator this makes me cringe. We don't have a lot of resources for servers, and I end up spending a…",{"value":87,"source":47,"evidence":88},"Would rather extend Puppeteer than a new library",[89],{"id":90,"by":91,"text":92},"16437878","isuckatcoding","Yeah I’d really rather that people made extensions to Pupeteer rather than a whole new library.",[94,101],{"value":95,"source":47,"evidence":96},"Support for proxies with authentication",[97],{"id":98,"by":99,"text":100},"16438088","hartator","I was stuck last time I was using headless chrome when I needed to use a proxy with an username and a paasword. Headless chrome just doesn't support it. Any changes on that?",{"value":102,"source":47,"evidence":103},"Support for scaling and distributing across machines",[104],{"id":105,"by":106,"text":107},"16439472","agotterer","Nice job! Can this be scaled and distributed to multiple machines?",[109,116],{"value":110,"source":47,"evidence":111},"How to access individual elements from the page body",[112],{"id":113,"by":114,"text":115},"16437591","bryanrasmussen","Can't see from examples, how do I get back individual elements from the body?",{"value":117,"source":47,"evidence":118},"How it handles pages that load links via JavaScript",[119,122],{"id":120,"by":114,"text":121},"16437698","also how does this handle pages that load with a small number of links and then uses JS to write in a bunch of DOM nodes and links?",{"id":123,"by":114,"text":124},"16441429","most naive code I see is like the following ``` const page = await crawler.browser.newPage(); await page.goto(url); await page.waitForSelector(\"a[href]\"); const hrefs = await…",[126,129,133,140],{"value":46,"source":47,"evidence":127},[128],{"id":64,"by":65,"text":66},{"value":130,"source":47,"evidence":131},"Selenium",[132],{"id":72,"by":73,"text":74},{"value":134,"source":47,"evidence":135},"Supercrawler",[136],{"id":137,"by":138,"text":139},"16440392","artur_makly","there's this too: https:\u002F\u002Fgithub.com\u002Fbrendonboshell\u002Fsupercrawler",{"value":141,"source":47,"evidence":142},"Browserless",[143],{"id":144,"by":138,"text":145},"16440410","maybe with this? https:\u002F\u002FBrowserless.io",{"comments":31,"answered":31,"key":147},[],{"commenters":149,"max_depth":32,"maker_replies":31,"first_comment_mins":150,"lifespan_hours":17},25,97,[152,154,156,160,164,167,168,171,175,176,179,181,184,186,188,191,193,196,198,202,205,209,213,215,218,221,223,225,228,229,232,235,236],[64,31,31,153,65,31],"other\u002Fneutral\u002Fnone",[90,31,31,155,91,31],"commenter\u002Fcritical\u002Fidea",[157,31,40,158,159,31],"16437894","project\u002Fneutral\u002Fdocs","robk",[161,31,31,162,163,31],"16441114","other\u002Fneutral\u002Fdocs","RossM",[165,31,31,162,166,31],"16441403","asadlionpk",[50,31,40,158,51,31],[169,31,31,153,170,31],"16440084","chatmasta",[172,31,31,173,174,31],"16441092","commenter\u002Fneutral\u002Fnone","kodablah",[72,29,31,155,73,31],[83,177,31,178,84,31],2,"other\u002Fcritical\u002Fperformance",[180,177,31,153,73,31],"16439331",[182,177,31,153,183,31],"16439632","superasn",[76,40,31,185,77,31],"other\u002Fneutral\u002Fperformance",[98,32,29,187,99,31],"project\u002Fcritical\u002Fexecution",[189,32,40,158,190,31],"16438208","jancurn",[192,32,29,187,99,31],"16438279",[194,32,40,158,195,31],"16438466","timstapl",[197,32,40,158,190,31],"16438469",[199,32,41,200,201,31],"16442354","project\u002Fpositive\u002Fexecution","transreal",[203,32,31,204,99,31],"16442428","commenter\u002Fpositive\u002Fexecution",[206,41,40,207,208,31],"16441474","project\u002Fneutral\u002Fidea","princehonest",[210,211,40,207,212,31],"16442220",6,"nikisweeting",[113,214,40,158,114,31],7,[216,214,40,158,217,31],"16437705","diggan",[219,214,40,158,220,31],"16437963","ej12n",[222,214,29,187,114,31],"16441368",[137,224,31,153,138,31],8,[105,226,40,227,106,31],9,"project\u002Fneutral\u002Fperformance",[144,226,31,153,138,31],[120,230,40,231,114,31],10,"project\u002Fneutral\u002Fexecution",[233,230,40,231,234,31],"16437829","trevyn",[123,230,29,187,114,31],[237,17,40,207,238,31],"16439253","londt8",[],[241,246,251,256,260,264],{"id":242,"name":243,"points":244,"category":245},"16468196","Six Degrees of Wikipedia",1176,"learning_science",{"id":247,"name":248,"points":249,"category":250},"16442888","Hacker News Classics",864,"consumer_social",{"id":252,"name":253,"points":254,"category":255},"16484040","Buy Me A Coffee",331,"web_saas",{"id":257,"name":258,"points":259,"category":19},"16452852","Mitmproxy 3.0 released",242,{"id":261,"name":262,"points":263,"category":19},"16446250","GitHub Trending Repos",203,{"id":265,"name":266,"points":267,"category":268},"16466147","Jumprock.co",161,"infrastructure",false,"v2",{"id":272,"name":5,"line":5,"ai":269,"cat":19,"year":12,"points":13,"comments":14,"rec":26,"s":273,"top":17,"kw":274,"maker":10,"domain":9,"tier":18,"attempts":31,"strip":275,"url":8,"mix":276,"praise":61,"objection":69,"sub":20},16437082,-0.06,[23,20,24,25],"003003000000013133503333310303313",[29,30,31,32],1794036563628]