[{"data":1,"prerenderedAt":374},["ShallowReactive",2],{"project-16366275":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":33,"aspects":34,"strip":42,"post_text":45,"what":46,"said":86,"maker_in_thread":217,"shape":228,"grid":232,"attempts":337,"same_week":338,"maker_shows":32,"maker_hidden":367,"reader_version":368,"card":369},"16366275","Apify Page Analyzer","Web scraping page analyzer","title","Show HN: Web scraping page analyzer","https:\u002F\u002Fwww.apify.com\u002Fpage-analyzer","apify.com","jardah",1518520484,2018,166,50,231,2155,11,"notable","web_saas","web scraping tool","web_app",[24,25,26,27],"web scraping","page analyzer","headless chrome","structured data","debated",0,{"positive":31,"neutral":18,"mixed":32,"critical":31},6,1,24,{"design":35,"execution":36,"idea":38,"docs":40,"privacy":41},{"positive":29,"critical":32},{"positive":29,"critical":37},4,{"positive":39,"critical":32},2,{"positive":29,"critical":29},{"positive":29,"critical":29},[32,29,29,43,43,43,29,44,43,29,32,29,29,29,32,29,29,43,29,29,43,32,32,43,29,29,43,44,43,29,43,43,43,29,44,44,44,29,32,44],3,5,null,{"problem":47,"audience":45,"pricing":45,"motivation":58,"time_to_build":45,"stack":68,"openness":83,"stage":84,"team_size":85},{"value":48,"source":49,"evidence":50},"Extracting data and metadata from web pages that need rendering or JavaScript.","inferred",[51,54],{"id":52,"by":11,"text":53},"16369299","This tool basicaly performs the simplest data loading, it opens the webpage, then waits till most xhr requests are done, wait's a second (tio give JS time to manipulate DOM) and…",{"id":55,"by":56,"text":57},"16368497","at_smith","Awesome tool! How do you handle scraping data that's hiding behind layers of ~fancy~ JS libraries? Is it as simple as triggering click events, pausing for loading, and then…",{"value":59,"source":60,"evidence":61},"Testing an idea that became a usable tool; also testing Apify cloud and proxies.","maker",[62,65],{"id":63,"by":11,"text":64},"16369378",":D yep the documentation needs a lot of work. It started as a test of an idea, then slowly became a usable tool and the code was getting incrementaly more complex without me…",{"id":66,"by":11,"text":67},"16368302","Nope, there is no caching now, every run of the tool has a single instance and writes the output into separate file. I'm using it to test stability of cloud when multiple users…",[69,77],{"value":70,"source":71,"evidence":72},"Headless Chrome","commenter",[73],{"id":74,"by":75,"text":76},"16371059","bdcravens","No, it says on the page it's using headless Chrome.",{"value":78,"source":60,"evidence":79},"Apify Actor platform",[80],{"id":81,"by":11,"text":82},"16367885","Good idea and I would implement that if I used an API from server to get the response. But currently I'm at the same time testing stability of Apify \"Actor\" solution and proxies,…","unknown","beta","company",{"praise":87,"objections":110,"requests":138,"questions":172,"alternatives":191},[88,103],{"value":89,"source":71,"evidence":90},"Useful, handy tool for scraping and analysis",[91,95,99],{"id":92,"by":93,"text":94},"16369560","chadlavi","We use this for some stuff at my office. It's handy.",{"id":96,"by":97,"text":98},"16368156","nickthemagicman","Just wanted to say great tool!",{"id":100,"by":101,"text":102},"16369584","alexroan","Love this.",{"value":104,"source":71,"evidence":105},"Clever JavaScript DSL idea",[106],{"id":107,"by":108,"text":109},"16368918","lozzo","why ? and how would you envisage authentication support to work ? (p.s. awesome tool guy...the clever part is to allow user the ability to describe their process via JavaScript.…",[111,120,131],{"value":112,"source":60,"evidence":113},"Cannot scrape authenticated sites or POST-based forms",[114,117],{"id":115,"by":11,"text":116},"16369694","Aha! I see, it shows data based on POST request from FORM on this page http:\u002F\u002Fwww.dsden93.ac-creteil.fr\u002Fspip\u002Fspip.php?page=annu1d so if you provide just a link to the results…",{"id":118,"by":11,"text":119},"16368273","Sadly not for now. Our company has a solution for that (for some websites), but currently this tool does not have this functionality, since I wanted it be as simple as possible.…",{"value":121,"source":71,"evidence":122},"Fails on some sites such as Amazon ratings and certain tables",[123,127],{"id":124,"by":125,"text":126},"16375512","petagonoral","hmm, no joy getting rating information from an amazon.com product page",{"id":128,"by":129,"text":130},"16369638","guilamu","Not giving me anything useful on this pretty straightforward table: http:\u002F\u002Fwww.dsden93.ac-creteil.fr\u002Fspip\u002Fspip.php?page=ecoles",{"value":132,"source":71,"evidence":133},"UX: Analyze button hidden below the input",[134],{"id":135,"by":136,"text":137},"16368394","tcmb","I entered a URL and pressed enter, wondering why nothing happened. Only then did I scroll down to find the 'Analyze' button. I wasn't looking for specific attributes, and the…",[139,150,157,164],{"value":140,"source":71,"evidence":141},"Authentication support",[142,146],{"id":143,"by":144,"text":145},"16368715","lanewinfield","Authentication support on this would make it an instant purchase for me.",{"id":147,"by":148,"text":149},"16368250","BrandoElFollito","Is there a way to access an authenticated web site?",{"value":151,"source":71,"evidence":152},"RDFa support",[153],{"id":154,"by":155,"text":156},"16368567","oevi","Nice work! It seems that it only supports microdata and not RDFa at the moment?",{"value":158,"source":71,"evidence":159},"Listing all page assets by type",[160],{"id":161,"by":162,"text":163},"16369207","JustARandomGuy","Suppose I wanted to extract an image that gets loaded async via Javascript (For example, a Pinterest page). How would that work? Looking at your documentation, it looks like I…",{"value":165,"source":71,"evidence":166},"Recording user actions like clicks and form inputs",[167,168],{"id":55,"by":56,"text":57},{"id":169,"by":170,"text":171},"16370114","cseelus","Could be an interesting enhancement. Sounds a little bit like what Capybara, a test framework for Ruby apps can do[1], things like ``` click_link('Link Text') fill_in('Password',…",[173,177,184],{"value":174,"source":71,"evidence":175},"How it handles JavaScript-heavy pages",[176],{"id":55,"by":56,"text":57},{"value":178,"source":71,"evidence":179},"Whether it works behind WAFs",[180],{"id":181,"by":182,"text":183},"16372759","cstrat","Looks awesome! Does the tool work when trying to access websites behind web application firewalls? eg. F5 WAF [1] https:\u002F\u002Ff5.com\u002Fglossary\u002Fweb-application-firewall",{"value":185,"source":71,"evidence":186},"Whether it handles digital certificates",[187],{"id":188,"by":189,"text":190},"16367544","rmateus","Is it able to deal with digital certificates?",[192,196,203,210],{"value":193,"source":71,"evidence":194},"Capybara",[195],{"id":169,"by":170,"text":171},{"value":197,"source":71,"evidence":198},"Horseman\u002FPhantomJS",[199],{"id":200,"by":201,"text":202},"16370294","razki","I'd have money on using Horseman with phantomjs in node.",{"value":204,"source":71,"evidence":205},"Microlink",[206],{"id":207,"by":208,"text":209},"16400186","Kikobeats","Similar but just for getting normalized metadata: https:\u002F\u002Fmicrolink.io",{"value":211,"source":71,"evidence":212},"Feedity",[213],{"id":214,"by":215,"text":216},"16372022","nreece","Cool tool! * shameless plug * Our little startup, Feedity - https:\u002F\u002Ffeedity.com, helps create custom RSS feeds for any webpage, utilizing Chrome for full-rendering and many other…",{"comments":218,"answered":219,"key":220},19,10,[221,224,227],{"id":222,"text":223},"16368456","Oh... Clearly I need to work on my UX skills, I will improve that in next iteration.",{"id":225,"text":226},"16369785","Just a quick update: Thank you for using it and playing around with it. Looking at the usage and results I found a quite a lot of things to improve. Which is great, since it's…",{"id":52,"text":53},{"commenters":229,"max_depth":31,"maker_replies":218,"first_comment_mins":230,"lifespan_hours":231},29,156,96,[233,235,237,239,241,242,244,245,247,251,253,254,256,258,260,262,263,265,267,270,273,275,278,281,284,286,288,290,293,295,298,301,303,304,306,308,313,314,315,317,320,321,322,324,325,327,329,331,333,335],[135,29,32,234,136,29],"project\u002Fcritical\u002Fdesign",[222,29,29,236,11,32],"maker\u002Fneutral\u002Fdesign",[225,32,29,238,11,32],"maker\u002Fneutral\u002Fnone",[55,39,43,240,56,29],"project\u002Fneutral\u002Fexecution",[52,39,29,240,11,32],[169,39,43,243,170,29],"project\u002Fneutral\u002Fidea",[200,39,43,240,201,29],[74,39,29,246,75,29],"commenter\u002Fneutral\u002Fnone",[248,39,29,249,250,29],"16375135","other\u002Fneutral\u002Fnone","oodavid",[214,43,44,252,215,29],"project\u002Fpositive\u002Fnone",[181,37,43,240,182,29],[255,37,29,240,11,32],"16374097",[128,44,32,257,129,29],"project\u002Fcritical\u002Fexecution",[259,44,29,240,11,32],"16369661",[261,44,29,240,11,32],"16369669",[115,44,29,240,11,32],[264,44,29,246,129,29],"16371122",[207,31,32,266,208,29],"project\u002Fcritical\u002Fidea",[268,269,29,238,11,32],"16367060",7,[271,269,29,246,272,29],"16367302","GSGSGS",[274,269,29,238,11,32],"16367320",[276,269,43,277,272,29],"16368307","project\u002Fmixed\u002Fdocs",[279,269,29,249,280,29],"16368823","jancurn",[282,269,29,249,283,29],"16373455","johnnyfived",[63,269,29,285,11,32],"maker\u002Fneutral\u002Fdocs",[188,269,43,287,189,29],"project\u002Fneutral\u002Fnone",[289,269,32,257,280,29],"16367728",[291,269,32,257,292,29],"16367706","_Chief",[81,269,29,294,11,32],"maker\u002Fneutral\u002Fexecution",[296,269,43,287,297,29],"16368111","ComputerGuru",[299,269,29,249,300,29],"16368598","bpicolo",[302,269,29,238,11,32],"16369222",[161,269,43,240,162,29],[305,269,29,240,11,32],"16369350",[92,307,44,252,93,29],8,[309,310,43,311,312,29],"16368190",9,"project\u002Fneutral\u002Fprivacy","dmarlow",[66,310,29,287,11,32],[154,219,43,240,155,29],[316,219,29,240,11,32],"16369159",[318,219,43,243,319,29],"16370740","anomie31",[147,18,43,243,148,29],[118,18,29,243,11,32],[143,18,44,323,144,29],"project\u002Fpositive\u002Fidea",[107,18,44,323,108,29],[326,18,29,243,11,32],"16369250",[96,328,44,252,97,29],12,[330,328,29,238,11,32],"16368311",[124,332,32,257,125,29],13,[334,332,29,240,11,32],"16375562",[100,336,44,252,101,29],14,[],[339,344,349,354,358,362],{"id":340,"name":341,"points":342,"category":343},"16339004","TablePlus",333,"dev_tools",{"id":345,"name":346,"points":347,"category":348},"16368653","ModelDepot",324,"ai",{"id":350,"name":351,"points":352,"category":353},"16352611","gqlgen",172,"languages",{"id":355,"name":356,"points":357,"category":343},"16357757","just-dashboard",159,{"id":359,"name":360,"points":361,"category":343},"16336575","Thanks",138,{"id":363,"name":364,"points":365,"category":366},"16364805","Takehome.io",134,"productivity",false,"v2",{"id":370,"name":5,"line":6,"ai":367,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":371,"maker":11,"domain":10,"tier":19,"attempts":29,"strip":372,"url":9,"mix":373,"praise":89,"objection":112,"sub":21},16366275,[24,25,26,27],"1003330530100010030031130035303330555015",[31,18,32,31],1794036563362]