[{"data":1,"prerenderedAt":326},["ShallowReactive",2],{"project-10420329":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":44,"post_text":45,"what":46,"said":95,"maker_in_thread":204,"shape":211,"grid":215,"attempts":290,"same_week":291,"maker_shows":266,"maker_hidden":319,"reader_version":320,"card":321},"10420329","Apifier","hosted web crawler for developers","title","Show HN: Apifier – hosted web crawler for developers","https:\u002F\u002Fwww.apifier.com\u002F","apifier.com","jancurn",1445360290,2015,101,35,244,2565,10,"notable","infrastructure","hosted web scraping","web_app",[24,25,26,27],"web crawler","web scraping","hosted scraper","phantomjs","curious",-0.01,{"positive":31,"neutral":32,"mixed":33,"critical":34},3,12,0,5,20,{"performance":37,"idea":39,"privacy":41,"pricing":42,"docs":43},{"positive":33,"critical":38},1,{"positive":40,"critical":31},2,{"positive":33,"critical":38},{"positive":33,"critical":33},{"positive":33,"critical":33},[38,33,38,31,33,38,33,31,31,31,33,31,31,31,31,31,33,38,33,38,33,31,33,33,31,33,31,33,34,33,34,33,34,33,33],null,{"problem":47,"audience":57,"pricing":61,"motivation":70,"time_to_build":45,"stack":74,"openness":93,"stage":94,"team_size":93},{"value":48,"source":49,"evidence":50},"Extracting data from complex websites that simple scrapers cannot handle","maker",[51,54],{"id":52,"by":11,"text":53},"10420338","Hello HN! Today we’re launching what we were building for the past couple of months. Apifier is a hosted web crawler for developers that enables them to extract data from any…",{"id":55,"by":11,"text":56},"10421767","You're right that most of the time you don't need to use JavaScript. But look at Google Groups for example - there is an infinite scroll to get all the topics, posts are also…",{"value":58,"source":49,"evidence":59},"Developers who want to scrape websites",[60],{"id":52,"by":11,"text":53},{"value":62,"source":49,"evidence":63},"Pay per GB of data transferred, price not yet published",[64,67],{"id":65,"by":11,"text":66},"10421360","Many thanks! You're absolutely right, we'll publish the price ASAP and also provide some long-term guarantee for users who will depend on our service. BTW do you think pricing…",{"id":68,"by":11,"text":69},"10420825","TBH we don't have the pricing defined yet, because we don't know what will be our server costs. In a few days we will know more and put up the prices.",{"value":71,"source":49,"evidence":72},"Existing scrapers trade off capability for simple UI; wanted JavaScript-based scraping",[73],{"id":52,"by":11,"text":53},[75,81,85,89],{"value":76,"source":49,"evidence":77},"PhantomJS",[78],{"id":79,"by":11,"text":80},"10420745","Thank you! The crawler is currently based on PhantomJS, there is a pool of worker nodes that distribute the crawling to more machines. We use Node.js + MongoDB backend and Meteor…",{"value":82,"source":49,"evidence":83},"Node.js",[84],{"id":79,"by":11,"text":80},{"value":86,"source":49,"evidence":87},"MongoDB",[88],{"id":79,"by":11,"text":80},{"value":90,"source":49,"evidence":91},"Meteor",[92],{"id":79,"by":11,"text":80},"unknown","launched",{"praise":96,"objections":120,"requests":140,"questions":160,"alternatives":189},[97,109],{"value":98,"source":99,"evidence":100},"Useful demos and examples, usable without registering","commenter",[101,105],{"id":102,"by":103,"text":104},"10420805","bentpins","I love the demos, and that you can use them without registering. One thing I couldn't find without making an account was what happens after you've used a Gigabyte. That would be…",{"id":106,"by":107,"text":108},"10421502","benjmn","As a website owner, is it easy to block a rude crawler by contacting you ? (How would I identify in the first place that the crawler is operated by you ? Would my server logfile…",{"value":110,"source":99,"evidence":111},"Would use it for home search or scraping",[112,116],{"id":113,"by":114,"text":115},"10420695","Raphmedia","Exactly what I was looking for in order to efficiently improve my searches for a new home. Thanks!",{"id":117,"by":118,"text":119},"10420691","misiti3780","I like the idea - would probably use it in the future - can you talk a little bit about what technologies you are using?",[121,132],{"value":122,"source":99,"evidence":123},"Using a full browser like PhantomJS is slow and expensive when plain HTTP requests suffice",[124,128],{"id":125,"by":126,"text":127},"10421632","thecodemonkey","I don't quite understand why you would use a full-blown browser like phantomjs for crawling (I've seen a lot of projects recently taking this approach, so this critique is not…",{"id":129,"by":130,"text":131},"10423404","pkulak","If you limit to just that, then there's no benefit over 10 lines of Ruby and Nokogiri.",{"value":133,"source":99,"evidence":134},"Pricing unclear and risky without knowing cost at scale",[135,139],{"id":136,"by":137,"text":138},"10421314","rgbrgb","Looks really cool! Pricing is the big stickler for me. I've been burned too many times to build any critical piece of my app with it without knowing how much it'll cost if it…",{"id":102,"by":103,"text":104},[141,146,153],{"value":142,"source":99,"evidence":143},"Publish pricing",[144,145],{"id":136,"by":137,"text":138},{"id":102,"by":103,"text":104},{"value":147,"source":99,"evidence":148},"Programmatic API access to results",[149],{"id":150,"by":151,"text":152},"10422564","necrodome","How do you access the latest crawling results programmatically? I hope you are not expecting me to click results link for a developer's tool.",{"value":154,"source":99,"evidence":155},"Caching and change detection on a schedule",[156],{"id":157,"by":158,"text":159},"10423438","danielharan","I'm hoping this could save me some work. A few questions, if founders are still around: -Can you cache pages \u002F download entire sites? -If caching, can you detect changes on a…",[161,172,182],{"value":162,"source":99,"evidence":163},"Does it respect robots.txt and publish IP ranges?",[164,168],{"id":165,"by":166,"text":167},"10420602","jacquesm","Do you respect robots.txt? Do you publish your IP ranges?",{"id":169,"by":170,"text":171},"10420642","jakubbalada","Yes, by default we respect robots.txt. There is a switch to disable it - on your own responsibility. We don't fully respect Crawl-delay, but minimum delay between requests for…",{"value":173,"source":99,"evidence":174},"How malicious user code is prevented",[175,179],{"id":176,"by":177,"text":178},"10420909","aakilfernandes","Cool! How do you stop users from trying to run malicious code?",{"id":180,"by":11,"text":181},"10420956","Thank you! The user JavaScript code runs in the context of the web pages in the same restricted environment as normal web page's JavaScript. Also, the crawling processes are…",{"value":183,"source":99,"evidence":184},"How to block the crawler and identify its traffic",[185,186],{"id":106,"by":107,"text":108},{"id":187,"by":11,"text":188},"10421889","Currently, there is no way to distinguish traffic from our crawlers, but of course, let us know at support@apifier.com and we will blacklist your websites from our crawlers.",[190,197],{"value":191,"source":99,"evidence":192},"morph.io",[193],{"id":194,"by":195,"text":196},"10422588","thomasfromcdnjs","Also similar to https:\u002F\u002Fmorph.io\u002F which has more of a trend towards open data sets.",{"value":198,"source":99,"evidence":199},"Running PhantomJS on spot instances",[200],{"id":201,"by":202,"text":203},"10421588","Eridrus","Why should I use this instead of just firing up some spot instances with phantomjs?",{"comments":205,"answered":206,"key":207},13,6,[208,209,210],{"id":55,"text":56},{"id":52,"text":53},{"id":65,"text":66},{"commenters":212,"max_depth":213,"maker_replies":205,"first_comment_mins":38,"lifespan_hours":214},18,4,22,[216,218,220,223,226,230,231,233,235,236,238,240,243,245,247,249,250,253,255,256,257,260,262,263,265,267,268,270,272,274,275,280,283,284,287],[125,33,38,217,126,33],"project\u002Fcritical\u002Fperformance",[55,33,33,219,11,38],"project\u002Fpositive\u002Fdesign",[221,33,38,222,126,33],"10425937","project\u002Fcritical\u002Fidea",[224,33,31,225,202,33],"10421782","project\u002Fneutral\u002Fidea",[227,33,33,228,229,33],"10423060","commenter\u002Fneutral\u002Fperformance","est",[129,33,38,222,130,33],[52,38,33,232,11,38],"project\u002Fneutral\u002Fnone",[165,38,31,234,166,33],"project\u002Fneutral\u002Fprivacy",[169,38,31,234,170,33],[136,40,31,237,137,33],"project\u002Fneutral\u002Fpricing",[65,40,33,239,11,38],"maker\u002Fneutral\u002Fpricing",[241,40,31,237,242,33],"10422961","nsp",[244,40,31,237,170,33],"10423014",[150,31,31,246,151,33],"project\u002Fneutral\u002Fdocs",[248,31,31,246,170,33],"10422616",[157,213,31,232,158,33],[251,213,33,252,11,38],"10423559","project\u002Fneutral\u002Fexecution",[106,34,38,254,107,33],"project\u002Fcritical\u002Fprivacy",[187,34,33,234,11,38],[201,206,38,222,202,33],[258,206,33,259,11,38],"10421695","project\u002Fpositive\u002Fexecution",[102,261,31,237,103,33],7,[68,261,33,237,11,38],[264,261,33,237,11,38],"10420840",[176,266,31,234,177,33],8,[180,266,33,234,11,38],[194,269,31,225,195,33],9,[271,269,33,219,11,38],"10422699",[117,18,34,273,118,33],"project\u002Fpositive\u002Fnone",[79,18,33,252,11,38],[276,277,34,278,279,33],"10421859",11,"project\u002Fpositive\u002Fidea","asterfield",[281,277,33,282,11,38],"10421876","maker\u002Fneutral\u002Fnone",[113,32,34,278,114,33],[285,32,33,286,11,38],"10421037","maker\u002Fpositive\u002Fnone",[288,32,33,289,137,33],"10421297","commenter\u002Fneutral\u002Fnone",[],[292,297,302,306,310,314],{"id":293,"name":294,"points":295,"category":296},"10425959","Metabase",287,"data",{"id":298,"name":299,"points":300,"category":301},"10426106","Clojure for the Brave and True",224,"learning_science",{"id":303,"name":304,"points":305,"category":301},"10405288","Arguman",170,{"id":307,"name":308,"points":309,"category":296},"10403820","CloudScrape",116,{"id":311,"name":312,"points":313,"category":301},"10406165","clock",109,{"id":315,"name":316,"points":317,"category":318},"10411756","startc",97,"languages",false,"v2",{"id":322,"name":5,"line":6,"ai":319,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":323,"maker":11,"domain":10,"tier":19,"attempts":33,"strip":324,"url":9,"mix":325,"praise":98,"objection":122,"sub":21},10420329,[24,25,26,27],"10130103330333330101030030305050500",[31,32,33,34],1794036536855]