[{"data":1,"prerenderedAt":395},["ShallowReactive",2],{"project-2310852":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":46,"post_text":48,"what":49,"said":103,"maker_in_thread":231,"shape":239,"grid":241,"attempts":359,"same_week":360,"maker_shows":45,"maker_hidden":387,"reader_version":388,"card":389},"2310852","Diffbot Article API","Readability-like API Using Machine Learning","title","Show HN: Readability-like API Using Machine Learning","http:\u002F\u002Fwww.diffbot.com\u002Fdocs\u002Fapi\u002Farticle","diffbot.com","miket",1299792201,2011,151,49,57,1097,6,"notable","ai","content extraction api","api",[24,25,26,27],"article extraction","readability api","web scraping","boilerplate removal","mixed",0.226,{"positive":31,"neutral":32,"mixed":33,"critical":34},12,14,1,4,31,{"docs":37,"design":39,"idea":40,"pricing":41,"execution":42,"performance":44},{"positive":33,"critical":38},0,{"positive":38,"critical":38},{"positive":34,"critical":38},{"positive":38,"critical":38},{"positive":38,"critical":43},3,{"positive":45,"critical":33},2,[38,43,47,38,43,38,43,43,47,43,38,38,38,43,43,38,47,43,47,47,38,33,47,38,47,38,43,43,43,47,33,33,38,33,38,43,47,47,47,47],5,null,{"problem":50,"audience":62,"pricing":78,"motivation":48,"time_to_build":48,"stack":85,"openness":101,"stage":102,"team_size":101},{"value":51,"source":52,"evidence":53},"Extracts the main article content from web pages, removing boilerplate and ads.","inferred",[54,58],{"id":55,"by":56,"text":57},"2312979","aaronkaplan","Article content starts out in a straightforward, easy-to-process form, as created by the reporter\u002Fauthor, in a content management system. Then the CMS chops it up into pages and…",{"id":59,"by":60,"text":61},"2311185","modeless","Cool, works better than other services like this I've found. I tried it on Ars Technica's review of the Xoom tablet and it found all 10 pages. It didn't find the embedded video…",{"value":63,"source":64,"evidence":65},"Developers building apps that process web articles.","commenter",[66,70,74],{"id":67,"by":68,"text":69},"2310988","tansey","I really like this. You guys should really open up a tagging API. As a developer working on a social site, I'd love to be able to auto-tag content that users upload.",{"id":71,"by":72,"text":73},"2311039","jranck","This looks great. I'd love to find out more about your API and what type of web scraping techniques you're using. It looks like this is going to be available publicly to…",{"id":75,"by":76,"text":77},"2311999","jamongkad","I'm an interested party as well. As a student of machine learning I would love to learn the techniques you've applied. I'm doing a data mining startup and this would be very…",{"value":79,"source":80,"evidence":81},"Commercial use allowed; terms revised by maker.","maker",[82],{"id":83,"by":11,"text":84},"2311064","Not at all, feel free to use it for commercial uses. I'm removing that from the terms.",[86,95],{"value":87,"source":80,"evidence":88},"Visual-based statistical classification",[89,92],{"id":90,"by":11,"text":91},"2311082","I'd be happy to talk about the visual-based statistical classification technology in a follow-up blog post if there's interest.",{"id":93,"by":11,"text":94},"2311867","In short, yes. The key innovation is that we've come up with a lossy, fixed-length representation of the visual features that we can use to do classification upon. I'll try to do…",{"value":96,"source":80,"evidence":97},"CSS rendering of page elements",[98],{"id":99,"by":11,"text":100},"2311265","It's fairly CPU intensive. Like many classification-based techniques, much of the computation is in constructing the features. For our case, this means we had to implement most…","unknown","launched",{"praise":104,"objections":135,"requests":155,"questions":184,"alternatives":203},[105,117,127],{"value":106,"source":64,"evidence":107},"Works better than comparable services and gives good results",[108,109,113],{"id":59,"by":60,"text":61},{"id":110,"by":111,"text":112},"2312011","npp","In an earlier comment, you said that much of the time was spent in constructing the features (e.g. you had to implement CSS). Did you mean implementation time, or…",{"id":114,"by":115,"text":116},"2310927","rickyyean","lol",{"value":118,"source":80,"evidence":119},"Fast processing, about 80ms per article",[120,123],{"id":121,"by":11,"text":122},"2311895","I just timed that Huffington Post example on my computer and it took around 80ms to extract at 100% CPU. The rub though is that, if you type in a totally new URL on our demo…",{"id":124,"by":125,"text":126},"2311030","mcfchan","That's a really nifty API. Performance is nice too. Would be interested in knowing more about it.",{"value":128,"source":64,"evidence":129},"Interesting machine learning approach",[130,134],{"id":131,"by":132,"text":133},"2311879","itsnotvalid","I like the \"machine learning\" part of the api, but there seems to be no way of improving the learning by giving feedback. Just testing another article on HN[1] that the tags are…",{"id":75,"by":76,"text":77},[136,140,144],{"value":137,"source":64,"evidence":138},"Returns only raw text, losing structure like headers and images",[139],{"id":59,"by":60,"text":61},{"value":141,"source":64,"evidence":142},"Tag quality is poor for some articles",[143],{"id":131,"by":132,"text":133},{"value":145,"source":64,"evidence":146},"Fails on some pages such as forums and paywalled sites",[147,151],{"id":148,"by":149,"text":150},"2311093","sqrt17","Duh... for the pages I tried, it always gives me the \"No article at this URL\". I'd like to have some kind of boilerplate removal that works well for forum content (e.g. phpBB and…",{"id":152,"by":153,"text":154},"2311887","buss","I noticed that this will happen for NYT articles behind their registration-wall.",[156,166,170,177],{"value":157,"source":64,"evidence":158},"Return simplified HTML structure",[159,163],{"id":160,"by":161,"text":162},"2312331","bravura","It would be great if you could return a normalized and simplified version of the HTML structure. I know a lot of people who would be interested in this.",{"id":164,"by":11,"text":165},"2311208","Right now, the API just returns back the raw text for simplicity's sake, but it would be possible to make an option for returning a bit of HTML structure, which would address the…",{"value":167,"source":64,"evidence":168},"Feedback mechanism to improve learning",[169],{"id":131,"by":132,"text":133},{"value":171,"source":64,"evidence":172},"Blog post explaining the technique",[173,174],{"id":90,"by":11,"text":91},{"id":175,"by":132,"text":176},"2311845","+1 for good computer science writeups",{"value":178,"source":64,"evidence":179},"Support for Wikipedia pages",[180],{"id":181,"by":182,"text":183},"2317413","Mamady","Looks good but doesn't work for wikipedia. Get it working there and you will have a lot more consumers.",[185,196],{"value":186,"source":64,"evidence":187},"How resource-intensive is it to run?",[188,192],{"id":189,"by":190,"text":191},"2311211","shantanubala","This is fantastic! How resource-intense is it to run? Machine Learning, depending on the implementation, can be pretty costly from what I understand.",{"id":193,"by":194,"text":195},"2311769","pjscott","Can you quantify that? Like, on a reasonable computer, how long does it take to process the Huffington Post example article? And (though this may be a little trickier to…",{"value":197,"source":64,"evidence":198},"What rendering engine is used for CSS?",[199],{"id":200,"by":201,"text":202},"2312356","ajays","What do you use to get the visual CSS rendering of the elements on the page? A library like libmoz, perhaps?",[204,214,218,225],{"value":205,"source":64,"evidence":206},"Boilerpipe",[207,211],{"id":208,"by":209,"text":210},"2311910","petercooper","Diffbot's stuff is in a different league (but it's a hosted service with a large dataset) but if anyone's vaguely interested in this area, I've been working on a Ruby library…",{"id":212,"by":161,"text":213},"2312301","There is an online demo of boilerpipe: http:\u002F\u002Fboilerpipe-web.appspot.com\u002F I use boilerpipe a lot and highly recommend it. We've discussed it before on MetaOptimize:…",{"value":215,"source":64,"evidence":216},"Pismo",[217],{"id":208,"by":209,"text":210},{"value":219,"source":64,"evidence":220},"Purifyr",[221],{"id":222,"by":223,"text":224},"2312413","alexdong","We are using http:\u002F\u002Fpurifyr.com\u002F for this. Pretty happy with the unicode support and 20-50 documents per second speed.",{"value":226,"source":64,"evidence":227},"Zemanta",[228],{"id":229,"by":68,"text":230},"2311118","Exactly. For instance, if a user is typing in a blog post, right now the flow would be: 1) User submits post 2) I create a page for their post 3) I call your API to analyze the…",{"comments":232,"answered":47,"key":233},9,[234,235,238],{"id":164,"text":165},{"id":236,"text":237},"2311079","Great suggestion, you can use the article API already to do this (providing the URL of the post and the 'tag' parameter), but maybe you are thinking of analyzing other types of…",{"id":83,"text":84},{"commenters":35,"max_depth":18,"maker_replies":232,"first_comment_mins":47,"lifespan_hours":240},115,[242,244,248,251,252,255,257,259,261,264,266,268,270,274,277,279,283,285,286,288,291,295,297,299,300,301,303,306,309,310,311,312,314,317,319,320,323,326,328,329,331,332,334,338,341,345,349,353,357],[208,38,38,243,209,38],"other\u002Fneutral\u002Fnone",[245,38,43,246,247,38],"2312219","project\u002Fneutral\u002Fdocs","alok-g",[249,38,47,250,209,38],"2312252","project\u002Fpositive\u002Fnone",[212,38,38,243,161,38],[253,38,43,254,132,38],"2313692","project\u002Fneutral\u002Fnone",[59,33,43,256,60,38],"project\u002Fmixed\u002Fdesign",[164,33,38,258,11,33],"maker\u002Fneutral\u002Fdesign",[160,33,43,260,161,38],"project\u002Fneutral\u002Fdesign",[262,33,43,260,263,38],"2327934","dfgonzalez",[67,45,47,265,68,38],"project\u002Fpositive\u002Fidea",[236,45,38,267,11,33],"maker\u002Fneutral\u002Fidea",[229,45,43,269,68,38],"project\u002Fneutral\u002Fidea",[271,45,38,272,273,38],"2312419","other\u002Fneutral\u002Fidea","flavy",[275,45,38,276,68,38],"2314872","other\u002Fpositive\u002Fnone",[278,45,38,243,273,38],"2324318",[280,43,43,281,282,38],"2311032","project\u002Fneutral\u002Fpricing","quan",[83,43,38,284,11,33],"maker\u002Fneutral\u002Fpricing",[71,34,43,254,72,38],[90,34,38,287,11,33],"maker\u002Fneutral\u002Fexecution",[289,34,47,265,290,38],"2311108","kmfrk",[292,34,43,293,294,38],"2311818","project\u002Fneutral\u002Fexecution","alextp",[93,34,38,296,11,33],"project\u002Fpositive\u002Fexecution",[175,34,47,298,132,38],"maker\u002Fpositive\u002Fdocs",[75,34,47,265,76,38],[55,47,38,243,56,38],[131,18,33,302,132,38],"project\u002Fcritical\u002Fperformance",[124,304,47,305,125,38],7,"project\u002Fpositive\u002Fperformance",[189,307,43,308,190,38],8,"project\u002Fneutral\u002Fperformance",[99,307,38,308,11,33],[193,307,47,305,194,38],[121,307,38,308,11,33],[110,307,43,313,111,38],"maker\u002Fneutral\u002Fperformance",[315,307,38,316,161,38],"2312319","commenter\u002Fneutral\u002Fperformance",[318,307,43,260,161,38],"2312316",[200,307,43,287,201,38],[321,232,47,265,322,38],"2311177","ronnoch",[148,324,33,325,149,38],10,"project\u002Fcritical\u002Fexecution",[327,324,38,246,11,33],"2311115",[152,324,33,325,153,38],[222,330,38,243,223,38],11,[181,31,33,325,182,38],[333,31,38,287,11,33],"2324889",[335,336,43,337,263,38],"2328034",13,"maker\u002Fneutral\u002Fnone",[339,32,43,269,340,38],"2311141","MrVitaliy",[342,343,47,250,344,38],"2311540",15,"normaluser",[346,347,47,250,348,38],"2310964",16,"bitanarch",[350,351,47,250,352,38],"2310865",17,"kevingao1",[354,351,47,355,356,38],"2310872","maker\u002Fpositive\u002Fnone","atse",[114,351,38,358,115,38],"commenter\u002Fneutral\u002Fnone",[],[361,366,371,376,380,384],{"id":362,"name":363,"points":364,"category":365},"2333570","ZetaBee",238,"productivity",{"id":367,"name":368,"points":369,"category":370},"2322985","colorapi.com",159,"creative",{"id":372,"name":373,"points":374,"category":375},"2324634","Domain Groovy Keyword Tool",61,"web_saas",{"id":377,"name":378,"points":379,"category":370},"2307813","Jazzroom",45,{"id":381,"name":382,"points":383,"category":375},"2315228","Profile.io",41,{"id":385,"name":386,"points":383,"category":375},"2332464","ducksboard",false,"v2",{"id":390,"name":5,"line":6,"ai":387,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":391,"top":18,"kw":392,"maker":11,"domain":10,"tier":19,"attempts":38,"strip":393,"url":9,"mix":394,"praise":106,"objection":137,"sub":21},2310852,0.23,[24,25,26,27],"0350303353000330535501505033351101035555",[31,32,33,34],1793990871338]