[{"data":1,"prerenderedAt":204},["ShallowReactive",2],{"project-32788010":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":33,"aspects":34,"strip":39,"post_text":29,"what":41,"said":71,"maker_in_thread":142,"shape":148,"grid":152,"attempts":165,"same_week":166,"maker_shows":196,"maker_hidden":197,"reader_version":198,"card":199},"32788010","content-parser","Extract Markdown, HTML or text from content-heavy websites","title","Show HN: Extract Markdown, HTML or text from content-heavy websites","https:\u002F\u002Fcontent-parser.com\u002F","content-parser.com","franciscop",1662787099,2022,49,7,978,2570,39,"shown","dev_tools","web content extraction api","api",[24,25,26,27],"html to markdown","article extraction","readability","web scraping api","quiet",null,{"positive":31,"neutral":32,"mixed":31,"critical":32},0,2,4,{"idea":35,"performance":37,"pricing":38},{"positive":31,"critical":36},1,{"positive":31,"critical":31},{"positive":31,"critical":36},[36,40,31,40,36,31,31],3,{"problem":42,"audience":29,"pricing":29,"motivation":51,"time_to_build":29,"stack":55,"openness":68,"stage":69,"team_size":70},{"value":43,"source":44,"evidence":45},"Turning content-heavy web pages into clean Markdown, HTML or text","maker",[46,48],{"id":47},"post",{"id":49,"by":11,"text":50},"32789902","Thanks for the feedback! So far I plan on making this a stepping stone for a fully integrated HN reader, where you can read the whole thing in-page, and for those pages that…",{"value":52,"source":44,"evidence":53},"Stepping stone for a fully integrated HN reader",[54],{"id":49,"by":11,"text":50},[56,60,64],{"value":57,"source":44,"evidence":58},"Readability",[59],{"id":49,"by":11,"text":50},{"value":61,"source":44,"evidence":62},"jsdom",[63],{"id":49,"by":11,"text":50},{"value":65,"source":44,"evidence":66},"fetch",[67],{"id":49,"by":11,"text":50},"unknown","launched","solo",{"praise":72,"objections":81,"requests":97,"questions":117,"alternatives":129},[73],{"value":74,"source":75,"evidence":76},"Commenters like the stripped-down web reading idea","commenter",[77],{"id":78,"by":79,"text":80},"32870253","bprew","I wrote something similar so I could save recipes and web pages for reading offline. And if you save in html, it will inline images, so you can have a single file. In markdown,…",[82,86,90],{"value":83,"source":75,"evidence":84},"Readability misses content on some pages, so it is finicky",[85],{"id":78,"by":79,"text":80},{"value":87,"source":75,"evidence":88},"Free web clippers and archive sites already do this",[89],{"id":78,"by":79,"text":80},{"value":91,"source":75,"evidence":92},"Per-page cost claim seems high for serverless compute",[93],{"id":94,"by":95,"text":96},"32795602","masukomi","Can you expand on this statement \"It costs me only several cents to parse an entire page\"? That sounds like quite a lot to me. We're talking _maybe_ a few seconds of compute time…",[98,105,109],{"value":99,"source":75,"evidence":100},"Option to enable or disable JavaScript",[101],{"id":102,"by":103,"text":104},"32789794","saltymimir","I maintain something similar today, and I'm guessing that the OP uses some combination of the following libraries too (?): - Readability (https:\u002F\u002Fgithub.com\u002Fmozilla\u002Freadability)…",{"value":106,"source":75,"evidence":107},"Front matter metadata option",[108],{"id":102,"by":103,"text":104},{"value":110,"source":75,"evidence":111},"Bypass paywalls by changing user agent or redirecting to archive",[112,113],{"id":102,"by":103,"text":104},{"id":114,"by":115,"text":116},"32791019","neodymiumphish","Could you automate browsing to an archive link instead when a link runs to a common paywalled site?",[118,125],{"value":119,"source":75,"evidence":120},"How is paywall bypassing done, such as by archive.today",[121],{"id":122,"by":123,"text":124},"32825972","akrymski","Archive.today is exceptional at bypassing paywalls (any idea how?) This doesn't seem to work on Bloomberg articles for example.",{"value":126,"source":75,"evidence":127},"How much does it cost to parse a page",[128],{"id":94,"by":95,"text":96},[130,134,138],{"value":131,"source":75,"evidence":132},"Clippy (open source recipe and web page saver)",[133],{"id":78,"by":79,"text":80},{"value":135,"source":75,"evidence":136},"Archive.today",[137],{"id":122,"by":123,"text":124},{"value":139,"source":75,"evidence":140},"Evernote web clipper",[141],{"id":78,"by":79,"text":80},{"comments":32,"answered":32,"key":143},[144,145],{"id":49,"text":50},{"id":146,"text":147},"32797018","OP here, I've added server timing headers to https:\u002F\u002Fcontent-parser.com\u002F, the total fetch+parse is taking me around 0.6-1.2s. The local parsing as a separated step is sync, so I…",{"commenters":149,"max_depth":40,"maker_replies":32,"first_comment_mins":150,"lifespan_hours":151},6,392,157,[153,155,157,159,160,162,163],[78,31,36,154,79,31],"project\u002Fcritical\u002Fidea",[102,36,40,156,103,31],"project\u002Fneutral\u002Fperformance",[49,36,31,158,11,36],"project\u002Fneutral\u002Fidea",[114,36,40,158,115,31],[94,36,36,161,95,31],"project\u002Fcritical\u002Fpricing",[146,36,31,156,11,36],[122,32,31,164,123,31],"other\u002Fneutral\u002Fnone",[],[167,172,177,182,187,191],{"id":168,"name":169,"points":170,"category":171},"32768590","Modern for HN",308,"productivity",{"id":173,"name":174,"points":175,"category":176},"32798891","HiFiScan",260,"creative",{"id":178,"name":179,"points":180,"category":181},"32828309","Infernal Sigil",238,"games",{"id":183,"name":184,"points":185,"category":186},"32837829","StackAid",226,"infrastructure",{"id":188,"name":189,"points":190,"category":20},"32828799","s3sqlite",160,{"id":192,"name":193,"points":194,"category":195},"32823589","pg_netstat",158,"data",13,false,"v2",{"id":200,"name":5,"line":6,"ai":197,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":201,"maker":11,"domain":10,"tier":19,"attempts":31,"strip":202,"url":9,"mix":203,"praise":74,"objection":83,"sub":21},32788010,[24,25,26,27],"1303100",[31,32,31,32],1794036624751]