[{"data":1,"prerenderedAt":545},["ShallowReactive",2],{"project-40033490":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":46,"post_text":47,"what":48,"said":94,"maker_in_thread":217,"shape":224,"grid":229,"attempts":507,"same_week":508,"maker_shows":39,"maker_hidden":538,"reader_version":539,"card":540},"40033490","markdowndown","A tool to clean and convert any webpage to Markdown","title","Show HN: I made a tool to clean and convert any webpage to Markdown","https:\u002F\u002Fmarkdowndown.vercel.app\u002F","markdowndown.vercel.app","asadalt",1713121438,2024,441,99,95,3515,3,"legend","productivity","web clipper","web_app",[24,25,26,27],"webpage to markdown","html to markdown","article scraper","readability","mixed",-0.08,{"positive":31,"neutral":32,"mixed":33,"critical":34},8,15,2,10,35,{"idea":37,"pricing":40,"execution":42,"performance":44,"design":45},{"positive":38,"critical":39},5,1,{"positive":41,"critical":39},0,{"positive":33,"critical":43},4,{"positive":39,"critical":18},{"positive":41,"critical":39},[18,41,41,41,18,41,38,39,39,41,41,18,41,18,41,41,18,38,41,41,38,41,41,41,41,41,38,39,18,39,41,39,18,43,18,43,18,39,41,41],"My partner usually writes substack posts which I then mirror to our website’s blog section.\n\nTo automate this, I made a simple tool to scrape the post and clean it so that I can drop it to our blog easily. This might be useful to others as well.\n\nOh and ofcourse you can instruct GPT to make any final edits :D",{"problem":49,"audience":55,"pricing":59,"motivation":60,"time_to_build":59,"stack":64,"openness":91,"stage":92,"team_size":93},{"value":50,"source":51,"evidence":52},"Turning web articles into clean Markdown for publishing","maker",[53],{"id":54},"post",{"value":56,"source":51,"evidence":57},"Writers mirroring posts to a blog",[58],{"id":54},null,{"value":61,"source":51,"evidence":62},"Automate mirroring a partner's Substack posts to their website blog",[63],{"id":54},[65,71,75,81,85],{"value":66,"source":51,"evidence":67},"Next.js",[68],{"id":69,"by":11,"text":70},"40036674","it has actually scaled pretty well and has been negligible cost-wise. I didn’t have to do anything to handle HN traffic. Just a boilerplate nextjs app.",{"value":72,"source":51,"evidence":73},"Vercel",[74],{"id":69,"by":11,"text":70},{"value":76,"source":51,"evidence":77},"GPT-4",[78],{"id":79,"by":11,"text":80},"40037353","this is slightly heavy due to loading a headless chrome instance. I will look into optimizing this part. other than that, gpt4 is expensive, but so far it’s been negligible so i…",{"value":82,"source":51,"evidence":83},"Headless Chrome",[84],{"id":79,"by":11,"text":80},{"value":86,"source":51,"evidence":87},"Readability",[88],{"id":89,"by":11,"text":90},"40033677","oh AI is optional here. I do use readability to clean the html before converting to .md.","unknown","launched","solo",{"praise":95,"objections":108,"requests":138,"questions":172,"alternatives":180},[96,104],{"value":97,"source":98,"evidence":99},"Clean output on complex marketing pages","commenter",[100],{"id":101,"by":102,"text":103},"40035654","alsetmusic","Just tried it on a complex marketing page and it did a great job. Congrats! I'm curious, if you care to share, what kind of load this places on your host? Is this something that…",{"value":105,"source":51,"evidence":106},"Handles HN traffic well on Vercel",[107],{"id":69,"by":11,"text":70},[109,120,131],{"value":110,"source":98,"evidence":111},"Gets stuck on cookie consent walls",[112,116],{"id":113,"by":114,"text":115},"40034908","old_dev","It looks like, if the website presents a cookie message, the tool just gets stuck on that and does not parse the actual content. As an example, I tried https:\u002F\u002Fwww.cnbc.com\u002F and…",{"id":117,"by":118,"text":119},"40035014","sigio","Tried it earlier today, but it was hugged to death. Tried it now, but it only gave me the contents of the cookie-wall, not the page I was looking for. Tried on another page of…",{"value":121,"source":98,"evidence":122},"Server overload errors and hugging to death",[123,127],{"id":124,"by":125,"text":126},"40033872","midzer","\"Failed to Convert. Either the URL is invalid or the server is too busy. Please try again later.\"",{"id":128,"by":129,"text":130},"40033894","julienreszka","I think it was hugged to death",{"value":132,"source":98,"evidence":133},"Fails on some sites like cnn.com",[134],{"id":135,"by":136,"text":137},"40037011","cchance","I did cnn.com and it just downloaded a file that said cnn.com can't be found lol",[139,150,161],{"value":140,"source":98,"evidence":141},"Open source the code",[142,146],{"id":143,"by":144,"text":145},"40034094","remorses","Is the code open source?",{"id":147,"by":148,"text":149},"40035123","radicalriddler","Is this open sourced anywhere by any chance? Are you using GPT to do the conversion, or just doing it yourself by ways of HTML -> Markdown substitutions?",{"value":151,"source":98,"evidence":152},"Turn into a browser extension",[153,157],{"id":154,"by":155,"text":156},"40034413","jamesstidard","Nice. Turn this into a browser extension and id install it. Feel like id forget about it otherwise",{"id":158,"by":159,"text":160},"40033706","ChadNauseam","This is awesome. I kind of want a browser extension that does this to every page I read and saves them somewhere.",{"value":162,"source":98,"evidence":163},"Fix tooltips on mobile",[164,168],{"id":165,"by":166,"text":167},"40037354","ohans","nitpick: the tooltip (triggered by the question mark icon) does not work on mobile - at least on my iPhone (both chrome & safari) May be worth taking a look at. Good stuff…",{"id":169,"by":170,"text":171},"40074107","neitsab","Tooltips next to options don't work for me on Firefox for Android 124.0.2, and the converter failed on \"https:\u002F\u002Fbittersoutherner.com\u002Ffeature\u002F2023\u002Fobituary-for-a-quiet-life\"…",[173,176],{"value":145,"source":98,"evidence":174},[175],{"id":143,"by":144,"text":145},{"value":177,"source":98,"evidence":178},"Does it use GPT or HTML-to-Markdown substitutions?",[179],{"id":147,"by":148,"text":149},[181,188,199,206],{"value":182,"source":98,"evidence":183},"Trafilatura, Newspaper4k, python-readability",[184],{"id":185,"by":186,"text":187},"40035347","screye","Converting websites to markdown comes with 3 distinct problems: 1. Throughly scraping the content of page (high recall) 2. Dropping all the ads\u002Fauxilliary content (high…",{"value":189,"source":98,"evidence":190},"Firecrawl, Urlbox, url2text, usescraper",[191,195],{"id":192,"by":193,"text":194},"40034771","jot","Great idea to offer image downloads and filtering with GPT! I built a similar tool last year that doesn't have those features: https:\u002F\u002Furl2text.com\u002F Apologies if the UI is slow -…",{"id":196,"by":197,"text":198},"40034921","cpeffer","Very cool. We posted about a similar tool we built yesterday https:\u002F\u002Fwww.firecrawl.dev\u002F It also crawls (although you can scrape single pages as well)",{"value":200,"source":98,"evidence":201},"Pandoc",[202],{"id":203,"by":204,"text":205},"40034344","blobcode","This is one of those things that the ever-amazing pandoc (https:\u002F\u002Fpandoc.org\u002F) does very well, on top of supporting virtually every other document format.",{"value":207,"source":98,"evidence":208},"MarkDownload browser extension",[209,213],{"id":210,"by":211,"text":212},"40035110","brokenmachine","I use markdownload extension for Firefox. Seems to work pretty ok.",{"id":214,"by":215,"text":216},"40034610","Terretta","This fork: https:\u002F\u002Fgithub.com\u002Fdeathau\u002Fmarkdownload With extension available for Firefox, Google Chrome, Microsoft Edge and Safari.",{"comments":38,"answered":39,"key":218},[219,222,223],{"id":220,"text":221},"40041459","you would still need to run. For js based websites.",{"id":69,"text":70},{"id":79,"text":80},{"commenters":225,"max_depth":226,"maker_replies":38,"first_comment_mins":227,"lifespan_hours":228},71,6,23,148,[230,232,235,238,242,244,247,250,253,256,257,259,263,265,269,272,275,277,281,283,286,288,291,294,297,301,304,306,308,313,315,318,320,323,326,328,331,333,336,339,342,344,346,348,351,354,356,358,359,362,365,368,371,374,377,380,383,384,387,390,393,394,399,403,407,410,414,417,421,423,427,430,434,436,440,444,446,449,452,455,458,460,464,466,471,473,475,479,481,483,485,488,490,492,496,500,502,504],[185,41,18,231,186,41],"maker\u002Fneutral\u002Fidea",[233,41,41,234,136,41],"40037017","other\u002Fneutral\u002Fnone",[236,41,41,234,237,41],"40037718","scary-size",[239,41,41,240,241,41],"40038016","commenter\u002Fneutral\u002Fnone","justech",[243,41,41,240,237,41],"40038118",[245,41,41,240,246,41],"40045130","rismay",[248,41,41,234,249,41],"40101653","Kikobeats",[251,41,41,234,252,41],"40038447","msp26",[254,41,41,234,255,41],"40041158","privatenumber",[220,41,41,240,11,39],[258,41,41,240,252,41],"40042259",[260,39,18,261,262,41],"40035144","project\u002Fneutral\u002Fpricing","selfie",[69,39,41,264,11,39],"project\u002Fpositive\u002Fperformance",[266,39,41,267,268,41],"40036969","other\u002Fneutral\u002Fpricing","compootr",[270,39,41,240,271,41],"40038520","aleksandern08",[273,39,41,267,274,41],"40041870","asadm",[192,33,38,276,193,41],"maker\u002Fpositive\u002Fidea",[278,33,39,279,280,41],"40035081","project\u002Fcritical\u002Fpricing","jph00",[282,33,41,267,193,41],"40035224",[284,33,41,234,285,41],"40050265","dctanner",[113,18,39,287,114,41],"project\u002Fcritical\u002Fexecution",[289,18,18,290,193,41],"40034962","project\u002Fneutral\u002Fexecution",[292,43,41,234,293,41],"40033720","cratermoon",[295,43,41,234,296,41],"40034751","fbdab103",[298,38,41,299,300,41],"40037027","other\u002Fneutral\u002Fidea","sabr",[302,38,41,234,303,41],"40048257","boatbuilder",[101,226,38,305,102,41],"maker\u002Fpositive\u002Fexecution",[79,226,41,307,11,39],"project\u002Fneutral\u002Fperformance",[309,310,39,311,312,41],"40033654",7,"project\u002Fcritical\u002Fidea","LeonidBugaev",[89,310,41,314,11,39],"project\u002Fneutral\u002Fidea",[316,310,41,234,317,41],"40033917","haddr",[319,310,41,234,193,41],"40035548",[321,310,18,314,322,41],"40034001","IanCal",[324,310,41,234,325,41],"40034103","chrisweekly",[327,310,41,240,322,41],"40034235",[329,310,41,234,330,41],"40034351","cactusfrog",[332,310,41,234,322,41],"40034426",[334,310,41,234,335,41],"40035184","genewitch",[337,310,41,234,338,41],"40034642","hombre_fatal",[340,310,18,290,341,41],"40034240","foundzen",[343,310,18,290,274,41],"40034356",[345,310,41,234,341,41],"40037108",[347,310,41,290,11,39],"40037365",[349,310,38,350,296,41],"40034694","project\u002Fpositive\u002Fexecution",[352,310,41,234,353,41],"40036071","nyokodo",[355,310,41,234,296,41],"40042741",[357,310,41,234,193,41],"40034977",[203,31,41,234,204,41],[360,31,41,234,361,41],"40034392","rubyn00bie",[363,31,41,234,364,41],"40034556","thangalin",[158,366,38,367,159,41],9,"project\u002Fpositive\u002Fidea",[369,366,41,234,370,41],"40033823","ElFitz",[372,366,41,234,373,41],"40034177","BHSPitMonkey",[375,366,41,234,376,41],"40036869","intelkishan",[378,366,41,234,379,41],"40033851","ZunarJ5",[381,366,41,234,382,41],"40033909","smartmic",[214,366,41,234,215,41],[385,366,41,234,386,41],"40034066","Davidbrcz",[388,366,41,234,389,41],"40034447","kwhitefoot",[391,366,41,234,392,41],"40034872","jwoq9118",[196,34,41,234,197,41],[395,396,41,397,398,41],"40034020",11,"commenter\u002Fneutral\u002Fidea","franciscop",[400,401,41,299,402,41],"40037269",12,"RamblingCTO",[404,405,41,299,406,41],"40035111",13,"areichert",[408,405,38,367,409,41],"40035154","nbbaier",[411,412,18,314,413,41],"40034472",14,"osener",[415,412,39,416,274,41],"40034488","project\u002Fcritical\u002Fperformance",[418,412,41,419,420,41],"40035103","other\u002Fneutral\u002Fperformance","ushakov",[143,32,18,422,144,41],"project\u002Fneutral\u002Fnone",[424,425,41,299,426,41],"40038225",16,"blacklight",[169,428,18,429,170,41],17,"project\u002Fmixed\u002Fexecution",[431,432,41,299,433,41],"40036878",18,"J_Shelby_J",[117,435,39,287,118,41],19,[437,438,41,299,439,41],"40034176",20,"rubymamis",[441,442,41,234,443,41],"40038526",21,"g105b",[128,445,39,416,129,41],22,[447,445,39,287,448,41],"40033941","Animats",[450,445,18,451,274,41],"40033996","maker\u002Fneutral\u002Fexecution",[453,445,18,454,274,41],"40034120","maker\u002Fneutral\u002Fperformance",[456,227,41,299,457,41],"40047620","luckman212",[147,459,18,422,148,41],24,[461,462,38,367,463,41],"40034963",25,"ben_ja_min",[154,465,18,314,155,41],26,[467,468,18,469,470,41],"40034102",27,"project\u002Fmixed\u002Fperformance","dazzaji",[472,468,18,454,274,41],"40034114",[474,468,38,264,470,41],"40048534",[476,477,38,367,478,41],"40057476",28,"cassidoo",[135,480,39,287,136,41],29,[482,480,18,290,148,41],"40037040",[210,484,41,299,211,41],30,[165,486,39,487,166,41],31,"project\u002Fcritical\u002Fdesign",[124,489,39,416,125,41],32,[491,489,18,454,274,41],"40034123",[493,494,41,234,495,41],"40038269",33,"FounderBurr",[497,498,41,299,499,41],"40034151",34,"katehikes88",[501,498,41,397,389,41],"40034459",[503,498,41,397,499,41],"40044002",[505,498,41,506,389,41],"40051370","commenter\u002Fcritical\u002Fidea",[],[509,514,519,524,529,534],{"id":510,"name":511,"points":512,"category":513},"40003710","OptiGap sensor",886,"hardware",{"id":515,"name":516,"points":517,"category":518},"40041198","gypsum",552,"learning_science",{"id":520,"name":521,"points":522,"category":523},"40017180","Mazelit",517,"games",{"id":525,"name":526,"points":527,"category":528},"40067677","Effort",419,"ai",{"id":530,"name":531,"points":532,"category":533},"40028111","index advisor",417,"data",{"id":535,"name":536,"points":537,"category":513},"40014937","Recovery Kit 2",403,false,"v2",{"id":541,"name":5,"line":6,"ai":538,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":542,"maker":11,"domain":10,"tier":19,"attempts":41,"strip":543,"url":9,"mix":544,"praise":97,"objection":110,"sub":21},40033490,[24,25,26,27],"3000305110030300350050000051310134343100",[31,32,33,34],1794036653121]