[{"data":1,"prerenderedAt":707},["ShallowReactive",2],{"project-47378781":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":47,"post_text":49,"what":50,"said":93,"maker_in_thread":204,"shape":212,"grid":217,"attempts":668,"same_week":669,"maker_shows":48,"maker_hidden":699,"reader_version":700,"card":701},"47378781","Hacker News archive (open-index)","Hacker News archive (47M+ items, 11.6GB) as Parquet, updated every 5m","title","Show HN: Hacker News archive (47M+ items, 11.6GB) as Parquet, updated every 5m","https:\u002F\u002Fhuggingface.co\u002Fdatasets\u002Fopen-index\u002Fhacker-news","huggingface.co","tamnd",1773508323,2026,408,167,81,4366,2,"legend","data","dataset archive","dataset",[24,25,26,27],"hacker news dataset","parquet","duckdb","hacker news archive","curious",-0.027,{"positive":31,"neutral":32,"mixed":33,"critical":34},9,34,0,10,53,{"pricing":37,"docs":38,"idea":40,"performance":41,"execution":42,"design":43,"privacy":45},{"positive":33,"critical":33},{"positive":33,"critical":39},4,{"positive":39,"critical":33},{"positive":33,"critical":33},{"positive":33,"critical":18},{"positive":44,"critical":33},1,{"positive":44,"critical":46},3,[44,33,46,48,46,33,33,18,46,48,33,33,33,39,46,46,33,44,46,46,39,48,39,44,33,44,46,46,33,33,33,33,44,33,33,48,44,46,44,33],5,null,{"problem":51,"audience":58,"pricing":49,"motivation":67,"time_to_build":49,"stack":71,"openness":90,"stage":91,"team_size":92},{"value":52,"source":53,"evidence":54},"Querying Hacker News data without running your own pipeline first","maker",[55],{"id":56,"by":11,"text":57},"47437282","[Author here] The whole pipeline runs on a single ~$10\u002Fmonth VPS, but it can process hundreds of TB even with just 12GB RAM and a 200GB SSD. The main reason I built this was to…",{"value":59,"source":60,"evidence":61},"Developers and data users who want easy, up-to-date HN data","inferred",[62,63],{"id":56,"by":11,"text":57},{"id":64,"by":65,"text":66},"47433345","jiggawatts","I could have used this just yesterday! I've been evaluating Gemini Embedding 2 using Hacker News comments and I wasted half a day making a wrapper for the HN API to collect some…",{"value":68,"source":53,"evidence":69},"Have HN data that is easy to query and always up to date",[70],{"id":56,"by":11,"text":57},[72,78,82,86],{"value":73,"source":53,"evidence":74},"Go",[75],{"id":76,"by":11,"text":77},"47437213","@keepamovin thanks, your project was a big inspiration for this. I built my own pipeline with a slightly different setup. I use Go to download and process the data, and update it…",{"value":79,"source":53,"evidence":80},"Parquet",[81],{"id":56,"by":11,"text":57},{"value":83,"source":53,"evidence":84},"Hugging Face",[85],{"id":56,"by":11,"text":57},{"value":87,"source":53,"evidence":88},"DuckDB",[89],{"id":56,"by":11,"text":57},"open_source","launched","unknown",{"praise":94,"objections":119,"requests":154,"questions":173,"alternatives":196},[95,111],{"value":96,"source":97,"evidence":98},"Useful dataset for projects and classroom use","commenter",[99,103,107],{"id":100,"by":101,"text":102},"47430209","Imustaskforhelp","As someone who had made a project analysing hackernews who had used clickhouse, I really feel like this is a project made for me (especially the updated every 5 minute aspect…",{"id":104,"by":105,"text":106},"47428547","Onavo","Is is possible to only download a subset? e.g. Show HNs or HN Whoishiring. The Show HNs and HN Whoishiring are very useful for classroom data science i.e. a very useful set of…",{"id":108,"by":109,"text":110},"47430553","imhoguy","Yay! So much knowledge in just 11GB. Adding to my end of the World hoarding stash!",{"value":112,"source":97,"evidence":113},"Frequent updates enable new ideas",[114,115],{"id":100,"by":101,"text":102},{"id":116,"by":117,"text":118},"47430798","politician","This is great. I've soured on this site over the past few years due to the heavy partisanship that wasn't as present in the early days (eternal September), but there are still…",[120,135,147],{"value":121,"source":97,"evidence":122},"Data licensing and terms of use unclear",[123,127,131],{"id":124,"by":125,"text":126},"47428618","bstsb","what’s the license? “do whatever the fuck you want with the data as long as you don’t get caught”? or does that only work for massive corporations",{"id":128,"by":129,"text":130},"47429940","tonymet","what's the license for HN content?",{"id":132,"by":133,"text":134},"47428809","GeoAtreides","is the legal page a placeholder, do words have no meaning? https:\u002F\u002Fwww.ycombinator.com\u002Flegal\u002F Mods, enforce your license terms, you're playing fast and loose with the law…",{"value":136,"source":97,"evidence":137},"No way to remove personal comments; privacy and GDPR concerns",[138,142,146],{"id":139,"by":140,"text":141},"47430411","robotswantdata","Where’s the opt out ?",{"id":143,"by":144,"text":145},"47431352","trwhite","Hello. I didn’t consent to any of my HN comments being used in this way. Please kindly remove them.",{"id":132,"by":133,"text":134},{"value":148,"source":97,"evidence":149},"Replacing the full file every 5 minutes is wasteful",[150],{"id":151,"by":152,"text":153},"47430512","vovavili","Replacing an 11.6GB Parquet file every 5 minutes strikes me as a bit wasteful. I would probably use Apache Iceberg here.",[155,162,169],{"value":156,"source":97,"evidence":157},"Release the update system source code",[158],{"id":159,"by":160,"text":161},"47430649","maxloh","Could you also release the source code behind the automatic update system?",{"value":163,"source":97,"evidence":164},"Upload to academic torrents",[165],{"id":166,"by":167,"text":168},"47429915","lyu07282","Please upload to https:\u002F\u002Facademictorrents.com\u002F as well if possible",{"value":170,"source":97,"evidence":171},"Subset download such as Show HN or Who is hiring",[172],{"id":104,"by":105,"text":106},[174,185,192],{"value":175,"source":97,"evidence":176},"Why update every 5 minutes and is the update current",[177,181],{"id":178,"by":179,"text":180},"47430036","kshacker","Good for demo but every 5 minutes? Why?",{"id":182,"by":183,"text":184},"47429284","mlhpdx","Static web content and dynamic data? > The archive currently spans from 2006-10 to 2026-03-16 23:55 UTC, with 47,358,772 items committed. That’s more than 5 minutes ago by a day…",{"value":186,"source":97,"evidence":187},"Why are deleted and dead stored as integers",[188],{"id":189,"by":190,"text":191},"47428932","0cf8612b2e1e","Under the Known Limitations section ``` deleted and dead are integers. They are stored as 0\u002F1 rather than booleans. ``` Is there a technical reason to do this? You have the type…",{"value":193,"source":97,"evidence":194},"Is the data license clear",[195],{"id":128,"by":129,"text":130},[197],{"value":198,"source":97,"evidence":199},"ClickHouse HN playground",[200],{"id":201,"by":202,"text":203},"47429019","xnx","The best source for this data used to be Clickhouse (https:\u002F\u002Fplay.clickhouse.com\u002Fplay?user=play#U0VMRUNUIG1heCh0aW1lKSBGUk9NIGhhY2tlcm5ld3NfaGlzdG9yeQ==), but it hasn't updated…",{"comments":31,"answered":205,"key":206},6,[207,208,211],{"id":76,"text":77},{"id":209,"text":210},"47439565","Connecting directly with the author of the project that inspired me is awesome. Let's collaborate and see how we can make our two projects work together. DuckDB has a feature…",{"id":56,"text":57},{"commenters":213,"max_depth":214,"maker_replies":31,"first_comment_mins":215,"lifespan_hours":216},106,8,5770,151,[218,222,226,229,231,233,235,238,241,244,248,251,253,255,256,258,260,263,266,268,270,274,276,278,281,283,285,287,290,292,294,297,300,303,306,308,311,314,316,317,320,321,324,326,328,332,335,338,341,344,346,349,352,354,357,358,359,362,366,370,373,376,381,383,386,388,391,393,397,400,403,405,409,411,414,417,420,424,426,429,432,434,437,440,443,447,451,454,457,459,461,463,465,469,471,474,476,478,480,482,485,488,491,493,495,498,500,503,505,507,510,512,515,518,521,523,526,528,531,533,536,540,543,546,549,552,555,557,559,561,563,565,567,569,571,574,576,579,583,586,589,592,594,596,599,602,605,608,611,613,616,619,622,625,627,630,633,635,637,641,644,646,649,651,654,657,660,662,665],[219,33,44,220,221,33],"47432774","project\u002Fcritical\u002Fnone","6thbit",[223,33,33,224,225,33],"47433483","commenter\u002Fneutral\u002Fnone","Wowfunhappy",[227,33,33,224,228,33],"47433790","stackghost",[230,33,33,224,225,33],"47433897",[232,33,33,224,228,33],"47433921",[234,33,33,224,225,33],"47434161",[236,33,33,224,237,33],"47434607","wzdd",[239,33,33,224,240,33],"47435865","hnfong",[242,33,33,224,243,33],"47443559","YVoyiatzis",[245,33,46,246,247,33],"47434195","project\u002Fneutral\u002Fnone","NewJazz",[249,33,46,246,250,33],"47435123","keepamovin",[76,33,33,252,11,44],"maker\u002Fpositive\u002Fnone",[254,33,48,252,250,33],"47439342",[209,33,33,252,11,44],[257,33,33,224,250,33],"47439768",[259,33,48,252,250,33],"47449898",[261,33,46,246,262,33],"47435823","admiralrohan",[264,33,46,246,265,33],"47437453","krapp",[267,33,46,246,160,33],"47462582",[56,44,33,269,11,44],"maker\u002Fneutral\u002Fexecution",[271,44,46,272,273,33],"47439225","maker\u002Fneutral\u002Fpricing","asalahli",[275,44,33,272,11,44],"47439435",[64,18,33,277,65,33],"other\u002Fneutral\u002Fnone",[279,18,33,277,280,33],"47433791","pjot",[282,18,33,277,65,33],"47434073",[284,18,33,224,280,33],"47434336",[286,18,33,277,65,33],"47434455",[288,18,33,277,289,33],"47434810","vienneraphael",[291,18,33,224,65,33],"47437731",[189,46,46,293,190,33],"project\u002Fneutral\u002Fdocs",[295,46,33,224,296,33],"47432054","albedoa",[298,46,44,299,190,33],"47432476","project\u002Fcritical\u002Fdocs",[301,46,46,293,302,33],"47432605","mulmen",[304,46,33,224,305,33],"47432721","sillysaurusx",[307,46,46,293,190,33],"47432747",[309,46,44,299,310,33],"47432506","endofreach",[312,46,33,313,296,33],"47441742","commenter\u002Fneutral\u002Fdocs",[100,39,48,315,101,33],"project\u002Fpositive\u002Fidea",[201,48,46,246,202,33],[318,48,33,277,319,33],"47431793","mceoin",[116,205,48,315,117,33],[322,205,33,277,323,33],"47433312","neom",[325,205,33,224,117,33],"47434528",[327,205,33,224,323,33],"47435002",[329,330,33,277,331,33],"47430504",7,"brtkwr",[333,330,33,277,334,33],"47430519","tantalor",[336,330,33,277,337,33],"47430907","ericfr11",[339,330,33,277,340,33],"47431453","christophilus",[342,330,33,277,343,33],"47431682","culi",[345,330,33,277,343,33],"47431684",[347,330,33,277,348,33],"47432219","liamwire",[350,330,33,277,351,33],"47432839","BloondAndDoom",[353,330,33,277,310,33],"47433996",[355,330,33,277,356,33],"47431692","nostrapollo",[166,214,46,246,167,33],[104,31,46,246,105,33],[360,31,46,246,361,33],"47428590","nelsondev",[363,34,48,364,365,33],"47449893","project\u002Fpositive\u002Fnone","dacapoday",[367,368,46,246,369,33],"47428709",11,"palmotea",[371,368,33,277,372,33],"47429933","BoredPositron",[374,368,46,375,101,33],"47430245","project\u002Fneutral\u002Fidea",[377,378,46,379,380,33],"47428939",12,"project\u002Fneutral\u002Fperformance","gkbrk",[382,378,46,379,202,33],"47429032",[384,378,46,379,385,33],"47429123","hirako2000",[387,378,46,379,190,33],"47429062",[389,378,33,277,390,33],"47431577","boznz",[392,378,33,224,302,33],"47432358",[394,395,33,277,396,33],"47429858",13,"alstonite",[398,395,33,277,399,33],"47429923","ghgr",[401,395,33,277,402,33],"47430035","tehjoker",[404,395,33,277,109,33],"47430472",[406,407,46,246,408,33],"47430549",14,"epogrebnyak",[410,407,33,224,408,33],"47430559",[412,407,33,224,413,33],"47431904","estimator7292",[151,415,44,416,152,33],15,"project\u002Fcritical\u002Fexecution",[418,415,44,416,419,33],"47430548","fabmilo",[421,415,46,422,423,33],"47430631","project\u002Fneutral\u002Fexecution","ai-inquisitor",[425,415,46,375,152,33],"47430738",[427,415,33,277,428,33],"47430751","tomrod",[430,415,33,224,431,33],"47430971","cyanydeez",[433,415,33,224,302,33],"47432304",[435,415,33,436,11,44],"47437294","maker\u002Fneutral\u002Fidea",[438,415,46,422,439,33],"47430878","btown",[441,415,33,224,442,33],"47431061","roncesvalles",[444,415,33,445,446,33],"47431221","commenter\u002Fneutral\u002Fexecution","enchilada",[448,415,46,449,450,33],"47437058","project\u002Fneutral\u002Fdesign","sureglymop",[452,415,46,449,453,33],"47430642","zerocrates",[455,415,48,456,428,33],"47430761","project\u002Fpositive\u002Fdesign",[178,458,46,375,179,33],16,[460,458,48,315,101,33],"47430224",[462,458,33,436,11,44],"47437338",[464,458,33,436,11,44],"47437310",[466,467,46,422,468,33],"47432444",17,"lhoestq",[108,470,48,315,109,33],18,[472,470,33,277,473,33],"47431646","sockaddr",[475,470,33,277,109,33],"47454270",[159,477,46,422,160,33],19,[479,477,33,269,11,44],"47437328",[182,481,44,299,183,33],20,[483,481,33,224,484,33],"47430213","xandrius",[486,481,33,224,487,33],"47430304","john_strinlai",[489,481,33,224,490,33],"47430349","voxic11",[492,481,33,313,490,33],"47430344",[494,481,33,224,487,33],"47430377",[496,481,33,224,497,33],"47430948","fatty_patty89",[499,481,33,224,487,33],"47431291",[501,481,33,224,502,33],"47432365","mpalmer",[504,481,33,224,487,33],"47432666",[506,481,44,299,183,33],"47434850",[508,481,33,509,11,44],"47437344","maker\u002Fneutral\u002Fdocs",[128,511,46,293,129,33],21,[513,511,33,277,514,33],"47430034","echelon",[516,511,33,277,517,33],"47430106","marginalia_nu",[519,511,33,277,520,33],"47430942","BowBun",[522,511,33,224,129,33],"47432607",[139,524,46,525,140,33],22,"project\u002Fneutral\u002Fprivacy",[527,524,46,525,487,33],"47430435",[529,524,33,224,530,33],"47432140","lofaszvanitt",[532,524,33,277,334,33],"47430526",[534,524,33,277,535,33],"47430612","ratg13",[537,524,33,538,539,33],"47430703","other\u002Fneutral\u002Fprivacy","fdghrtbrt",[541,524,33,277,542,33],"47430809","socksy",[544,524,33,277,545,33],"47431963","Diederich",[547,524,33,277,548,33],"47432288","adi_kurian",[550,524,33,277,551,33],"47438479","selfhoster11",[553,524,33,538,554,33],"47431107","coppsilgold",[556,524,33,277,539,33],"47431171",[558,524,33,538,133,33],"47431293",[560,524,33,277,343,33],"47431706",[562,524,33,277,343,33],"47431702",[564,524,33,277,133,33],"47431278",[566,524,33,538,520,33],"47430956",[568,524,33,538,140,33],"47431047",[570,524,33,538,530,33],"47432136",[572,524,33,224,573,33],"47432410","pkilgore",[575,524,33,224,520,33],"47445026",[132,577,44,578,133,33],23,"project\u002Fcritical\u002Fprivacy",[580,577,33,581,582,33],"47428931","other\u002Fcritical\u002Fprivacy","andrewmcwatters",[584,577,33,538,585,33],"47428943","Retr0id",[587,577,33,538,588,33],"47429083","ungruntled",[590,577,33,591,133,33],"47429108","commenter\u002Fneutral\u002Fprivacy",[593,577,33,591,588,33],"47429186",[595,577,33,538,133,33],"47429096",[597,577,33,538,598,33],"47429603","ryandvm",[600,577,33,591,601,33],"47429738","jmalicki",[603,577,33,591,604,33],"47429992","AndrewKemendo",[606,577,33,591,607,33],"47429816","zamadatix",[609,577,33,591,610,33],"47431011","Bewelge",[612,577,33,591,607,33],"47437813",[614,577,46,525,615,33],"47429057","hsuduebc2",[617,577,48,618,598,33],"47429572","project\u002Fpositive\u002Fprivacy",[620,577,33,591,621,33],"47429839","Ylpertnodi",[623,577,33,591,624,33],"47429931","kmeisthax",[626,577,33,591,514,33],"47430049",[628,577,33,591,629,33],"47430101","petercooper",[631,577,44,578,632,33],"47431290","hrmtst93837",[124,634,46,525,125,33],24,[636,634,46,525,372,33],"47430001",[638,639,33,538,640,33],"47429958",25,"lokimoon",[642,639,33,277,643,33],"47430507","waynesonfire",[143,645,44,578,144,33],26,[647,645,33,591,648,33],"47431392","RIMR",[650,645,33,591,144,33],"47431437",[652,645,33,538,653,33],"47431440","Kye",[655,645,33,538,656,33],"47431450","s0ss",[658,645,33,538,659,33],"47431455","cj",[661,645,33,538,144,33],"47431473",[663,645,33,277,664,33],"47432207","owyn",[666,645,33,591,667,33],"47431414","nextaccountic",[],[670,675,680,685,689,694],{"id":671,"name":672,"points":673,"category":674},"47344548","s@",414,"languages",{"id":676,"name":677,"points":678,"category":679},"47427299","SVG backgrounds",403,"creative",{"id":681,"name":682,"points":683,"category":684},"47400868","Godogen",337,"dev_tools",{"id":686,"name":687,"points":688,"category":684},"47412569","zeroboot",311,{"id":690,"name":691,"points":692,"category":693},"47428650","Will my flight have Starlink?",276,"consumer_social",{"id":695,"name":696,"points":697,"category":698},"47431671","llm-circuit-finder",265,"ai",false,"v2",{"id":702,"name":5,"line":6,"ai":699,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":703,"top":18,"kw":704,"maker":11,"domain":10,"tier":19,"attempts":33,"strip":705,"url":9,"mix":706,"praise":96,"objection":121,"sub":21},47378781,-0.03,[24,25,26,27],"1035300235000433013345410133000010051310",[31,32,33,34],1794036696152]