[{"data":1,"prerenderedAt":214},["ShallowReactive",2],{"project-31867179":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":41,"post_text":42,"what":43,"said":65,"maker_in_thread":130,"shape":133,"grid":137,"attempts":170,"same_week":175,"maker_shows":205,"maker_hidden":206,"reader_version":207,"card":208},"31867179","warc-parquet","Easily Convert WARC (Web Archive) into Parquet, Then Query with DuckDB","title","Show HN: Easily Convert WARC (Web Archive) into Parquet, Then Query with DuckDB","https:\u002F\u002Fgithub.com\u002Fmaxcountryman\u002Fwarc-parquet","github.com","llambda",1656095206,2022,116,15,535,2570,21,"notable","data","web archive conversion","cli",[24,25,26,27],"warc","parquet","duckdb","web archive","mixed",0.259,{"positive":31,"neutral":32,"mixed":33,"critical":34},5,3,2,1,11,{"idea":37,"performance":39,"design":40},{"positive":34,"critical":38},0,{"positive":32,"critical":34},{"positive":34,"critical":38},[32,38,38,38,38,31,32,32,32,32,31,31,31,34,31],null,{"problem":44,"audience":42,"pricing":42,"motivation":42,"time_to_build":42,"stack":53,"openness":64,"stage":64,"team_size":64},{"value":45,"source":46,"evidence":47},"Querying WARC web archive files is hard; converting them to Parquet enables columnar querying with DuckDB","inferred",[48,50],{"id":49},"post",{"id":51,"by":11,"text":52},"31867489","It's a great question: fundamentally the Parquet format offers columnar orientation. With datasets like these, there's some research[0] indicating this is a preferable way of…",[54,60],{"value":55,"source":56,"evidence":57},"DuckDB","maker",[58,59],{"id":49},{"id":51,"by":11,"text":52},{"value":61,"source":56,"evidence":62},"Parquet",[63],{"id":51,"by":11,"text":52},"unknown",{"praise":66,"objections":91,"requests":101,"questions":106,"alternatives":114},[67,83],{"value":68,"source":69,"evidence":70},"DuckDB is much faster than SQLite for analytical queries on columnar data","commenter",[71,75,79],{"id":72,"by":73,"text":74},"31868562","1egg0myegg0","Good question! As a disclaimer, I work for DuckDB Labs. There are 2 big benefits to working with Parquet files in DuckDB, and both relate to speed! DuckDB can query parquet right…",{"id":76,"by":77,"text":78},"31868650","wenc","DuckDB has SQLite semantics but is natively built around columnar formats (parquet, in-memory Arrow) and strong types (including dates). It also supports very complex SQL. SQLite…",{"id":80,"by":81,"text":82},"31873335","henrydark","I've gone almost completely off Athena in favor of duckdb. Duckdb running with 12 vcpus on mid-size data (1tb) that's on ssd is routinely faster than Athena with the data on s3.…",{"value":84,"source":69,"evidence":85},"Parquet's columnar format and compression are well suited to this data",[86,90],{"id":87,"by":88,"text":89},"31871505","pmeira","Another benefit that is often overlooked is that Parquet support various compression methods, right? SQLite's official support for compression is a commercial product, so…",{"id":51,"by":11,"text":52},[92],{"value":93,"source":69,"evidence":94},"DuckDB lacks distributed computation, so large datasets need a separate setup",[95,98],{"id":96,"by":77,"text":97},"31869980","DuckDB user here. As far as I can tell, DuckDB doesn’t support distributed computation so you have to set that up yourself, whereas Athena is essentially Presto — it handles that…",{"id":99,"by":77,"text":100},"31876208","In the distributed computing world, the rule is you start to scale horizontally when your compute workload is too large to fit in the memory of a single machine. So it depends on…",[102],{"value":103,"source":69,"evidence":104},"GeoParquet support in DuckDB",[105],{"id":87,"by":88,"text":89},[107],{"value":108,"source":69,"evidence":109},"How does this compare with SQLite approaches?",[110],{"id":111,"by":112,"text":113},"31867335","wahnfrieden","How does this compare with SQLite approaches shared recently?",[115,122],{"value":116,"source":69,"evidence":117},"SQLite with sqlite-parquet-vtable",[118],{"id":119,"by":120,"text":121},"31868485","infogulch","Well there's a virtual table extension to read parquet files in SQLite. I've not tried it myself. https:\u002F\u002Fgithub.com\u002Fcldellow\u002Fsqlite-parquet-vtable",{"value":123,"source":69,"evidence":124},"AWS Athena",[125,129],{"id":126,"by":127,"text":128},"31869779","arpinum","Do you see DuckDB as a possible replacement for AWS Athena? Where would Athena still be better than DuckDB + Parquet + Lambda?",{"id":80,"by":81,"text":82},{"comments":34,"answered":34,"key":131},[132],{"id":51,"text":52},{"commenters":134,"max_depth":135,"maker_replies":34,"first_comment_mins":136,"lifespan_hours":18},12,6,9,[138,140,142,144,147,149,151,152,154,158,159,160,161,162,166],[111,38,32,139,112,38],"project\u002Fneutral\u002Fidea",[51,38,38,141,11,34],"project\u002Fpositive\u002Fdesign",[119,38,38,143,120,38],"other\u002Fneutral\u002Fnone",[145,38,38,143,146,38],"31870328","westurner",[148,38,38,143,120,38],"31872142",[72,38,31,150,73,38],"project\u002Fpositive\u002Fperformance",[126,38,32,139,127,38],[96,38,32,153,77,38],"project\u002Fmixed\u002Fperformance",[155,38,32,156,157,38],"31874306","project\u002Fneutral\u002Fperformance","youngtaff",[99,38,32,153,77,38],[80,38,31,150,81,38],[87,38,31,141,88,38],[76,38,31,150,77,38],[163,38,34,164,165,38],"31871775","project\u002Fcritical\u002Fperformance","1vuio0pswjnm7",[167,34,31,168,169,38],"31868244","project\u002Fpositive\u002Fidea","mritchie712",[171,174],{"id":172,"t":173,"points":32,"comments":38},"31824150",1655820634,{"id":4,"t":12,"points":14,"comments":15},[176,181,186,191,196,201],{"id":177,"name":178,"points":179,"category":180},"31903076","min-dalle",663,"ai",{"id":182,"name":183,"points":184,"category":185},"31925871","TAXSIM.app",497,"finance_crypto",{"id":187,"name":188,"points":189,"category":190},"31897430","PRQL 0.2",378,"languages",{"id":192,"name":193,"points":194,"category":195},"31910622","Legible News",320,"productivity",{"id":197,"name":198,"points":199,"category":200},"31921830","Brevity 500",193,"learning_science",{"id":202,"name":203,"points":204,"category":195},"31891608","Yboard",165,4,false,"v2",{"id":209,"name":5,"line":6,"ai":206,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":210,"top":18,"kw":211,"maker":11,"domain":10,"tier":19,"attempts":33,"strip":212,"url":9,"mix":213,"praise":68,"objection":93,"sub":21},31867179,0.26,[24,25,26,27],"300005333355515",[31,32,33,34],1793991047753]