[{"data":1,"prerenderedAt":147},["ShallowReactive",2],{"project-21195062":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":31,"aspects":32,"strip":33,"post_text":29,"what":34,"said":51,"maker_in_thread":83,"shape":89,"grid":93,"attempts":109,"same_week":110,"maker_shows":91,"maker_hidden":140,"reader_version":141,"card":142},"21195062","fsdup","Deduplicating NTFS file systems (fsdup)","title","Show HN: Deduplicating NTFS file systems (fsdup)","https:\u002F\u002Fgithub.com\u002Fbinwiederhier\u002Ffsdup","github.com","binwiederhier",1570557506,2019,3,6,2120,2195,97,"shown","infrastructure","backup deduplication","cli",[24,25,26,27],"deduplication","ntfs","backup storage","block store","quiet",null,{"positive":31,"neutral":31,"mixed":31,"critical":31},0,{},[31,31,31,31,31,31],{"problem":35,"audience":29,"pricing":29,"motivation":45,"time_to_build":29,"stack":49,"openness":50,"stage":50,"team_size":50},{"value":36,"source":37,"evidence":38},"Storage cost of many duplicate blocks across large NTFS backup sets.","maker",[39,42],{"id":40,"by":11,"text":41},"21197982","The idea here is to do this on a very very very large scale. My employer has a > 500 PB cloud of (mainly Windows) backups. Right now everything is on ZFS, but we're thinking…",{"id":43,"by":11,"text":44},"21195088","There's a write-up here with some details on the why\u002Fhow and a few experiments (deduping >56T in 125 raw disks):…",{"value":46,"source":37,"evidence":47},"Maker's employer has over 500 PB of mostly Windows backups on ZFS and is considering moving to a block store.",[48],{"id":40,"by":11,"text":41},[],"unknown",{"praise":52,"objections":61,"requests":68,"questions":69,"alternatives":74},[53],{"value":54,"source":55,"evidence":56},"Interesting idea and a good write-up.","commenter",[57],{"id":58,"by":59,"text":60},"21197878","tiernano","nice write up. interesting idea too... very cool... one question though: I wonder what ZFS + DeDup or Windows Storage Spaces + DeDup would do with the same data, just out of…",[62],{"value":63,"source":37,"evidence":64},"Block-level dedup on NTFS gives better ratios but costs much more metadata.",[65],{"id":66,"by":11,"text":67},"21198329","That said, block level dedup on NTFS will lead to better dedup ratios, but at the cost of a lot more metadata and a very high metadata to data ratio.",[],[70],{"value":71,"source":55,"evidence":72},"How would ZFS dedup or Windows Storage Spaces dedup handle the same data?",[73],{"id":58,"by":59,"text":60},[75,79],{"value":76,"source":55,"evidence":77},"ZFS deduplication",[78],{"id":58,"by":59,"text":60},{"value":80,"source":55,"evidence":81},"Windows Storage Spaces deduplication",[82],{"id":58,"by":59,"text":60},{"comments":84,"answered":31,"key":85},4,[86,87,88],{"id":43,"text":44},{"id":40,"text":41},{"id":66,"text":67},{"commenters":90,"max_depth":91,"maker_replies":84,"first_comment_mins":90,"lifespan_hours":92},2,5,15,[94,97,99,101,103,106],[43,31,31,95,11,96],"project\u002Fpositive\u002Fdocs",1,[58,31,31,98,59,31],"commenter\u002Fneutral\u002Fnone",[40,31,31,100,11,96],"project\u002Fneutral\u002Fidea",[66,31,31,102,11,96],"project\u002Fneutral\u002Fperformance",[104,31,31,105,59,31],"21201382","commenter\u002Fneutral\u002Fidea",[107,31,31,108,11,96],"21197995","other\u002Fneutral\u002Fnone",[],[111,116,121,126,131,136],{"id":112,"name":113,"points":114,"category":115},"21175492","FeastGenius",422,"productivity",{"id":117,"name":118,"points":119,"category":120},"21157463","Imagededup",290,"dev_tools",{"id":122,"name":123,"points":124,"category":125},"21207132","uPlot.js",260,"data",{"id":127,"name":128,"points":129,"category":130},"21157066","Able",248,"web_saas",{"id":132,"name":133,"points":134,"category":135},"21147058","Scale Heaven",231,"creative",{"id":137,"name":138,"points":139,"category":115},"21179190","spotify-tui",212,false,"v2",{"id":143,"name":5,"line":6,"ai":140,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":29,"top":18,"kw":144,"maker":11,"domain":10,"tier":19,"attempts":31,"strip":145,"url":9,"mix":146,"praise":54,"objection":63,"sub":21},21195062,[24,25,26,27],"000000",[31,31,31,31],1794036581722]