[{"data":1,"prerenderedAt":516},["ShallowReactive",2],{"project-47104667":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":36,"aspects":37,"strip":47,"post_text":49,"what":50,"said":79,"maker_in_thread":165,"shape":177,"grid":182,"attempts":478,"same_week":479,"maker_shows":42,"maker_hidden":508,"reader_version":509,"card":510},"47104667","ntransformer","Llama 3.1 70B on a single RTX 3090 via NVMe-to-GPU bypassing the CPU","title","Show HN: Llama 3.1 70B on a single RTX 3090 via NVMe-to-GPU bypassing the CPU","https:\u002F\u002Fgithub.com\u002Fxaskasdf\u002Fntransformer","github.com","xaskasdf",1771707450,2026,395,101,89,4366,3,"legend","ai","local llm inference","library",[24,25,26,27,28],"llm inference","nvme offload","gpudirect storage","consumer gpu","llama 70b","curious",-0.015,{"positive":32,"neutral":33,"mixed":34,"critical":35},7,28,0,10,45,{"idea":38,"performance":41,"pricing":44,"execution":45,"docs":46},{"positive":39,"critical":40},4,1,{"positive":42,"critical":43},2,8,{"positive":34,"critical":40},{"positive":40,"critical":34},{"positive":34,"critical":34},[18,18,34,18,40,18,34,34,34,40,18,18,40,48,18,18,18,18,34,18,34,18,18,18,18,18,18,34,18,40,18,48,18,40,34,34,18,18,48,34],5,"Hi everyone, I'm kinda involved in some retrogaming and with some experiments I ran into the following question: \"It would be possible to run transformer models bypassing the cpu\u002Fram, connecting the gpu to the nvme?\"\n\nThis is the result of that question itself and some weekend vibecoding (it has the linked library repository in the readme as well), it seems to work, even on consumer gpus, it should work better on professional ones tho",{"problem":51,"audience":57,"pricing":57,"motivation":58,"time_to_build":57,"stack":62,"openness":76,"stage":77,"team_size":78},{"value":52,"source":53,"evidence":54},"Running large transformer models on a single consumer GPU with limited VRAM","maker",[55],{"id":56},"post",null,{"value":59,"source":53,"evidence":60},"Curiosity from retrogaming experiments asking whether the GPU could read from NVMe bypassing CPU and RAM",[61],{"id":56},[63,71],{"value":64,"source":65,"evidence":66},"CUDA","inferred",[67],{"id":68,"by":69,"text":70},"47108350","turingsroot","This is a really impressive piece of systems engineering. The 3-tier adaptive caching (VRAM resident > pinned RAM > NVMe\u002Fmmap) is essentially reimplementing what the Linux…",{"value":72,"source":73,"evidence":74},"C++ with custom GEMM kernels","commenter",[75],{"id":68,"by":69,"text":70},"open_source","prototype","solo",{"praise":80,"objections":92,"requests":116,"questions":136,"alternatives":148},[81,85],{"value":82,"source":73,"evidence":83},"Impressive systems engineering with tiered caching",[84],{"id":68,"by":69,"text":70},{"value":86,"source":73,"evidence":87},"Clever bypass of CPU for NVMe-to-GPU transfer",[88],{"id":89,"by":90,"text":91},"47109998","MarcLore","Bypassing CPU for NVMe-to-GPU transfer is clever. The bottleneck for running large models locally has always been the memory hierarchy — this essentially treats NVMe as extended…",[93,104,109],{"value":94,"source":73,"evidence":95},"Around 0.2 to 0.5 tokens per second is too slow for interactive use",[96,100],{"id":97,"by":98,"text":99},"47105824","randomtoast","0.2 tok\u002Fs is fine for experimentation, but it is not interactive in any meaningful sense. For many use cases, a well-quantized 8B or 13B that stays resident will simply deliver a…",{"id":101,"by":102,"text":103},"47110365","7777777phil","Cool hack but 0.5 tok\u002Fs on 70B when a 7B does 30+ on the same card. NVIDIA's own research says 40-70% of agentic tasks could run on sub-10B models and the quality gap has closed…",{"value":105,"source":73,"evidence":106},"Smaller quantized models stay resident and perform better",[107,108],{"id":97,"by":98,"text":99},{"id":101,"by":102,"text":103},{"value":110,"source":73,"evidence":111},"Cost per token is poor compared with hosted APIs",[112],{"id":113,"by":114,"text":115},"47108385","esquire_900","Cost wise it does not seem very effective. .5 token \u002F sec (the optimized one) is 3600 tokens an hour, which costs about 200-300 watts for an active 3090+system. Running 3600…",[117,125,129],{"value":118,"source":73,"evidence":119},"Benchmarks on Gen5 NVMe or faster PCIe links",[120,121],{"id":68,"by":69,"text":70},{"id":122,"by":123,"text":124},"47110592","davideom0414","Really interesting experiment i should have done this before Do you have numbers on effective throughput vs PCIe theoretical bandwidth? I’m curious whether this is primarily…",{"value":126,"source":73,"evidence":127},"Comparison with Apple unified memory M-series",[128],{"id":89,"by":90,"text":91},{"value":130,"source":73,"evidence":131},"More details on the DKMS patching process for consumer GPUs",[132],{"id":133,"by":134,"text":135},"47108644","Aurornis","Cool project. Can you provide more details about your DKMS patching process for consumer GPUs? This would be fun to try out, but I’d need some more details on that patch process…",[137,141],{"value":138,"source":73,"evidence":139},"Is throughput latency-bound or bandwidth-bound",[140],{"id":122,"by":123,"text":124},{"value":142,"source":73,"evidence":143},"Can AMD GPUs do direct IO like this",[144],{"id":145,"by":146,"text":147},"47110191","stuaxo","Interesting. Can AMD GPUs do direct io like this?",[149,160],{"value":150,"source":73,"evidence":151},"llama.cpp with CPU and GPU layer offload",[152,156],{"id":153,"by":154,"text":155},"47107992","fluoridation","That's slower than just running it off CPU+GPU. I can easily hit 1.5 tokens\u002Fs on a 7950X+3090 and a 20480-token context.",{"id":157,"by":158,"text":159},"47127405","rao-v","So llama.cpp currently statically puts overflow MoE experts in RAM and inferences them on CPU, so you get a mix of GPU + CPU inferencing. You are rooflined by RAM->CPU bandwidth…",{"value":161,"source":73,"evidence":162},"Running smaller 7B to 8B models fully in VRAM",[163,164],{"id":101,"by":102,"text":103},{"id":97,"by":98,"text":99},{"comments":166,"answered":32,"key":167},17,[168,171,174],{"id":169,"text":170},"47111593","Ya know, here on the local market there are a bunch of optanes hanging around, I'll try to manage one to check if there's any improvement",{"id":172,"text":173},"47114152","yeah, the mobo upgrade is something I gotta do anyway, so I'll cover that up more or less, the optane is something I didn't thought about",{"id":175,"text":176},"47107112","yeah, actually, I'm bottlenecked af since my mobo got pcie3 only :(",{"commenters":178,"max_depth":179,"maker_replies":166,"first_comment_mins":180,"lifespan_hours":181},72,6,120,51,[183,187,190,194,196,200,202,205,209,212,214,217,221,224,227,230,233,235,238,241,242,245,247,251,254,257,259,263,267,269,272,274,277,280,281,284,286,290,293,297,300,303,305,307,309,311,313,316,318,320,324,325,328,330,332,334,335,337,340,342,345,347,350,352,355,358,360,364,367,371,373,376,378,382,385,388,390,394,398,400,402,404,407,409,412,416,419,422,424,428,431,434,437,440,443,446,449,451,454,457,459,463,466,469,472,475],[184,34,18,185,186,34],"47107549","project\u002Fneutral\u002Fidea","01100011",[188,34,18,185,189,34],"47107754","javchz",[191,34,34,192,193,34],"47107979","other\u002Fneutral\u002Fnone","TechSquidTV",[169,34,34,195,11,40],"maker\u002Fneutral\u002Fexecution",[197,34,18,198,199,34],"47113210","project\u002Fneutral\u002Fperformance","jonassm",[172,34,34,201,11,40],"maker\u002Fneutral\u002Fperformance",[203,34,34,192,204,34],"47108402","ElectricalUnion",[206,34,48,207,208,34],"47109102","project\u002Fpositive\u002Fidea","lmeyerov",[210,34,34,192,211,34],"47112850","bhewes",[97,40,40,213,98,34],"project\u002Fcritical\u002Fperformance",[215,40,40,213,216,34],"47106193","tyfon",[218,40,34,219,220,34],"47106464","other\u002Fneutral\u002Fperformance","tgrowazay",[222,40,40,213,223,34],"47106575","vlovich123",[225,40,18,198,226,34],"47106600","someguy2026",[228,40,34,229,220,34],"47127899","commenter\u002Fneutral\u002Fperformance",[231,40,34,229,232,34],"47106680","zozbot234",[234,40,34,229,220,34],"47127835",[236,40,34,229,237,34],"47106836","uf00lme",[239,40,34,229,240,34],"47107352","wtallis",[175,40,34,201,11,40],[243,40,34,192,244,34],"47106449","Wuzado",[246,40,34,185,11,40],"47107002",[248,40,34,249,250,34],"47107152","commenter\u002Fneutral\u002Fnone","derstander",[252,40,18,198,253,34],"47108082","anoncow",[255,40,40,213,256,34],"47108321","fc417fc802",[258,40,34,195,11,40],"47111944",[260,40,18,261,262,34],"47108562","maker\u002Fneutral\u002Fpricing","eleventyseven",[264,40,34,265,266,34],"47108575","other\u002Fneutral\u002Fpricing","superkuh",[268,40,34,195,11,40],"47111512",[270,40,18,195,271,34],"47112779","green-salt",[273,40,18,195,199,34],"47112889",[275,40,34,276,11,40],"47114159","project\u002Fpositive\u002Fperformance",[278,40,34,192,279,34],"47130947","SmithRenaldo",[153,40,40,213,154,34],[282,42,48,207,283,34],"47106326","rl3",[285,42,34,201,11,40],"47107030",[287,18,34,288,289,34],"47107188","other\u002Fneutral\u002Fidea","jacquesm",[291,18,34,288,292,34],"47108707","serendip-ml",[294,39,18,295,296,34],"47108485","maker\u002Fneutral\u002Fnone","civicsquid",[298,39,34,192,299,34],"47108583","rustyhancock",[301,39,18,185,302,34],"47108875","charcircuit",[304,39,34,295,11,40],"47111342",[306,39,34,249,299,34],"47112206",[308,39,34,185,11,40],"47111330",[310,48,18,185,244,34],"47106523",[312,48,18,185,158,34],"47106558",[314,48,34,249,315,34],"47106631","hedgehog",[317,48,34,192,158,34],"47127244",[319,48,34,192,315,34],"47128538",[321,48,34,322,323,34],"47106640","commenter\u002Fneutral\u002Fidea","svnt",[157,48,34,219,158,34],[326,48,18,198,327,34],"47106666","reitzensteinm",[329,48,34,219,232,34],"47106708",[331,48,34,219,327,34],"47106828",[333,48,34,288,158,34],"47113537",[122,179,18,198,123,34],[336,179,34,201,11,40],"47111553",[338,32,18,185,339,34],"47106010","throwaway2027",[341,32,18,185,226,34],"47106269",[343,32,34,344,11,40],"47107845","maker\u002Fneutral\u002Fidea",[346,32,18,185,199,34],"47109603",[348,43,48,276,349,34],"47106222","jauntywundrkind",[101,351,40,213,102,34],9,[353,351,34,249,354,34],"47110458","valianteffort",[356,351,34,249,357,34],"47111078","tclancy",[133,35,18,359,134,34],"project\u002Fneutral\u002Fdocs",[361,35,18,362,363,34],"47109499","project\u002Fneutral\u002Fnone","Maxious",[365,35,34,366,11,40],"47111389","maker\u002Fneutral\u002Fdocs",[368,369,18,198,370,34],"47109848",11,"spwa4",[372,369,34,201,11,40],"47111381",[374,369,18,198,375,34],"47111557","nathan_compton",[377,369,34,249,370,34],"47119246",[379,380,34,192,381,34],"47106827",12,"exabrial",[383,380,34,192,384,34],"47110594","garethsprice",[386,380,34,387,381,34],"47111643","other\u002Fpositive\u002Fnone",[145,389,18,362,146,34],13,[391,392,18,362,393,34],"47109513",14,"sylware",[395,396,40,213,397,34],"47108929",15,"timzaman",[399,396,34,249,11,40],"47111366",[89,401,48,207,90,34],16,[403,401,34,201,11,40],"47111268",[405,401,40,213,406,34],"47112964","fabifabulous",[408,401,48,276,208,34],"47118490",[410,401,34,192,411,34],"47118799","placatedmayhem",[413,401,40,414,415,34],"47113765","project\u002Fcritical\u002Fidea","3abiton",[417,401,34,249,418,34],"47119899","YetAnotherNick",[420,166,48,207,421,34],"47108151","umairnadeem123",[113,166,40,423,114,34],"project\u002Fcritical\u002Fpricing",[425,166,18,426,427,34],"47108476","project\u002Fneutral\u002Fpricing","Aerroon",[429,166,34,249,430,34],"47108777","teo_zero",[432,166,34,249,433,34],"47109617","IsTom",[435,166,34,249,436,34],"47111411","dotancohen",[438,166,34,249,439,34],"47109857","thatwasunusual",[441,166,34,249,442,34],"47112982","xienze",[444,166,34,265,445,34],"47110367","qoez",[447,166,34,249,448,34],"47113706","culopatin",[450,166,18,426,262,34],"47108549",[452,166,18,198,453,34],"47109277","ekianjo",[455,166,18,198,456,34],"47110021","teaearlgraycold",[458,166,34,201,11,40],"47111491",[460,461,34,192,462,34],"47108898",18,"dhjjdjjjd",[68,464,48,465,69,34],19,"project\u002Fpositive\u002Fexecution",[467,464,18,468,134,34],"47108638","project\u002Fneutral\u002Fexecution",[470,464,34,192,471,34],"47109137","snovv_crash",[473,464,34,192,474,34],"47109394","IshKebab",[476,464,34,192,477,34],"47109786","flux3125",[],[480,485,490,494,499,503],{"id":481,"name":482,"points":483,"category":484},"47075124","Micasa",657,"productivity",{"id":486,"name":487,"points":488,"category":489},"47114530","CIA World Factbook Archive",496,"learning_science",{"id":491,"name":492,"points":493,"category":20},"47131225","Steerling-8B",328,{"id":495,"name":496,"points":497,"category":498},"47123631","PgDog",326,"infrastructure",{"id":500,"name":501,"points":502,"category":20},"47143755","Moonshine Open-Weights STT models",316,{"id":504,"name":505,"points":506,"category":507},"47132102","X86CSS",275,"languages",false,"v2",{"id":511,"name":5,"line":6,"ai":508,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":512,"top":18,"kw":513,"maker":11,"domain":10,"tier":19,"attempts":34,"strip":514,"url":9,"mix":515,"praise":82,"objection":94,"sub":21},47104667,-0.01,[24,25,26,27,28],"3303130001331533330303333330313531003350",[32,33,34,35],1794036693957]