[{"data":1,"prerenderedAt":435},["ShallowReactive",2],{"project-49783133":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":47,"post_text":49,"what":50,"said":69,"maker_in_thread":168,"shape":178,"grid":182,"attempts":397,"same_week":398,"maker_shows":33,"maker_hidden":427,"reader_version":428,"card":429},"49783133","Mini-AGI","Dynamic continual learning model trained on 8GB VRAM","title","Show HN: Mini-AGI – Dynamic continual learning model trained on 8GB VRAM","https:\u002F\u002Fgithub.com\u002Fvolotat\u002Fmini-AGI\u002F","github.com","volotat",1789965757,2026,277,80,167,4366,4,"legend","ai","continual learning model","other",[24,25,26,27],"continual learning","catastrophic forgetting","mixture of experts","batch-1 training","debated",-0.027,{"positive":31,"neutral":32,"mixed":33,"critical":34},11,10,1,18,40,{"idea":37,"performance":40,"docs":43,"design":45,"execution":46},{"positive":38,"critical":39},7,6,{"positive":41,"critical":42},3,8,{"positive":44,"critical":41},0,{"positive":33,"critical":44},{"positive":44,"critical":33},[33,41,44,33,33,44,44,41,33,48,33,48,41,41,48,48,48,44,48,33,33,33,33,44,44,33,33,41,48,41,33,33,48,48,41,44,44,44,44,44],5,"Sorry for the pretentious name, I know, I know.. It just contains all the pieces I would like to see a AGI model to have, and I can't stand the temptation. Before throwing rocks at me, please take a glance at the Readme, and I hope it will cover your mood a little bit.\n\nSo, first of all it does work and you can see the sample from the whole training run here: https:\u002F\u002Fraw.githubusercontent.com\u002Fvolotat\u002Fmini-AGI\u002Frefs\u002Fheads\u002Fmain\u002Fruns\u002Fsamples.txt\n\nHere is the scaling law graph I have so far, and it looks very promising:\nhttps:\u002F\u002Fgithub.com\u002Fvolotat\u002Fmini-AGI\u002Fblob\u002Fmain\u002Fassets\u002Fscaling.png\n\nThe model was",{"problem":51,"audience":60,"pricing":60,"motivation":61,"time_to_build":60,"stack":65,"openness":66,"stage":67,"team_size":68},{"value":52,"source":53,"evidence":54},"Continual learning without catastrophic forgetting, trained on consumer hardware from a single stream","maker",[55,57],{"id":56},"post",{"id":58,"by":11,"text":59},"49799913","I am not sure about eventually, but it learns on the steady paste so far. The main thing to keep in mind is that it learns on the single STREAM of data. Not randomized batched…",null,{"value":62,"source":53,"evidence":63},"Wants full control over what the model trains on, aligned with his interests rather than corporations",[64],{"id":56},[],"open_source","prototype","solo",{"praise":70,"objections":90,"requests":123,"questions":144,"alternatives":163},[71,83],{"value":72,"source":73,"evidence":74},"Continual learning approach on a single stream is interesting","commenter",[75,79],{"id":76,"by":77,"text":78},"49784414","advael","Seems interesting, I've been messing with a lot of continuous learning approaches lately and it's cool to see something that's built from the ground up for avoiding catastrophic…",{"id":80,"by":81,"text":82},"49818411","jmatthews","I built a smaller scale replication. Most of your claims prove out but the lever is essentially Biderman et als \"learn less forget less\". My first pass actually made this mistake…",{"value":84,"source":73,"evidence":85},"Expert swapping architecture is nice",[86],{"id":87,"by":88,"text":89},"49786232","imtringued","The expert swapping architecture is very nice. Have you considered doing nested reinforcement learning where you use the nesting as a sort of low pass filter? The concept is as…",[91,105,116],{"value":92,"source":73,"evidence":93},"Claims of AGI and learning are overstated; outputs are not coherent",[94,98,101],{"id":95,"by":96,"text":97},"49786083","ilaksh","If you actually scroll through the transcript he links to, you will see that something that looks like it could be training is happening, but no coherent responses are coming out…",{"id":99,"by":96,"text":100},"49787089","I would be okay if you shared it as a potential idea and possibly interesting early result, but the language you are actually using to characterize it is misleading or…",{"id":102,"by":103,"text":104},"49791815","seanhunter","To pick a couple of examples: ``` --- chess --- prompt: '\u003Cg>1700 1-0 1. e4 e5 2. ' [raw] repeated 8-grams 83% Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Nxd5 Rxd5…",{"value":106,"source":73,"evidence":107},"No benchmarks or performance analysis",[108,112],{"id":109,"by":110,"text":111},"49786171","synctext","Using the term AGI and not including any performance analysis. My AI calls it: \"massive marketing overreach\". Somebody called this slop in the comments. As a professor who…",{"id":113,"by":114,"text":115},"49785730","lostmsu","This is slop. 8M parameter dense model with context length 64 that you train on enwik9 in 2h will have 1.15 bpb. This model has 1.8 (bits per byte, lower is better).",{"value":117,"source":73,"evidence":118},"Name is misleading",[119],{"id":120,"by":121,"text":122},"49791369","HarHarVeryFunny","Mini-AGI is a totally inappropriate name - it seems what this project is shooting for, but not delivering on, is being a language model with \"continual learning\". Where it seems…",[124,133],{"value":125,"source":73,"evidence":126},"Run standard benchmarks and publish weights",[127,130],{"id":128,"by":11,"text":129},"49787261","For sure. As it will pass through the whole corpus I will share the weights, run it through established benchmarks for small models and share all of this as an update. I am also…",{"id":131,"by":11,"text":132},"49787277","I got you, will add later to the repository.",{"value":134,"source":73,"evidence":135},"Explain how catastrophic forgetting is avoided in the docs",[136,140],{"id":137,"by":138,"text":139},"49800283","tnspacetime","I read your github page and there's mention of continual learning but no argument how catastrophic forgetting is avoided. I think you should explain that better.",{"id":141,"by":142,"text":143},"49787679","Schlagbohrer","I wish I could understand what a single graph in that nice graphic of graphs meant. No explanation for any vertical or horizontal axis. Looks pretty though.",[145,156],{"value":146,"source":73,"evidence":147},"Does the architecture generalize or only memorize?",[148,152],{"id":149,"by":150,"text":151},"49784604","cpldcpu","Is this architecture actually able to generalize or is it mostly based on memorization? Have you tried some basic tasks that require generalization? e.g. number addition etc?",{"id":153,"by":154,"text":155},"49785448","awfm9","What's the advantage of doing this, versus becoming good at context management and RAG? I always found trained knowledge unreliable, given that it is lossy by construction.",{"value":157,"source":73,"evidence":158},"How does it behave after topic shifts?",[159],{"id":160,"by":161,"text":162},"49784823","ilusion","Have you tested what it remembers from early in the stream after a shift in the topics thrown at it?",[164],{"value":165,"source":73,"evidence":166},"Standard LLMs and RAG",[167],{"id":153,"by":154,"text":155},{"comments":169,"answered":48,"key":170},13,[171,172,175],{"id":58,"text":59},{"id":173,"text":174},"49786397","There is no special algorithm, the finding is that slowing down the LR or the trunk, while keeping the LR of the experts is enough to eliminate most of the forgetting in the…",{"id":176,"text":177},"49786893","It is a goalpost that is easy to move. By \"works\" I mean learning from a continuous single (meaning batch-1) stream of data. The fact that it produces full words and full…",{"commenters":179,"max_depth":42,"maker_replies":169,"first_comment_mins":180,"lifespan_hours":181},42,159,126,[183,187,189,191,195,198,200,203,205,207,209,210,213,215,217,219,223,227,228,231,232,234,236,238,241,243,245,249,252,256,257,261,263,266,269,274,277,280,284,286,290,292,295,297,300,303,305,308,310,312,314,317,319,320,322,324,327,331,335,339,343,346,350,353,355,356,358,362,367,372,376,378,380,382,384,386,389,391,393,395],[184,44,33,185,186,44],"49787729","project\u002Fcritical\u002Fidea","abeppu",[58,44,44,188,11,33],"project\u002Fpositive\u002Fidea",[120,33,41,190,121,44],"project\u002Fmixed\u002Fidea",[192,33,44,193,194,44],"49800422","other\u002Fneutral\u002Fnone","OutOfHere",[196,33,44,193,197,44],"49800500","mpalmer",[199,33,44,193,121,44],"49801125",[95,201,33,202,96,44],2,"project\u002Fcritical\u002Fperformance",[109,201,33,204,110,44],"project\u002Fcritical\u002Fdocs",[206,201,33,202,96,44],"49786273",[208,201,33,185,110,44],"49786415",[173,201,44,188,11,33],[211,201,44,212,96,44],"49786523","commenter\u002Fcritical\u002Fperformance",[176,201,44,214,11,33],"commenter\u002Fneutral\u002Fidea",[99,201,44,216,96,44],"commenter\u002Fcritical\u002Fexecution",[128,201,44,218,11,33],"maker\u002Fpositive\u002Fexecution",[220,201,41,221,222,44],"49794900","project\u002Fneutral\u002Fidea","fuzzfactor",[224,201,44,225,226,44],"49790860","commenter\u002Fpositive\u002Fidea","bigbadfeline",[102,201,33,202,103,44],[229,41,44,193,230,44],"49784149","whizzter",[76,18,48,188,77,44],[233,18,33,202,114,44],"49785747",[235,18,33,185,121,44],"49791267",[237,18,41,221,222,44],"49794129",[239,48,48,188,240,44],"49785424","bananaflag",[149,39,41,242,150,44],"project\u002Fneutral\u002Fperformance",[244,39,44,242,11,33],"49784715",[246,39,48,247,248,44],"49784827","project\u002Fpositive\u002Fperformance","jacquesm",[250,39,33,202,251,44],"49786209","dinfinity",[253,38,41,254,255,44],"49786335","project\u002Fneutral\u002Fdesign","K0balt",[80,42,48,247,81,44],[258,259,48,260,81,44],"49787795",9,"maker\u002Fpositive\u002Fidea",[87,32,48,262,88,44],"project\u002Fpositive\u002Fdesign",[264,31,48,188,265,44],"49784236","skeledrew",[267,31,44,268,11,33],"49784266","maker\u002Fpositive\u002Fdesign",[270,271,44,272,273,44],"49787572",12,"other\u002Fneutral\u002Fidea","xtracto",[275,271,44,193,276,44],"49788730","codethief",[278,169,44,272,279,44],"49798139","scottsiume",[281,282,48,188,283,44],"49784071",14,"hexley19",[285,282,44,193,222,44],"49794714",[287,288,33,185,289,44],"49789872",15,"dnautics",[137,291,33,204,138,44],16,[293,291,44,294,11,33],"49800371","maker\u002Fneutral\u002Fdocs",[296,291,33,185,138,44],"49803188",[298,291,44,299,11,33],"49806508","maker\u002Fneutral\u002Fidea",[301,291,33,302,138,44],"49808091","project\u002Fcritical\u002Fexecution",[153,304,41,221,154,44],17,[306,304,44,307,251,44],"49786120","commenter\u002Fneutral\u002Fnone",[309,304,44,307,154,44],"49844583",[311,304,44,307,251,44],"49845866",[313,304,44,307,154,44],"49855178",[315,304,33,185,316,44],"49802976","truejaian",[318,304,44,193,154,44],"49844612",[141,34,33,204,142,44],[113,321,33,202,114,44],19,[160,323,41,242,161,44],20,[325,323,44,326,11,33],"49784889","maker\u002Fneutral\u002Fperformance",[328,329,44,193,330,44],"49809012",21,"SKYNET800",[332,333,48,247,334,44],"49797437",22,"DylanMerigaud",[336,337,41,254,338,44],"49790903",23,"rescbr",[340,341,41,254,342,44],"49785745",24,"maaaaattttt",[344,341,33,202,345,44],"49786113","killerstorm",[347,348,41,242,349,44],"49785536",25,"comboy",[351,348,44,352,11,33],"49785553","project\u002Fneutral\u002Fdocs",[354,348,33,202,349,44],"49786905",[131,348,44,294,11,33],[357,348,48,188,349,44],"49788557",[359,360,48,188,361,44],"49784662",26,"hanselot",[363,364,44,365,366,44],"49784566",27,"other\u002Fcritical\u002Fnone","loopydosuette",[368,369,44,370,371,44],"49791299",28,"other\u002Fcritical\u002Fidea","gslepak",[373,374,41,221,375,44],"49786904",29,"bubblegumcrisis",[377,374,44,221,11,33],"49787218",[379,374,44,193,375,44],"49790164",[381,374,44,307,375,44],"49792206",[383,374,44,307,11,33],"49794778",[385,374,44,193,375,44],"49796322",[387,374,44,193,388,44],"49798064","nextaccountic",[390,374,44,193,375,44],"49799594",[392,374,44,193,388,44],"49812997",[394,374,44,193,375,44],"49799833",[396,374,44,193,375,44],"49794298",[],[399,404,409,414,418,422],{"id":400,"name":401,"points":402,"category":403},"49823738","Bastardica",862,"creative",{"id":405,"name":406,"points":407,"category":408},"49740105","Setup",251,"productivity",{"id":410,"name":411,"points":412,"category":413},"49801329","Drop",193,"dev_tools",{"id":415,"name":416,"points":417,"category":413},"49800574","JevBench",154,{"id":419,"name":420,"points":421,"category":20},"49776523","Tiny Brains",116,{"id":423,"name":424,"points":425,"category":426},"49746346","Snapdrop",112,"infrastructure",false,"v2",{"id":430,"name":5,"line":6,"ai":427,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":431,"top":18,"kw":432,"maker":11,"domain":10,"tier":19,"attempts":44,"strip":433,"url":9,"mix":434,"praise":72,"objection":92,"sub":21},49783133,-0.03,[24,25,26,27],"1301100315153355505111100113531155300000",[31,32,33,34],1794036710545]