[{"data":1,"prerenderedAt":663},["ShallowReactive",2],{"project-40252569":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":36,"aspects":37,"strip":47,"post_text":50,"what":51,"said":86,"maker_in_thread":231,"shape":240,"grid":244,"attempts":625,"same_week":626,"maker_shows":49,"maker_hidden":655,"reader_version":656,"card":657},"40252569","Secret Llama","A free in-browser Llama 3 chatbot powered by WebGPU","title","Show HN: I built a free in-browser Llama 3 chatbot powered by WebGPU","https:\u002F\u002Fgithub.com\u002Fabi\u002Fsecret-llama","github.com","abi",1714771606,2024,547,139,55,3515,2,"legend","ai","local llm chat","web_app",[24,25,26,27,28],"webgpu","llama 3","in-browser llm","local llm","private ai chat","mixed",-0.068,{"positive":32,"neutral":33,"mixed":34,"critical":35},14,26,4,21,65,{"performance":38,"execution":39,"idea":42,"docs":44,"design":46},{"positive":34,"critical":34},{"positive":40,"critical":41},1,13,{"positive":43,"critical":34},7,{"positive":45,"critical":45},0,{"positive":45,"critical":45},[48,48,45,48,49,45,40,48,48,45,48,48,45,45,48,48,48,40,49,45,49,45,45,49,49,48,40,18,18,40,40,18,18,18,45,40,18,34,48,48],3,5,"I spent the last few days building out a nicer ChatGPT-like interface to use Mistral 7B and Llama 3 fully within a browser (no deps and installs).\n\nI’ve used the WebLLM project by MLC AI for a while to interact with LLMs in the browser when handling sensitive data but I found their UI quite lacking for serious use so I built a much better interface around WebLLM.\n\nI’ve been using it as a therapist and coach. And it’s wonderful knowing that my personal information never leaves my local computer.\n\nShould work on Desktop with Chrome or Edge. Other browsers are adding WebGPU support as well - see ",{"problem":52,"audience":58,"pricing":59,"motivation":63,"time_to_build":58,"stack":67,"openness":83,"stage":84,"team_size":85},{"value":53,"source":54,"evidence":55},"Chatting with LLMs without sending sensitive data to a server","maker",[56],{"id":57},"post",null,{"value":60,"source":54,"evidence":61},"free",[62],{"id":57},{"value":64,"source":54,"evidence":65},"Wanted a better UI than WebLLM's for private use such as therapy and coaching",[66],{"id":57},[68,75,79],{"value":69,"source":54,"evidence":70},"WebLLM (MLC AI)",[71,72],{"id":57},{"id":73,"by":11,"text":74},"40260072","Yes. Web-llm is a wrapper of tvmjs: https:\u002F\u002Fgithub.com\u002Fapache\u002Ftvm Just wrappers all the way down",{"value":76,"source":54,"evidence":77},"TVM.js",[78],{"id":73,"by":11,"text":74},{"value":80,"source":54,"evidence":81},"WebGPU",[82],{"id":57},"open_source","launched","unknown",{"praise":87,"objections":111,"requests":156,"questions":183,"alternatives":202},[88,104],{"value":89,"source":90,"evidence":91},"Fast and local; keeps data private","commenter",[92,96,100],{"id":93,"by":94,"text":95},"40257168","NikhilVerma","This is absolutely wonderful, I am a HUGE fan of local first apps. Running models locally is such a powerful thing I wish more companies could leverage it to build smarter apps…",{"id":97,"by":98,"text":99},"40255925","NayamAmarshe","This is amazing! I always wanted something like this, thank you so much!",{"id":101,"by":102,"text":103},"40253934","simple10","Amazing! It's surprisingly fast to load and run given the size of the downloaded models. Do you think it would be feasible to extend it to support web browsing? I'd like to help…",{"value":105,"source":90,"evidence":106},"Better UI than the original WebLLM demo",[107],{"id":108,"by":109,"text":110},"40253749","ngshiheng","Nice demo! I briefly tried it out and the demo felt much better than the original WebLLM one! On a side note, i've been trying to do something similar too for similar reasons…",[112,119,134,145],{"value":113,"source":90,"evidence":114},"Fails to load models after download (IndexedDB cache errors)",[115],{"id":116,"by":117,"text":118},"40261481","raylad","After the model is supposedly fully downloaded (about 4GB) I get: Could not load the model because Error: ArtifactIndexedDBCache failed to fetch:…",{"value":120,"source":90,"evidence":121},"No WebGPU on Linux, Firefox, or some devices",[122,126,130],{"id":123,"by":124,"text":125},"40256886","notarealllama","Tested on Ubuntu 22.04 with Chrome, sure enough, \"Could not load the model because Error: Cannot find adapter that matches the request\". It really is too bad WebGPU isn't…",{"id":127,"by":128,"text":129},"40254700","koolala","On Firefox Nightly on my Steam Deck it \"cannot find WebGPU in the environment\".",{"id":131,"by":132,"text":133},"40257102","r0fl","Could not load the model because Error: Cannot find WebGPU in the environment",{"value":135,"source":90,"evidence":136},"Downloads several GB of model data to the browser",[137,141],{"id":138,"by":139,"text":140},"40253432","knowaveragejoe","Is this downloading a ~5gb model to my machine and storing it locally for subsequent use?",{"id":142,"by":143,"text":144},"40274247","hpeter","It's great but I hope it don't catch on because then every website will make me download models. My hard drive will be full, too much bloat. I think the web is not good for this.…",{"value":146,"source":90,"evidence":147},"Small models give poor or incoherent answers",[148,152],{"id":149,"by":150,"text":151},"40254601","dosinga","Phi 1.5 isn't quite there is it? Caught it having some inner dialog. What does that mean? Instruct: I'm sorry, but I'm not sure what you mean by \"wutt\". Output: Oh, I'm sorry. I…",{"id":153,"by":154,"text":155},"40275778","BargirPezza","Llama 3 seems to not be there yet at all. Can it read earlier prompts and what you have written before? It doesn't seem like it. I asked it to help come up with a song name for a…",[157,164,168,176],{"value":158,"source":90,"evidence":159},"Chat history",[160],{"id":161,"by":162,"text":163},"40253752","joshstrange","Very cool! I wish there was chat history. Also if you click the \"New Chat\" button while an answer is generating I think some of the output gets fed back into the model, it causes…",{"value":165,"source":90,"evidence":166},"Web browsing support",[167],{"id":101,"by":102,"text":103},{"value":169,"source":90,"evidence":170},"Support for GGUF files and Ollama",[171,175],{"id":172,"by":173,"text":174},"40253756","indit","Could we use an already downloaded .gguf file?",{"id":142,"by":143,"text":144},{"value":177,"source":90,"evidence":178},"Dynamic text width",[179],{"id":180,"by":181,"text":182},"40257204","Jackson_Fleck","This is amazing but can we please set the .prose width to be dynamic? the text column in 3 inches wide on my monitor, it should take up a % of the browser window.",[184,195],{"value":185,"source":90,"evidence":186},"How to use it as a therapist or coach",[187,191],{"id":188,"by":189,"text":190},"40257590","wg0","How do people use something like this as coach or therapist? This is genuine question. Side note, impressive project. Future of AI is offline mostly with few APIs in the cloud…",{"id":192,"by":193,"text":194},"40256665","Dowwie","What therapy prompts have you found useful?",{"value":196,"source":90,"evidence":197},"Whether quality differs from native runners",[198],{"id":199,"by":200,"text":201},"40256640","zerop","Question - Do I compromise on quality on answers if I use models using WebLLM (like this) compare to using them on system console.",[203,214,221,227],{"value":204,"source":90,"evidence":205},"Ollama",[206,210],{"id":207,"by":208,"text":209},"40255260","littlestymaar","This is very cool, it's something I wish existed since Llama came out, having to install Ollama + Cuda to get locally working LLM didn't felt right to me when there's all what's…",{"id":211,"by":212,"text":213},"40260228","earleybird","Likewise (same error) with Chrome on Windows. Currently running Ollama \u002F Open WebUI and finding lama3:8B quite useful for writing snippets of powershell, javascript, golang etc.",{"value":215,"source":90,"evidence":216},"LM Studio",[217],{"id":218,"by":219,"text":220},"40257303","Jedd","I've managed to avoid ollama and just toyed with lmstudio. It's non-free software, but extremely easy to get into, uses llama.cpp under the hood, cross-platform, yada yada.…",{"value":222,"source":54,"evidence":223},"Window AI",[224],{"id":225,"by":11,"text":226},"40260059","Window AI (https:\u002F\u002Fwindowai.io\u002F) is an attempt to do something like this with a browser extension.",{"value":228,"source":90,"evidence":229},"llamafile",[230],{"id":207,"by":208,"text":209},{"comments":232,"answered":233,"key":234},12,6,[235,238,239],{"id":236,"text":237},"40260061","Appreciate the kind words :)",{"id":225,"text":226},{"id":73,"text":74},{"commenters":241,"max_depth":233,"maker_replies":232,"first_comment_mins":242,"lifespan_hours":243},100,121,218,[245,247,249,251,255,259,262,265,268,271,273,275,279,283,287,291,294,297,301,304,306,309,312,316,318,321,323,325,327,330,332,335,338,341,344,347,350,353,356,359,362,366,369,372,374,376,378,381,384,386,389,392,394,396,399,402,405,407,410,413,416,418,421,424,426,429,432,435,438,441,444,448,451,454,456,458,460,463,466,469,471,475,477,480,483,485,487,490,491,493,495,498,500,503,507,510,512,513,517,519,522,523,524,527,529,531,534,536,540,542,544,546,549,552,554,556,559,561,564,567,569,572,575,579,583,586,589,592,594,598,599,603,606,608,611,613,616,619,623],[93,45,49,246,94,45],"project\u002Fpositive\u002Fperformance",[236,45,45,248,11,40],"maker\u002Fpositive\u002Fnone",[149,40,40,250,150,45],"project\u002Fcritical\u002Fperformance",[252,40,48,253,254,45],"40255693","project\u002Fneutral\u002Fnone","PhilippGille",[256,40,45,257,258,45],"40256101","other\u002Fneutral\u002Fnone","moffkalast",[260,40,45,257,261,45],"40256955","phito",[263,40,45,257,264,45],"40257993","afavour",[266,40,45,257,267,45],"40259022","stevenicr",[269,40,45,257,270,45],"40259211","edgineer",[272,40,45,257,258,45],"40260305",[274,40,45,257,264,45],"40260687",[276,40,48,277,278,45],"40258546","project\u002Fneutral\u002Fexecution","ComputerGuru",[280,40,48,281,282,45],"40259445","project\u002Fmixed\u002Fperformance","andai",[284,18,49,285,286,45],"40258866","project\u002Fpositive\u002Fidea","_akhe",[288,18,45,289,290,45],"40258927","other\u002Fneutral\u002Fidea","flawsofar",[292,18,45,257,293,45],"40259104","KennyBlanken",[295,18,45,257,296,45],"40259267","gordinmitya",[298,18,45,299,300,45],"40259346","commenter\u002Fneutral\u002Fnone","squigz",[302,18,45,257,303,45],"40260244","creesch",[305,18,45,257,300,45],"40260378",[307,18,45,257,308,45],"40263971","nolist_policy",[310,18,45,257,311,45],"40263057","paulmd",[313,18,40,314,315,45],"40259409","project\u002Fcritical\u002Fidea","Cheer2171",[225,18,45,317,11,40],"maker\u002Fneutral\u002Fidea",[319,48,48,277,320,45],"40258004","low_tech_punk",[73,48,45,322,11,40],"maker\u002Fneutral\u002Fnone",[324,48,45,257,286,45],"40271166",[161,34,48,326,162,45],"project\u002Fmixed\u002Fexecution",[328,34,45,257,329,45],"40256741","brianzelip",[331,34,45,257,162,45],"40257052",[333,34,45,257,334,45],"40260047","bbkane",[336,34,45,337,11,40],"40260088","maker\u002Fneutral\u002Fexecution",[339,49,45,257,340,45],"40253844","manlobster",[342,49,45,257,343,45],"40254576","doctorpangloss",[345,49,45,257,346,45],"40254757","westurner",[348,49,45,257,349,45],"40255455","TeMPOraL",[351,49,48,253,352,45],"40255391","exe34",[354,49,45,257,355,45],"40258028","roywiggins",[357,49,45,257,358,45],"40256760","swores",[360,49,45,257,361,45],"40258612","zozbot234",[363,233,48,364,365,45],"40254302","project\u002Fneutral\u002Fidea","threatofrain",[367,233,45,257,368,45],"40254920","KeplerBoy",[370,233,45,257,371,45],"40254988","dannyw",[373,233,45,317,11,40],"40255100",[375,233,45,257,368,45],"40255325",[377,233,45,257,349,45],"40255403",[379,233,45,257,380,45],"40256108","SXX",[382,233,45,257,383,45],"40256268","JimDabell",[385,233,45,257,383,45],"40255882",[387,233,45,257,388,45],"40257223","yokto",[390,233,48,364,391,45],"40257818","blurpesec",[393,233,48,364,143,45],"40274318",[138,43,48,395,139,45],"project\u002Fneutral\u002Fperformance",[397,43,48,395,398,45],"40253592","sp332",[400,43,45,401,11,40],"40253724","maker\u002Fneutral\u002Fperformance",[403,43,48,364,404,45],"40253772","simonw",[406,43,45,322,11,40],"40253974",[408,43,45,257,409,45],"40254540","FL33TW00D",[411,43,48,395,412,45],"40253810","kamikazeturtles",[414,43,45,257,415,45],"40254024","zamadatix",[417,43,45,299,412,45],"40254060",[419,420,40,314,340,45],"40254170",8,[422,420,48,322,423,45],"40254180","BoorishBears",[188,425,49,285,189,45],9,[427,425,49,285,428,45],"40257676","kushie",[430,425,48,364,431,45],"40258266","cal85",[433,425,45,257,434,45],"40258850","pseudosavant",[436,425,45,437,293,45],"40259113","other\u002Fneutral\u002Fprivacy",[439,425,45,440,11,40],"40260100","project\u002Fpositive\u002Fprivacy",[442,425,45,257,443,45],"40259544","intended",[445,446,49,285,447,45],"40256774",10,"nojvek",[449,446,45,257,450,45],"40257166","mcculley",[452,446,45,257,453,45],"40258584","neurostimulant",[455,446,45,257,450,45],"40259438",[457,446,45,257,447,45],"40264006",[459,446,45,257,450,45],"40264288",[461,446,45,257,462,45],"40257911","zitterbewegung",[464,446,45,257,465,45],"40258224","maxboone",[467,446,45,257,468,45],"40272444","tome",[470,446,45,257,468,45],"40275956",[472,473,48,253,474,45],"40257178",11,"andrewfromx",[476,473,45,322,11,40],"40257435",[478,232,49,246,479,45],"40253926","mentos",[481,232,49,246,482,45],"40254603","bastawhiz",[484,232,45,248,11,40],"40255317",[486,232,45,257,258,45],"40256134",[488,232,45,257,489,45],"40256577","navigate8310",[97,41,49,285,98,45],[116,32,40,492,117,45],"project\u002Fcritical\u002Fexecution",[494,32,45,337,11,40],"40261708",[496,32,40,492,497,45],"40338285","trog",[131,499,40,492,132,45],15,[501,499,48,253,502,45],"40257279","Lex-2008",[504,499,48,505,506,45],"40257299","project\u002Fneutral\u002Fdocs","MayeulC",[508,499,40,492,509,45],"40257569","pjmlp",[207,511,49,285,208,45],16,[123,511,40,492,124,45],[514,511,49,515,516,45],"40259027","project\u002Fpositive\u002Fexecution","ac29",[518,511,40,492,124,45],"40278083",[520,511,40,492,521,45],"40259315","whartung",[211,511,40,492,212,45],[218,511,40,314,219,45],[525,511,45,299,526,45],"40257947","spmurrayzzz",[528,511,45,299,208,45],"40258935",[530,511,45,299,526,45],"40259060",[532,511,45,533,208,45],"40262880","commenter\u002Fpositive\u002Fidea",[535,511,40,250,526,45],"40265954",[537,538,40,492,539,45],"40253816",17,"geor9e",[541,538,45,337,11,40],"40253984",[142,543,40,314,143,45],18,[101,545,49,285,102,45],19,[547,545,49,548,102,45],"40253959","project\u002Fpositive\u002Fnone",[550,545,40,492,551,45],"40257269","ashellunts",[553,545,40,492,102,45],"40259994",[172,555,48,253,173,45],20,[557,35,48,364,558,45],"40256285","Its_Padar",[127,560,40,492,128,45],22,[562,560,48,253,563,45],"40254790","eyegor",[565,560,45,257,566,45],"40255806","ojosilva",[568,560,45,299,563,45],"40256571",[570,560,45,257,571,45],"40256917","yokoprime",[573,560,48,277,574,45],"40255770","1f60c",[576,577,40,250,578,45],"40255063",23,"Snoozus",[580,577,45,581,582,45],"40255386","commenter\u002Fneutral\u002Fperformance","lastdong",[584,577,45,581,585,45],"40255400","lukan",[587,588,40,492,574,45],"40255802",24,[590,588,48,277,591,45],"40257005","spacebanana7",[593,588,48,277,509,45],"40257576",[595,596,49,246,597,45],"40261953",25,"gitinit",[199,33,48,395,200,45],[600,601,48,395,602,45],"40258374",27,"adontz",[604,601,45,581,605,45],"40258621","valine",[192,607,48,364,193,45],28,[609,607,45,257,610,45],"40257608","Y_Y",[108,612,48,281,109,45],29,[614,615,48,364,181,45],"40257438",30,[180,617,48,618,181,45],31,"project\u002Fmixed\u002Fdesign",[620,621,49,248,622,45],"40255875",32,"Bradd3rs",[153,624,40,250,154,45],33,[],[627,631,636,641,646,650],{"id":628,"name":629,"points":630,"category":20},"40300126","GrafyChat",459,{"id":632,"name":633,"points":634,"category":635},"40250557","BandMatch",452,"productivity",{"id":637,"name":638,"points":639,"category":640},"40260035","Dillo",430,"dev_tools",{"id":642,"name":643,"points":644,"category":645},"40273470","GetAnyPlant",427,"consumer_social",{"id":647,"name":648,"points":649,"category":20},"40272339","AI climbing coach",320,{"id":651,"name":652,"points":653,"category":654},"40260259","gpudeploy.com",267,"infrastructure",false,"v2",{"id":658,"name":5,"line":6,"ai":655,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":659,"top":18,"kw":660,"maker":11,"domain":10,"tier":19,"attempts":45,"strip":661,"url":9,"mix":662,"praise":89,"objection":113,"sub":21},40252569,-0.07,[24,25,26,27,28],"3303501330330033315050055312211222012433",[32,33,34,35],1794036654232]