[{"data":1,"prerenderedAt":632},["ShallowReactive",2],{"project-41105130":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":5,"maker":10,"posted_at":11,"year":12,"points":13,"comments":14,"rank_in_year":15,"shown_in_year":16,"top":17,"tier":18,"category":19,"subcategory":20,"format":21,"keywords":22,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":47,"post_text":49,"what":50,"said":97,"maker_in_thread":243,"shape":254,"grid":259,"attempts":595,"same_week":596,"maker_shows":257,"maker_hidden":624,"reader_version":625,"card":626},"41105130","embedding.io","Turn any website into a knowledge base for LLMs","title","Show HN: Turn any website into a knowledge base for LLMs","https:\u002F\u002Fwww.embedding.io\u002F","tompec",1722300845,2024,305,122,198,3515,6,"legend","ai","rag knowledge base api","api",[23,24,25,26,27],"rag","vector database","llm knowledge base","web crawling","embeddings","curious",-0.149,{"positive":31,"neutral":32,"mixed":33,"critical":34},8,30,1,18,57,{"execution":37,"pricing":40,"idea":41,"privacy":44,"docs":45,"design":46},{"positive":38,"critical":39},0,3,{"positive":33,"critical":39},{"positive":42,"critical":43},5,7,{"positive":38,"critical":39},{"positive":33,"critical":33},{"positive":38,"critical":38},[39,38,42,33,33,38,39,42,33,38,42,38,38,38,33,39,33,42,33,48,33,33,38,42,33,33,33,42,39,33,39,33,39,39,42,39,39,39,33,42],2,"I built this tool because I wanted a way to just take a bunch of URLs or domains, and query their content in RAG applications.\n\nIt takes away the pain of crawling, extracting content, chunking, vectorizing, and updating periodically.\n\nI'm curious to see if it can be useful to others. I meant to launch this six months ago but life got in the way...",{"problem":51,"audience":57,"pricing":65,"motivation":75,"time_to_build":79,"stack":80,"openness":95,"stage":96,"team_size":95},{"value":52,"source":53,"evidence":54},"Crawling, extracting, chunking, vectorizing and updating website content for RAG apps","maker",[55],{"id":56},"post",{"value":58,"source":59,"evidence":60},"Developers building RAG applications or semantic search","inferred",[61,62],{"id":56},{"id":63,"by":10,"text":64},"41105235","Give it URLs or domains, and it will crawl and extract their content, embed them in a vector database, and give you an endpoint that you can then query when doing RAG stuff or…",{"value":66,"source":53,"evidence":67},"Small plans planned; enterprise via contact us",[68,71],{"id":69,"by":10,"text":70},"41123821","Thanks! I'm still figuring things out about pricing, but there will be small plans available.",{"id":72,"by":73,"text":74},"41124514","rvz","> Enterprise: Contact Us If there is no certifications or compliance information then I don't think there is anything to discuss about any enterprise plan.",{"value":76,"source":53,"evidence":77},"Maker wanted a way to query website content in RAG apps without the pain of crawling and vectorizing",[78],{"id":56},null,[81,87,91],{"value":82,"source":53,"evidence":83},"Serverless Laravel",[84],{"id":85,"by":10,"text":86},"41107447","Tech stack is a mix of serverless Laravel, with Cloudflare and AWS functions, and some Pinecone for vector storage. Still experimenting on a few things but don't want to…",{"value":88,"source":53,"evidence":89},"Cloudflare and AWS functions",[90],{"id":85,"by":10,"text":86},{"value":92,"source":53,"evidence":93},"Pinecone vector storage",[94],{"id":85,"by":10,"text":86},"unknown","launched",{"praise":98,"objections":118,"requests":155,"questions":192,"alternatives":221},[99,111],{"value":100,"source":101,"evidence":102},"Useful abstraction over embeddings and vector databases","commenter",[103,107],{"id":104,"by":105,"text":106},"41122405","crowcroft","I like this. Abstracting away the management of embeddings and vector database is something I desperately want, and adding in website crawling is useful as well.",{"id":108,"by":109,"text":110},"41122142","23B1","I tried it out. This would be extremely useful to me to the point I'd be willing to happily pay for it, as it's something I would have otherwise had to spend a long time hacking…",{"value":112,"source":101,"evidence":113},"Good documentation and launch",[114],{"id":115,"by":116,"text":117},"41129053","hluska","I like the concept, the documentation is very good and I even enjoy the domain name. This is an excellent launch and congratulations on getting it out.",[119,134,141,145],{"value":120,"source":101,"evidence":121},"Scraping websites without consent or robots.txt respect is unethical",[122,126,130],{"id":123,"by":124,"text":125},"41125612","jakubsuchy","I feel like this is unethical. You built yet another bot scraper. It would only be an ethical tool if it validated I own the website I am scraping before it starts.",{"id":127,"by":128,"text":129},"41123630","Cynddl","I find it interesting that as an (edit: UK) academic researcher, I would be likely be forbidden to use tools like this, that fail basic ethics standards, regulations such as…",{"id":131,"by":132,"text":133},"41122836","srameshc","Valid question and I am sure it doesn't.",{"value":135,"source":101,"evidence":136},"Sites will add bot gates, making the service less viable",[137],{"id":138,"by":139,"text":140},"41122486","muggermuch","I like this a lot! But: I feel the more of these services come to being, the more likely it is that every website starts putting up gates to keep the bots away. Sort of like a…",{"value":142,"source":101,"evidence":143},"Enterprise pricing lacks certifications or compliance info",[144],{"id":72,"by":73,"text":74},{"value":146,"source":101,"evidence":147},"Server errors and load issues",[148,152],{"id":149,"by":150,"text":151},"41129607","vulture916","Experiencing many Internal Server Errors.",{"id":153,"by":10,"text":154},"41123878","Sorry about that, a bit too much load at the moment",[156,167,178,185],{"value":157,"source":101,"evidence":158},"Support for login-gated content",[159,163],{"id":160,"by":161,"text":162},"41106660","blackeyeblitzar","Is there a way to deal with websites where you need to login? Like subscription based sites?",{"id":164,"by":165,"text":166},"41129927","suyash","Can it get content that is gated\u002Fbehind login ?",{"value":168,"source":101,"evidence":169},"Local or on-prem deployment",[170,174],{"id":171,"by":172,"text":173},"41122549","khanan","Can this be deployed on-prem or is it an cloud-toy?",{"id":175,"by":176,"text":177},"41128168","replete","Does anyone know of a way to do this locally with Ollama? The 'chat with documentation' thing is something I was thinking of a week ago when dealing with hallucinating cloud AI.…",{"value":179,"source":101,"evidence":180},"Image embedding",[181],{"id":182,"by":183,"text":184},"41125615","boredemployee","does it embbed images as well? if not, do you plan to do so?",{"value":186,"source":101,"evidence":187},"Ability to add own sitemap",[188],{"id":189,"by":190,"text":191},"41122442","rcarmo","How do I feed it a sitemap?",[193,200,207,214],{"value":194,"source":101,"evidence":195},"How chunking and embeddings work",[196],{"id":197,"by":198,"text":199},"41124920","kaycebasques","I spent a lot of time thinking about how to manage embeddings for docs sites. This is basically the same solution that I landed on but never got around to shipping as a…",{"value":201,"source":101,"evidence":202},"Which LLM model is used",[203],{"id":204,"by":205,"text":206},"41124984","olalonde","Which LLM model is it using?",{"value":208,"source":101,"evidence":209},"How changes to sites are detected and re-crawled",[210],{"id":211,"by":212,"text":213},"41132390","lua-steve","How does this handle changes to the website? Does it re-crawl the whole site periodically and regenerate the embeddings? Or is there some sort of diff-checker that only picks up…",{"value":215,"source":101,"evidence":216},"How much it costs",[217],{"id":218,"by":219,"text":220},"41129172","ckluis","How much does it cost?",[222,229,236],{"value":223,"source":101,"evidence":224},"contentmap Python library",[225],{"id":226,"by":227,"text":228},"41128255","jeanloolz","I built a similar thing as a python library that does just that: https:\u002F\u002Fgithub.com\u002Fphilippe2803\u002Fcontentmap Blog post that explains the rationale behind the library:…",{"value":230,"source":101,"evidence":231},"chat_any_site open source app",[232],{"id":233,"by":234,"text":235},"41122341","mkw5053","I made a similar open source app a year ago or so https:\u002F\u002Fgithub.com\u002Fmkwatson\u002Fchat_any_site",{"value":237,"source":101,"evidence":238},"warc-gpt",[239],{"id":240,"by":241,"text":242},"41149025","toomuchtodo","Could you support ingesting WARC files? https:\u002F\u002Fgithub.com\u002Fharvard-lil\u002Fwarc-gpt…",{"comments":244,"answered":245,"key":246},16,12,[247,248,251],{"id":69,"text":70},{"id":249,"text":250},"41123628","Thanks! The chat demo is actually just a small thing I put together as a preview of what can be done, but the main product is the API. But seeing that most users seem to like…",{"id":252,"text":253},"41124959","I chunk pages and generate embeddings for each chunk. So there's no real size limit per page.",{"commenters":255,"max_depth":256,"maker_replies":244,"first_comment_mins":257,"lifespan_hours":258},85,9,4,116,[260,264,267,269,273,275,278,280,284,286,288,292,296,299,302,305,309,312,315,317,318,320,322,325,327,328,331,334,337,339,341,344,347,350,352,354,356,358,360,363,364,367,370,374,378,381,385,388,390,394,396,399,402,406,407,411,414,417,420,423,427,428,431,433,436,439,441,444,447,449,452,455,458,461,463,466,469,471,475,478,482,484,488,492,494,498,500,502,505,509,510,514,518,520,523,525,527,528,530,532,536,538,540,542,544,546,548,552,553,556,559,561,564,567,571,573,577,579,583,586,589,592],[261,38,39,262,263,38],"41126763","project\u002Fneutral\u002Fnone","vladde",[265,38,39,262,266,38],"41141804","soundblaster",[226,33,38,268,227,38],"other\u002Fneutral\u002Fidea",[270,33,38,271,272,38],"41130076","other\u002Fneutral\u002Fnone","xrd",[274,33,38,271,227,38],"41147294",[276,33,38,268,277,38],"41144257","samstave",[108,48,39,279,109,38],"project\u002Fmixed\u002Fexecution",[281,48,42,282,283,38],"41122987","project\u002Fpositive\u002Fpricing","dmje",[69,48,38,285,10,33],"maker\u002Fneutral\u002Fpricing",[249,48,38,287,10,33],"maker\u002Fneutral\u002Fnone",[289,39,33,290,291,38],"41125276","project\u002Fcritical\u002Fidea","MattDaEskimo",[293,39,38,294,295,38],"41125644","commenter\u002Fneutral\u002Fidea","muratsu",[297,39,38,294,298,38],"41125683","obilgic",[300,39,38,294,301,38],"41126920","sooheon",[303,39,33,290,304,38],"41126044","temporary_name",[306,39,38,307,308,38],"41126492","commenter\u002Fpositive\u002Fidea","purple-leafy",[310,39,38,294,311,38],"41127665","CPLX",[313,39,38,314,116,38],"41129040","commenter\u002Fneutral\u002Fnone",[197,257,39,316,198,38],"maker\u002Fneutral\u002Fexecution",[252,257,38,316,10,33],[319,257,39,316,198,38],"41125282",[321,257,38,316,10,33],"41125745",[323,257,38,314,324,38],"41128858","chasd00",[104,42,42,326,105,38],"project\u002Fpositive\u002Fidea",[138,17,33,290,139,38],[329,17,38,271,330,38],"41123720","marcellus23",[332,17,38,271,333,38],"41125035","dankwizard",[335,17,38,271,336,38],"41125204","diggan",[338,17,38,271,301,38],"41126911",[340,17,38,271,330,38],"41129681",[342,17,42,326,343,38],"41124915","tremarley",[345,17,38,268,346,38],"41125769","throw10920",[348,17,38,271,349,38],"41127406","pjc50",[351,17,38,271,346,38],"41127787",[353,17,38,271,349,38],"41127837",[355,17,38,271,346,38],"41127876",[357,17,38,271,349,38],"41128231",[359,17,38,271,346,38],"41128797",[361,17,38,314,362,38],"41133114","ben_w",[175,43,38,271,176,38],[365,43,38,271,366,38],"41128576","dartos",[368,43,38,271,369,38],"41129026","navbaker",[371,31,33,372,373,38],"41128672","project\u002Fcritical\u002Fpricing","have_faith",[375,31,39,376,377,38],"41129525","project\u002Fneutral\u002Fpricing","crngefest",[379,31,33,372,380,38],"41131486","notatoad",[382,256,39,383,384,38],"41129283","project\u002Fneutral\u002Fidea","is_true",[386,256,38,271,387,38],"41131360","mmkos",[389,256,38,271,384,38],"41138534",[391,392,33,290,393,38],"41131080",10,"dmitrygr",[240,395,39,262,241,38],11,[397,245,38,271,398,38],"41146323","ashwinnair99",[211,400,39,401,212,38],13,"project\u002Fneutral\u002Fexecution",[403,404,42,326,405,38],"41106444",14,"samuria",[85,404,38,316,10,33],[408,404,33,409,410,38],"41121669","maker\u002Fcritical\u002Fprivacy","stevenicr",[412,404,38,413,410,38],"41123669","other\u002Fneutral\u002Fprivacy",[415,404,39,401,416,38],"41107894","lou1306",[418,404,39,401,419,38],"41109851","kordlessagain",[421,404,38,271,422,38],"41126553","oneshtein",[424,425,39,401,426,38],"41121686",15,"dazbradbury",[123,244,33,290,124,38],[429,244,38,314,430,38],"41125650","corn13read2",[432,244,33,290,124,38],"41127799",[434,244,38,314,435,38],"41131363","talldatethrow",[437,244,38,271,438,38],"41126707","visarga",[440,244,38,271,349,38],"41127444",[442,244,33,290,443,38],"41126835","realusername",[445,244,38,314,446,38],"41127234","elric",[448,244,38,271,443,38],"41127796",[450,244,38,314,451,38],"41128526","fkyoureadthedoc",[453,244,38,314,454,38],"41129909","kmoser",[115,456,42,457,116,38],17,"project\u002Fpositive\u002Fdocs",[459,34,39,262,460,38],"41122741","danirogerc",[462,34,38,287,10,33],"41123587",[127,464,33,465,128,38],19,"project\u002Fcritical\u002Fprivacy",[467,464,38,314,468,38],"41129183","popcorncowboy",[470,464,33,465,128,38],"41129528",[472,473,39,262,474,38],"41117625",20,"ancras",[476,473,38,477,10,33],"41117788","project\u002Fpositive\u002Fexecution",[479,473,33,480,481,38],"41121746","project\u002Fcritical\u002Fexecution","webappguy",[483,473,38,287,10,33],"41123527",[485,486,39,262,487,38],"41130756",21,"nashashmi",[489,490,38,271,491,38],"41127863",22,"oars",[493,490,38,271,366,38],"41128609",[495,496,42,326,497,38],"41128286",23,"alok-g",[160,499,39,262,161,38],24,[501,499,38,290,10,33],"41107412",[503,499,39,262,504,38],"41126112","brianfryer",[506,507,33,480,508,38],"41121792",25,"michaelmior",[153,507,38,287,10,33],[511,512,38,271,513,38],"41125416",26,"breck",[515,516,39,262,517,38],"41125931",27,"mattfrommars",[164,519,39,262,165,38],28,[521,519,38,314,522,38],"41132136","onemoresoop",[524,519,38,314,165,38],"41133186",[149,526,33,480,150,38],29,[72,32,33,372,73,38],[529,32,38,287,10,33],"41124591",[204,531,39,262,205,38],31,[533,534,39,262,535,38],"41128722",32,"barrenko",[189,537,39,262,190,38],33,[539,537,38,287,10,33],"41123466",[218,541,39,376,219,38],34,[182,543,39,262,183,38],35,[545,543,38,287,10,33],"41126050",[547,543,42,326,183,38],"41135491",[549,550,39,262,551,38],"41105152",36,"cranberryturkey",[63,550,38,287,10,33],[554,550,39,287,555,38],"41106461","xiconfjs",[557,550,39,287,558,38],"41107488","ramon156",[560,550,38,271,419,38],"41109877",[562,550,38,271,563,38],"41130852","sunir",[565,566,39,262,165,38],"41129914",37,[568,569,39,262,570,38],"41121735",38,"r0b05",[233,572,38,271,234,38],39,[574,575,39,262,576,38],"41122186",40,"pryelluw",[131,575,33,578,132,38],"project\u002Fcritical\u002Fnone",[580,575,33,581,582,38],"41122961","project\u002Fcritical\u002Fdocs","danirod",[584,575,38,585,10,33],"41123782","project\u002Fneutral\u002Fdocs",[587,575,42,588,576,38],"41124393","maker\u002Fpositive\u002Fnone",[171,590,39,591,172,38],41,"project\u002Fneutral\u002Fdesign",[593,590,38,594,10,33],"41123688","",[],[597,602,606,610,615,619],{"id":598,"name":599,"points":600,"category":601},"41068719","Haystack",546,"dev_tools",{"id":603,"name":604,"points":605,"category":601},"41070428","Wat",393,{"id":607,"name":608,"points":609,"category":601},"41088273","Semantic Grep",356,{"id":611,"name":612,"points":613,"category":614},"41086620","Opslane",319,"infrastructure",{"id":616,"name":617,"points":618,"category":601},"41080189","gg",205,{"id":620,"name":621,"points":622,"category":623},"41095839","statsbomb-3d-viz",182,"creative",false,"v2",{"id":627,"name":5,"line":6,"ai":624,"cat":19,"year":12,"points":13,"comments":14,"rec":28,"s":628,"top":17,"kw":629,"maker":10,"domain":5,"tier":18,"attempts":38,"strip":630,"url":9,"mix":631,"praise":100,"objection":120,"sub":20},41105130,-0.15,[23,24,25,26,27],"3051103510500013151211051115313133533315",[31,32,33,34],1793991096881]