[{"data":1,"prerenderedAt":390},["ShallowReactive",2],{"project-37050532":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":35,"aspects":36,"strip":43,"post_text":45,"what":46,"said":81,"maker_in_thread":196,"shape":209,"grid":213,"attempts":353,"same_week":354,"maker_shows":39,"maker_hidden":383,"reader_version":384,"card":385},"37050532","Airbyte","Chat with your data using LangChain, Pinecone, and Airbyte","title","Show HN: Chat with your data using LangChain, Pinecone, and Airbyte","https:\u002F\u002Fairbyte.com\u002Ftutorials\u002Fchat-with-your-data-using-openai-pinecone-airbyte-and-langchain","airbyte.com","mtricot",1691508733,2023,220,59,249,3109,9,"notable","ai","retrieval chatbot tutorial","other",[24,25,26,27,28],"langchain","pinecone","vector database","airbyte","llm chatbot","curious",0,{"positive":32,"neutral":33,"mixed":34,"critical":32},3,16,1,23,{"performance":37,"docs":38,"privacy":40,"idea":41,"execution":42},{"positive":30,"critical":30},{"positive":39,"critical":30},2,{"positive":34,"critical":34},{"positive":30,"critical":30},{"positive":30,"critical":34},[32,30,32,30,32,44,30,30,30,30,30,34,34,32,30,32,30,30,30,32,32,34,44,32,30,32,32,32,30,32,30,30,32,32,30,30,30,44,32,30],5,"Hi HN,\n\nA few of our team members at Airbyte (and Joe, who killed it!) recently played with building our own internal support chat bot, using Airbyte, Langchain, Pinecone and OpenAI, that would answer any questions we ask when developing a new connector on Airbyte.\n\nAs we prototyped it, we realized that it could be applied for many other use cases and sources of data, so... we created a tutorial that other community members can leverage [http:\u002F\u002Fairbyte.com\u002Ftutorials\u002Fchat-with-your-data-using-openai-pinecone-airbyte-and-langchain] and the Github repo to run it [https:\u002F\u002Fgithub.com\u002Fairbytehq\u002Ftuto",{"problem":47,"audience":53,"pricing":57,"motivation":58,"time_to_build":57,"stack":62,"openness":78,"stage":79,"team_size":80},{"value":48,"source":49,"evidence":50},"Asking questions over proprietary or unstructured data using an LLM","maker",[51],{"id":52},"post",{"value":54,"source":49,"evidence":55},"Developers building LLM apps over their own data, such as Airbyte connector developers",[56],{"id":52},null,{"value":59,"source":49,"evidence":60},"Internal support bot prototype that was generalised into a community tutorial",[61],{"id":52},[63,66,70,74],{"value":5,"source":49,"evidence":64},[65],{"id":52},{"value":67,"source":49,"evidence":68},"LangChain",[69],{"id":52},{"value":71,"source":49,"evidence":72},"Pinecone",[73],{"id":52},{"value":75,"source":49,"evidence":76},"OpenAI",[77],{"id":52},"open_source","prototype","company",{"praise":82,"objections":99,"requests":125,"questions":152,"alternatives":177},[83],{"value":84,"source":85,"evidence":86},"Well written, practical tutorial with substance","commenter",[87,91,95],{"id":88,"by":89,"text":90},"37050684","bnchrch","Always so happy to see a tutorial with actual substance. So much on LLMs lately is mostly blog spam for SEO but this actually is information dense and practical. Definitely…",{"id":92,"by":93,"text":94},"37054105","gz5","Very well written and illustrated, thank you. When using a local vector db, what is the security model between my data and Airbyte? For example, do I need to permit Airbyte IPs…",{"id":96,"by":97,"text":98},"37056906","everythingmeta","nice to see a tutorial that recognizes the case where the underlying data can change and the embedding needs to be updated. Any plans to write a tutorial for fine-tuning local…",[100,111,118],{"value":101,"source":85,"evidence":102},"Sending data to Pinecone and OpenAI raises privacy and PII concerns",[103,107],{"id":104,"by":105,"text":106},"37054678","ramesh31","I don't know what datasets you guys are working with that have no issues being shared in plain text across three separate proprietary paid services, but this is a nonstarter for…",{"id":108,"by":109,"text":110},"37054807","r_thambapillai","How are you thinking about preventing customer PII making it to OpenAI?",{"value":112,"source":85,"evidence":113},"Pinecone is not open source",[114],{"id":115,"by":116,"text":117},"37057358","wanderingmind","When there are so many awesome FOSS vector databases available, I wonder what motivated the airbyte team to use Pinecone, the one database that is anti-FOSS?",{"value":119,"source":85,"evidence":120},"Too many moving parts for prototyping",[121],{"id":122,"by":123,"text":124},"37062150","anupsurendran","I feel that there are too many moving pieces here especially for prototyping. There was a much more simpler app recently I took a look at on a recent hackernews post :…",[126,137,145],{"value":127,"source":85,"evidence":128},"Support for pgvector as a destination",[129,133],{"id":130,"by":131,"text":132},"37052651","mritchie712","have you considered supporting pgvector? I'd imagine that'd be easier since you already have pg as a destination.",{"id":134,"by":135,"text":136},"37059448","sandGorgon","hi folks, when will you have pgvector as a destination ? we (https:\u002F\u002Fgithub.com\u002Farakoodev\u002Fedgechains) work with a lot of enterprises and they would not move away from using redis…",{"value":138,"source":85,"evidence":139},"Support for non-OpenAI embedding models",[140,141],{"id":134,"by":135,"text":136},{"id":142,"by":143,"text":144},"37073802","zarazas","What about the embedding?",{"value":146,"source":85,"evidence":147},"Direct Pinecone destination support",[148],{"id":149,"by":150,"text":151},"37052819","rschwabco","A version supporting Pinecone directly is coming soon!",[153,167],{"value":154,"source":85,"evidence":155},"Can local LLMs like Llama 2 be used instead of OpenAI",[156,160,164],{"id":157,"by":158,"text":159},"37051571","jmorgan","LangChain supports local LLMs like Llama 2 with Ollama (https:\u002F\u002Fgithub.com\u002Fjmorganca\u002Follama) as of this morning, in both their Python and Javascript versions:…",{"id":161,"by":162,"text":163},"37059040","samspenc","> - OpenAI: you can host an OSS model if you want to Just to confirm: you mean models like Facebook's Llama 2 and variants right? Since OpenAI hasn't released any OSS models.",{"id":165,"by":11,"text":166},"37064136","correct",{"value":168,"source":85,"evidence":169},"How large a dataset can be submitted",[170,174],{"id":171,"by":172,"text":173},"37051797","johndhi","How large of a dataset can I submit? I have hundreds of thousands of words of text.",{"id":175,"by":11,"text":176},"37052125","Shouldn't have any limits here. Can you let us know how it goes?",[178,185,189],{"value":179,"source":85,"evidence":180},"Weaviate and other vector databases",[181],{"id":182,"by":183,"text":184},"37054784","_pdp_","A fantastic starting point for beginners! Personally, I believe this tutorial provides a solid foundation, but there's so much more to explore. Building something truly effective…",{"value":186,"source":85,"evidence":187},"Ollama for local LLMs",[188],{"id":157,"by":158,"text":159},{"value":190,"source":85,"evidence":191},"Kadoa for data extraction",[192],{"id":193,"by":194,"text":195},"37058575","hubraumhugo","You're right, LLMs are really good at extracting and structuring data from third party sources. I've been working on a \"Zapier for data extraction\" for this reason:…",{"comments":197,"answered":198,"key":199},13,8,[200,203,206],{"id":201,"text":202},"37052146","I am sure we can build something around that. Going to take a look at it. Thanks for mentioning it.",{"id":204,"text":205},"37050767","Let me know how that works out for you and if you would add anything to this tutorial!",{"id":207,"text":208},"37056355","When reading the tutorial, we are describing one stack to build a specific app. But the stack is made of building blocks that you can replace with others if you need to. -…",{"commenters":210,"max_depth":211,"maker_replies":197,"first_comment_mins":18,"lifespan_hours":212},37,4,294,[214,216,218,221,225,229,233,235,237,239,242,245,247,249,251,252,255,257,259,260,261,262,263,265,267,270,273,276,279,281,285,288,290,293,295,299,301,304,308,310,311,313,315,316,320,323,325,327,328,330,333,336,339,341,343,345,349,351],[157,30,32,215,158,30],"project\u002Fneutral\u002Fnone",[201,30,30,217,11,34],"maker\u002Fpositive\u002Fnone",[219,30,32,215,220,30],"37053348","hnhg",[222,30,30,223,224,30],"37053925","commenter\u002Fneutral\u002Fnone","zwaps",[226,30,30,227,228,30],"37058930","other\u002Fneutral\u002Fnone","wahnfrieden",[230,30,32,231,232,30],"37057548","project\u002Fneutral\u002Fperformance","bestcoder69",[234,30,32,215,143,30],"37073728",[88,34,44,236,89,30],"project\u002Fpositive\u002Fdocs",[204,34,30,238,11,34],"maker\u002Fneutral\u002Fnone",[240,34,30,223,241,30],"37055535","BoorishBears",[243,34,30,223,244,30],"37055934","kaveet",[246,34,30,223,89,30],"37056479",[248,34,30,223,89,30],"37056306",[250,34,30,223,89,30],"37056431",[193,34,30,227,194,30],[253,34,30,227,254,30],"37058889","omarfarooq",[115,39,34,256,116,30],"project\u002Fcritical\u002Fnone",[104,32,34,258,105,30],"project\u002Fcritical\u002Fprivacy",[207,32,30,238,11,34],[161,32,32,238,162,30],[165,32,30,238,11,34],[142,32,32,215,143,30],[182,211,32,264,183,30],"project\u002Fmixed\u002Fnone",[266,211,30,238,11,34],"37056102",[268,211,30,223,269,30],"37057416","rahimnathwani",[271,211,30,223,272,30],"37058445","replwoacause",[274,211,30,227,275,30],"37058968","oefnak",[277,211,30,227,278,30],"37059327","fakedang",[134,44,32,280,135,30],"project\u002Fneutral\u002Fidea",[282,283,32,215,284,30],"37203256",6,"amanivan",[122,286,34,287,123,30],7,"project\u002Fcritical\u002Fexecution",[92,198,44,289,93,30],"project\u002Fpositive\u002Fprivacy",[291,198,30,292,11,34],"37056256","maker\u002Fneutral\u002Fprivacy",[108,18,32,294,109,30],"project\u002Fneutral\u002Fprivacy",[296,18,30,297,298,30],"37054946","commenter\u002Fneutral\u002Fprivacy","frankfrank13",[300,18,30,292,11,34],"37056215",[302,18,32,294,303,30],"37057974","toomuchtodo",[305,306,32,280,307,30],"37051606",10,"swyx",[309,306,30,238,11,34],"37052114",[130,306,32,280,131,30],[312,306,30,238,11,34],"37056143",[314,306,30,227,143,30],"37073800",[149,306,32,238,150,30],[317,318,30,227,319,30],"37056141",11,"amelius",[321,318,30,227,322,30],"37056271","kingforaday",[324,318,30,238,11,34],"37056292",[171,326,32,215,172,30],12,[175,326,30,238,11,34],[329,326,32,280,172,30],"37056897",[331,197,30,227,332,30],"37055380","zby",[334,197,30,335,241,30],"37055574","other\u002Fcritical\u002Fnone",[337,197,30,223,338,30],"37058872","electrondood",[340,197,30,223,338,30],"37058858",[96,342,44,236,97,30],14,[344,342,30,238,11,34],"37056959",[346,347,32,215,348,30],"37053474",15,"croes",[350,347,30,238,11,34],"37056322",[352,347,30,223,272,30],"37058463",[],[355,360,365,370,375,379],{"id":356,"name":357,"points":358,"category":359},"36987920","Hydra 1.0",269,"data",{"id":361,"name":362,"points":363,"category":364},"37034951","Phind V2",221,"dev_tools",{"id":366,"name":367,"points":368,"category":369},"37064152","ToSNotify",190,"productivity",{"id":371,"name":372,"points":373,"category":374},"37062422","Putting the \"You\" in CPU",187,"learning_science",{"id":376,"name":377,"points":378,"category":20},"37038053","ChainForge",177,{"id":380,"name":381,"points":382,"category":20},"37034575","Axilla",161,false,"v2",{"id":386,"name":5,"line":6,"ai":383,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":30,"top":18,"kw":387,"maker":11,"domain":10,"tier":19,"attempts":30,"strip":388,"url":9,"mix":389,"praise":84,"objection":101,"sub":21},37050532,[24,25,26,27,28],"3030350000011303000331530333030033000530",[32,33,34,32],1794036639665]