[{"data":1,"prerenderedAt":269},["ShallowReactive",2],{"project-15360571":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":34,"aspects":35,"strip":38,"post_text":40,"what":41,"said":70,"maker_in_thread":132,"shape":144,"grid":148,"attempts":233,"same_week":234,"maker_shows":32,"maker_hidden":261,"reader_version":262,"card":263},"15360571","Hivemind","Distributed jobs using AWS Lambda functions","title","Show HN: Hivemind – Distributed jobs using AWS Lambda functions","https:\u002F\u002Fgithub.com\u002Flittlstar\u002Fhivemind","github.com","madamelic",1506631495,2017,66,35,657,2242,30,"notable","infrastructure","distributed job processing","library",[24,25,26,27],"aws lambda","distributed jobs","serverless","scatter gather","mixed",-0.128,{"positive":31,"neutral":32,"mixed":33,"critical":32},1,2,0,5,{"idea":36,"performance":37},{"positive":33,"critical":31},{"positive":33,"critical":31},[39,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,33,39,31,33,33,33,33,33,33,33,33,33,31,33,34],3,null,{"problem":42,"audience":40,"pricing":40,"motivation":53,"time_to_build":40,"stack":57,"openness":67,"stage":68,"team_size":69},{"value":43,"source":44,"evidence":45},"Running large batch jobs in parallel across many AWS Lambda invocations beyond single-function time limits","maker",[46,49],{"id":47,"by":11,"text":48},"15361701",">How is this working for you? Where do you see it going? It actually works quite well. I use it for processing large amounts of data quickly. I originally built this to process…",{"id":50,"by":51,"text":52},"15361007","teej","Thanks for sharing! I ended up building a similar featureset in Python but using invocation type Event instead of RequestResponse. This ended up being a pain in the ass because…",{"value":54,"source":44,"evidence":55},"Maker needed to process large amounts of data quickly; a job estimated at about 14 days was cut to minutes",[56],{"id":47,"by":11,"text":48},[58,62],{"value":59,"source":44,"evidence":60},"Node.js",[61],{"id":47,"by":11,"text":48},{"value":63,"source":64,"evidence":65},"AWS Lambda","commenter",[66],{"id":50,"by":51,"text":52},"open_source","launched","unknown",{"praise":71,"objections":80,"requests":95,"questions":105,"alternatives":110},[72],{"value":73,"source":64,"evidence":74},"Useful for processing large data quickly",[75,76],{"id":47,"by":11,"text":48},{"id":77,"by":78,"text":79},"15361424","dvdhnt","I definitely appreciate that, and I didn’t mean to take anything away from your work. Cheers.",[81,87],{"value":82,"source":64,"evidence":83},"Tied to a single vendor (AWS)",[84],{"id":85,"by":78,"text":86},"15360883","I’m sure this is great, but for some of us, being tied to a single vendor is a non-starter.",{"value":88,"source":64,"evidence":89},"Lambda retry and timeout limits are restrictive",[90,91],{"id":50,"by":51,"text":52},{"id":92,"by":93,"text":94},"15361328","ak217","FYI, I believe you can disable retries and replace them with a DLQ (which you can then ignore, if you don't care about failures). I would be interested, though, if anyone has…",[96],{"value":97,"source":64,"evidence":98},"Support for chaining and orchestration of Lambdas",[99,102],{"id":100,"by":51,"text":101},"15361857","Most of the things I've added are for logging and orchestration. The most important unlock for us was the ability for us to enable chaining of Lambdas that worked seamlessly with…",{"id":103,"by":51,"text":104},"15361753","Amazon doesnt really give you any direct access to the underlying \"worker\" so it doesnt work like that. It's really as simple as: - the active lambda decides to stop performing…",[106],{"value":107,"source":64,"evidence":108},"How is it working in practice and where is it going",[109],{"id":50,"by":51,"text":52},[111,118,125],{"value":112,"source":64,"evidence":113},"PyWren",[114],{"id":115,"by":116,"text":117},"15361056","posnet","More mature python based solution. http:\u002F\u002Fpywren.io\u002F",{"value":119,"source":64,"evidence":120},"AWS Step Functions",[121],{"id":122,"by":123,"text":124},"15361513","joseph","Have you tried Step Functions? I've used them to orchestrate Lambdas with good success when I needed something that will run longer than Lambdas can individually.",{"value":126,"source":64,"evidence":127},"SQS with Lambda",[128],{"id":129,"by":130,"text":131},"15362272","tyingq","There's an SQS\u002FLamba integration. Push 10k messages in a queue as jobs. The lambdas pull them off and delete them once done.…",{"comments":133,"answered":31,"key":134},6,[135,138,141],{"id":136,"text":137},"15364645","The comment was a bit tongue-in-cheek, but I'll accept that I oversold it.",{"id":139,"text":140},"15361711",">The pattern we use is to monitor the execution time left and queue up a follow-up once there's \u003C30s left. Not every job we do can be composed this way but it helps a lot. Hmm.…",{"id":142,"text":143},"15364655","That's... really awesome. I think that may have to be a follow-up feature for this project. Thanks!",{"commenters":145,"max_depth":133,"maker_replies":133,"first_comment_mins":146,"lifespan_hours":147},20,43,50,[149,151,155,159,162,165,166,169,171,173,176,177,181,184,187,189,190,192,194,197,200,203,206,208,210,211,212,214,216,217,221,224,227,229,231],[115,33,39,150,116,33],"project\u002Fneutral\u002Fidea",[152,31,33,153,154,33],"15362852","commenter\u002Fneutral\u002Fexecution","defined",[156,31,33,157,158,33],"15363348","commenter\u002Fneutral\u002Fnone","nikkwong",[160,31,33,157,161,33],"15363591","Kiro",[163,31,33,157,164,33],"15364407","J-dawg",[136,31,33,157,11,31],[167,31,33,153,168,33],"15363646","barryhoodlum",[170,31,33,157,164,33],"15364374",[172,31,33,157,168,33],"15366642",[174,32,33,175,93,33],"15361375","other\u002Fneutral\u002Fidea",[122,32,33,175,123,33],[178,32,33,179,180,33],"15361770","other\u002Fcritical\u002Fpricing","kondro",[182,32,33,179,183,33],"15369022","tfjaeckel",[185,32,33,175,186,33],"15361774","btashton",[188,32,33,175,51,33],"15361539",[129,32,33,175,130,33],[191,32,33,175,51,33],"15363138",[193,32,33,175,130,33],"15364436",[195,39,33,175,196,33],"15361975","zachrose",[198,39,33,175,199,33],"15362006","empath75",[201,39,33,175,202,33],"15375430","snaky",[50,204,39,205,51,33],4,"maker\u002Fneutral\u002Fidea",[92,204,31,207,93,33],"project\u002Fcritical\u002Fperformance",[209,204,33,175,51,33],"15361522",[139,204,33,205,11,31],[103,204,33,175,51,33],[142,204,33,213,11,31],"maker\u002Fpositive\u002Fidea",[47,204,33,215,11,31],"maker\u002Fpositive\u002Fperformance",[100,204,33,175,51,33],[218,34,33,219,220,33],"15361476","other\u002Fneutral\u002Fnone","Edmond",[222,34,33,223,11,31],"15361635","maker\u002Fneutral\u002Fnone",[225,34,33,219,226,33],"15364024","kitd",[85,133,31,228,78,33],"project\u002Fcritical\u002Fidea",[230,133,33,223,11,31],"15360899",[77,133,34,232,78,33],"maker\u002Fpositive\u002Fnone",[],[235,240,244,249,253,257],{"id":236,"name":237,"points":238,"category":239},"15393446","Thousand Ether Homepage",344,"creative",{"id":241,"name":242,"points":243,"category":239},"15383336","fast-style-transfer-deeplearnjs",266,{"id":245,"name":246,"points":247,"category":248},"15357394","RMStudio",154,"dev_tools",{"id":250,"name":251,"points":252,"category":248},"15367645","ARKit Playground",151,{"id":254,"name":255,"points":256,"category":248},"15386981","InstaBuild",124,{"id":258,"name":259,"points":260,"category":248},"15356599","Sitebulb",120,false,"v2",{"id":264,"name":5,"line":6,"ai":261,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":265,"top":18,"kw":266,"maker":11,"domain":10,"tier":19,"attempts":33,"strip":267,"url":9,"mix":268,"praise":73,"objection":82,"sub":21},15360571,-0.13,[24,25,26,27],"30000000000000000000031000000000105",[31,32,33,32],1794036559360]