[{"data":1,"prerenderedAt":476},["ShallowReactive",2],{"project-37600019":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":29,"sentiment":30,"stance_mix":31,"scored_comments":36,"aspects":37,"strip":46,"post_text":47,"what":48,"said":104,"maker_in_thread":203,"shape":213,"grid":217,"attempts":438,"same_week":439,"maker_shows":421,"maker_hidden":468,"reader_version":469,"card":470},"37600019","automatic_log_collector_and_analyzer","My Single-File Python Script I Used to Replace Splunk in My Startup","title","Show HN: My Single-File Python Script I Used to Replace Splunk in My Startup","https:\u002F\u002Fgithub.com\u002FDicklesworthstone\u002Fautomatic_log_collector_and_analyzer","github.com","eigenvalue",1695313604,2023,313,79,137,3109,5,"legend","infrastructure","log collection and analysis","cli",[24,25,26,27,28],"log analysis","log collection","splunk alternative","sqlite logs","datasette","roasted",-0.061,{"positive":32,"neutral":33,"mixed":34,"critical":35},7,2,1,10,20,{"idea":38,"execution":41,"docs":42,"design":44,"performance":45},{"positive":39,"critical":40},4,3,{"positive":34,"critical":40},{"positive":43,"critical":34},0,{"positive":43,"critical":34},{"positive":43,"critical":34},[43,43,18,43,43,43,43,43,43,40,34,34,40,18,34,34,43,43,43,43,43,43,43,43,43,43,43,43,43,18,34,43,40,40,43,34,18,43,40,34],"My immediate reaction to today's news that Splunk was being acquired was to comment in the HN discussion for that story:\n\n\"I hated Splunk so much that I spent a couple days a few months ago writing a single 1200 line python script that does absolutely everything I need in terms of automatic log collection, ingestion, and analysis from a fleet of cloud instances. It pulls in all the log lines, enriches them with useful metadata like the IP address of the instance, the machine name, the log source, the datetime, etc. and stores it all in SQlite, which it then exposes to a very convenient web int",{"problem":49,"audience":55,"pricing":62,"motivation":66,"time_to_build":70,"stack":74,"openness":101,"stage":102,"team_size":103},{"value":50,"source":51,"evidence":52},"Collecting and analyzing logs from a small fleet of cloud instances without paying for Splunk","maker",[53],{"id":54},"post",{"value":56,"source":51,"evidence":57},"Small teams with a modest number of machines who dislike Splunk",[58,59],{"id":54},{"id":60,"by":11,"text":61},"37600944","Thanks, based on the dismissive replies to my original comment in the Splunk acquisition discussion, I thought this would get a lot of hostile takes saying that it was dumb, that…",{"value":63,"source":51,"evidence":64},"free",[65],{"id":54},{"value":67,"source":51,"evidence":68},"Replace Splunk at the maker's startup, which was too cumbersome and costly",[69],{"id":54},{"value":71,"source":51,"evidence":72},"a couple days",[73],{"id":54},[75,79,83,87,93],{"value":76,"source":51,"evidence":77},"Python",[78],{"id":54},{"value":80,"source":51,"evidence":81},"SQLite",[82],{"id":54},{"value":84,"source":51,"evidence":85},"Datasette",[86],{"id":54},{"value":88,"source":51,"evidence":89},"Paramiko",[90],{"id":91,"by":11,"text":92},"37603350","Thanks for the feedback. I do use Paramiko for some things. I tried to use it for everything in the project but ran into some weird stuff that wouldn't work reliably for me,…",{"value":94,"source":95,"evidence":96},"Redis","commenter",[97],{"id":98,"by":99,"text":100},"37706937","prabhatsharma","So it requires redis. Would have loved if it was just a simple script or binary.","open_source","launched","unknown",{"praise":105,"objections":138,"requests":171,"questions":172,"alternatives":177},[106,117,127],{"value":107,"source":95,"evidence":108},"Simple, practical tool that suits small-scale needs",[109,113],{"id":110,"by":111,"text":112},"37600818","rollcat","\"This simple tool solves X at my org\" is probably the most underrated type of project. There's not enough room to overcomplicate something that isn't a core part of the business,…",{"id":114,"by":115,"text":116},"37604311","facorreia","It's not that much code and it has sensible function names. I appreciate that OP took the time to share his tool with us.",{"value":118,"source":95,"evidence":119},"Using SQLite and Datasette is a good approach",[120,123],{"id":121,"by":115,"text":122},"37604271","Your project sounds like something that definitely could come in handy. I forked it as a \"bookmark\". I particularly like the idea of storing the data in a local SQLite database.…",{"id":124,"by":125,"text":126},"37601618","simonw","I love this! Log analysis isn't one of the core use-cases for Datasette, but I've done my own experiments with it that have worked pretty well - anything up to 10GB or so of data…",{"value":128,"source":95,"evidence":129},"Thanks for sharing",[130,134],{"id":131,"by":132,"text":133},"37600467","rgrieselhuber","Thanks for sharing.",{"id":135,"by":136,"text":137},"37600516","surfingdino","\"Hell hath no fury like a Python dev annoyed\" :-) Thank you for sharing!",[139,149,156,167],{"value":140,"source":95,"evidence":141},"Does not scale to large enterprise log volumes",[142,146],{"id":143,"by":144,"text":145},"37600773","dsXLII","Volume. 1GB of data per day is rounding error. If you have tens of thousands of servers, each generating hundreds of gigabytes of data per day, tail -f and grep don't scale…",{"id":147,"by":11,"text":148},"37600848","I misspoke there-- meant to say: \"The application has been tested with log files several gigabytes in size from dozens of machines and can process all of it in minutes.\" That's…",{"value":150,"source":95,"evidence":151},"Pull-based collection over SSH is fragile compared to shipping logs from each machine",[152],{"id":153,"by":154,"text":155},"37601568","xorcist","Fetching logs regularly sounds hard? Wouldn't you need to keep track of the position of all files, with heuristics around file rotations? And if something catastrophic happens,…",{"value":157,"source":95,"evidence":158},"Code lacks comments and has unused functions and globals",[159,163],{"id":160,"by":161,"text":162},"37601020","oblvious-earth","Quickly skimming some points that would irratate me if I had to maintain this script: * Importing Paramiko but regularly call `ssh` via subprocess * Unused functions like…",{"id":164,"by":165,"text":166},"37603251","msto","* barely any comments and not a single docstring in the entire kiloline file",{"value":168,"source":95,"evidence":169},"Requires Redis despite being a single script",[170],{"id":98,"by":99,"text":100},[],[173],{"value":174,"source":95,"evidence":175},"How does it handle log file rotation and position tracking?",[176],{"id":153,"by":154,"text":155},[178,185,192,199],{"value":179,"source":95,"evidence":180},"Graylog",[181],{"id":182,"by":183,"text":184},"37602593","runjake","Neat! Definitely a better solution for single source logs. Splunk is ridiculous and Cisco acquiring it isn't going to make that better. For others with a bit more complex needs,…",{"value":186,"source":95,"evidence":187},"Grafana with Graphite",[188],{"id":189,"by":190,"text":191},"37600972","KaiserPro","I used to work at a splunk shop. It was used for alerting, graphing & prediction. It was critical to how the company functioned. There was lots of stuff that relied on splunk,…",{"value":193,"source":95,"evidence":194},"Datadog",[195],{"id":196,"by":197,"text":198},"37600643","hliyan","A long long time ago, I used a series of tail -f's and unix pipes to aggregate logs, and grep, less and awk to analyse them. There were about 20 different services written in…",{"value":200,"source":95,"evidence":201},"Datasette with sqlite-utils",[202],{"id":124,"by":125,"text":126},{"comments":204,"answered":34,"key":205},9,[206,207,210],{"id":60,"text":61},{"id":208,"text":209},"37604654","The real power is that with datasette you can instantly turn that SQLite DB into a full fledged responsive web app with arbitrarily complex filtering and sorting, which all gets…",{"id":211,"text":212},"37604683","Ansible is definitely all about solving from a specified target state and ensuring that it is followed. It’s even at the level of syntax for ansible, which is how it can be used…",{"commenters":214,"max_depth":18,"maker_replies":204,"first_comment_mins":215,"lifespan_hours":216},54,30,192,[218,220,222,226,229,232,234,236,239,241,244,247,251,254,257,261,262,264,266,268,271,274,276,278,281,285,289,291,293,295,299,301,304,307,310,312,315,317,319,321,324,326,327,328,331,334,337,339,342,345,348,350,353,356,359,362,365,368,369,371,372,375,378,381,385,387,390,392,394,397,400,405,408,410,414,417,419,423,426,431,436],[110,43,43,219,111,43],"other\u002Fpositive\u002Fidea",[60,43,43,221,11,34],"maker\u002Fneutral\u002Fnone",[223,43,43,224,225,43],"37601340","other\u002Fneutral\u002Fnone","egwor",[227,43,43,224,228,43],"37602148","RexM",[230,43,43,224,231,43],"37603250","_aavaa_",[121,43,18,233,115,43],"project\u002Fpositive\u002Fidea",[208,43,43,235,11,34],"project\u002Fpositive\u002Fdesign",[237,43,43,224,238,43],"37605411","silasb",[240,43,43,224,111,43],"37606019",[242,43,43,224,243,43],"37601059","olafalo",[245,43,43,224,246,43],"37601225","aaviator42",[248,43,43,249,250,43],"37601374","other\u002Fneutral\u002Fidea","emodendroket",[252,43,43,224,253,43],"37601639","dynisor",[255,43,43,256,154,43],"37601714","commenter\u002Fmixed\u002Fdocs",[258,43,43,259,260,43],"37604512","commenter\u002Fneutral\u002Fnone","matrss",[211,43,43,259,11,34],[263,43,43,259,260,43],"37609248",[265,43,43,259,154,43],"37605414",[267,43,43,259,260,43],"37609466",[269,43,40,270,111,43],"37605836","project\u002Fneutral\u002Fidea",[272,43,43,224,273,43],"37602438","adamrezich",[160,34,34,275,161,43],"project\u002Fcritical\u002Fexecution",[164,34,34,277,165,43],"project\u002Fcritical\u002Fdocs",[279,34,43,280,11,34],"37603600","maker\u002Fneutral\u002Fdocs",[282,34,40,283,284,43],"37604349","maker\u002Fmixed\u002Fdocs","fastasucan",[286,34,43,287,288,43],"37606846","commenter\u002Fneutral\u002Fdocs","msm_",[114,34,18,290,115,43],"project\u002Fpositive\u002Fexecution",[91,34,43,292,11,34],"project\u002Fneutral\u002Fexecution",[294,34,34,275,161,43],"37607304",[296,34,43,297,298,43],"37612712","commenter\u002Fneutral\u002Fexecution","knowsuchagency",[300,34,34,275,161,43],"37625309",[302,33,43,224,303,43],"37601119","CliffStoll",[305,33,43,224,306,43],"37601161","neilk",[308,33,43,224,309,43],"37601208","h0p3",[311,33,43,224,303,43],"37667141",[313,33,43,314,309,43],"37667295","other\u002Fpositive\u002Fnone",[316,33,43,224,303,43],"37668018",[318,33,43,224,11,34],"37604726",[320,33,43,224,303,43],"37667202",[322,33,43,224,323,43],"37641332","mercer",[325,33,43,224,303,43],"37667180",[196,40,43,224,197,43],[143,40,43,224,144,43],[329,40,43,224,330,43],"37600907","vincnetas",[332,40,43,224,333,43],"37601472","jononomo",[335,40,43,224,336,43],"37608458","moqmar",[338,40,43,224,330,43],"37613186",[340,40,43,259,341,43],"37600963","getrealyall",[343,40,43,259,344,43],"37601260","phyrex",[346,40,43,224,347,43],"37602023","10000truths",[349,40,43,224,341,43],"37601034",[351,40,43,224,352,43],"37602640","diarrhea",[354,40,43,224,355,43],"37608714","quickthrower2",[357,40,43,224,358,43],"37601385","fragmede",[360,39,43,224,361,43],"37600509","babuloseo",[363,39,43,224,364,43],"37600691","codetrotter",[366,39,43,224,367,43],"37602748","BerislavLopac",[189,18,43,224,190,43],[124,370,18,233,125,43],6,[182,32,18,233,183,43],[153,373,34,374,154,43],8,"project\u002Fcritical\u002Fidea",[376,373,43,377,11,34],"37602238","project\u002Fneutral\u002Fdesign",[379,373,43,380,154,43],"37602655","commenter\u002Fneutral\u002Fidea",[382,373,43,383,384,43],"37609998","commenter\u002Fneutral\u002Fperformance","hjvalente",[386,373,43,383,11,34],"37614935",[388,373,40,270,389,43],"37618934","johnthescott",[391,373,34,374,154,43],"37658271",[135,204,18,393,136,43],"maker\u002Fpositive\u002Fnone",[395,204,43,259,396,43],"37600743","ozfive",[398,35,43,249,399,43],"37706600","occamsrazorwit",[401,402,43,403,404,43],"37601066",11,"other\u002Fneutral\u002Fpricing","catlover76",[98,406,34,407,99,43],12,"project\u002Fcritical\u002Fdesign",[131,409,18,393,132,43],13,[411,412,43,403,413,43],"37600863",14,"nurettin",[415,412,43,403,416,43],"37601400","candiddevmike",[418,412,43,403,413,43],"37603947",[420,421,34,374,422,43],"37603353",15,"languagehacker",[424,421,18,233,425,43],"37603671","cooper_ganglia",[427,428,34,429,430,43],"37602028",16,"project\u002Fcritical\u002Fnone","dingdong33",[432,433,34,434,435,43],"37600731",17,"project\u002Fcritical\u002Fperformance","Okkef",[147,433,43,437,11,34],"project\u002Fpositive\u002Fperformance",[],[440,445,450,454,458,463],{"id":441,"name":442,"points":443,"category":444},"37613054","Paisa",662,"productivity",{"id":446,"name":447,"points":448,"category":449},"37663270","Raverie",644,"games",{"id":451,"name":452,"points":453,"category":444},"37636701","chatgpt-history-export-to-md",295,{"id":455,"name":456,"points":457,"category":444},"37642671","E-Ink Day Schedule",294,{"id":459,"name":460,"points":461,"category":462},"37661767","Magentic",283,"dev_tools",{"id":464,"name":465,"points":466,"category":467},"37611457","Piano Tabs",222,"learning_science",false,"v2",{"id":471,"name":5,"line":6,"ai":468,"cat":20,"year":13,"points":14,"comments":15,"rec":29,"s":472,"top":18,"kw":473,"maker":11,"domain":10,"tier":19,"attempts":43,"strip":474,"url":9,"mix":475,"praise":107,"objection":140,"sub":21},37600019,-0.06,[24,25,26,27,28],"0050000003113511000000000000051033015031",[32,33,34,35],1794036642158]