[{"data":1,"prerenderedAt":457},["ShallowReactive",2],{"project-34970917":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":34,"aspects":35,"strip":47,"post_text":49,"what":50,"said":86,"maker_in_thread":196,"shape":209,"grid":212,"attempts":418,"same_week":419,"maker_shows":39,"maker_hidden":449,"reader_version":450,"card":451},"34970917","Crul","Query Any Webpage or API","title","Show HN: Crul – Query Any Webpage or API","https:\u002F\u002Fwww.crul.com\u002F","crul.com","portInit",1677600778,2023,241,79,205,3109,7,"notable","dev_tools","web scraping and api query tool","desktop_app",[24,25,26,27],"web scraping","api data extraction","webpage to csv","data feeds","mixed",-0.029,{"positive":18,"neutral":31,"mixed":32,"critical":33},19,2,12,40,{"idea":36,"execution":38,"design":41,"pricing":43,"docs":44,"privacy":45,"performance":46},{"positive":37,"critical":37},3,{"positive":39,"critical":40},1,6,{"positive":42,"critical":39},0,{"positive":42,"critical":39},{"positive":39,"critical":42},{"positive":42,"critical":42},{"positive":42,"critical":39},[48,37,42,42,48,37,42,39,37,39,37,37,39,37,48,37,37,37,48,37,37,39,39,48,37,48,37,37,42,42,42,39,42,39,39,37,37,39,48,39],5,null,{"problem":51,"audience":49,"pricing":58,"motivation":68,"time_to_build":49,"stack":72,"openness":83,"stage":84,"team_size":85},{"value":52,"source":53,"evidence":54},"Getting data out of webpages and APIs is hard, brittle and doesn't scale","maker",[55],{"id":56,"by":11,"text":57},"34970922","Hi HN, we’re Carl and Nic, the creators of crul (https:\u002F\u002Fwww.crul.com), and we’ve been hard at work for the last year and a half building our dream of turning the web into a…",{"value":59,"source":53,"evidence":60},"Free to query; scheduler and domain policies are enterprise features",[61,62,65],{"id":56,"by":11,"text":57},{"id":63,"by":11,"text":64},"34975021","Yes that is possible, although we currently have the scheduler set as an enterprise feature. We should look into a free trial, I enjoyed the flow of the EasyDataTransform…",{"id":66,"by":11,"text":67},"34974795","Yeah the default domain throttle policy is 1 req per second per domain. Configurable through domain policies https:\u002F\u002Fwww.crul.com\u002Fdocs\u002Ffeatures\u002Fdomain-policies - although…",{"value":69,"source":53,"evidence":70},"Founders struggled wrangling web data with scraping tools and scripts",[71],{"id":56,"by":11,"text":57},[73,77],{"value":74,"source":53,"evidence":75},"Node.js",[76],{"id":66,"by":11,"text":67},{"value":78,"source":53,"evidence":79},"Puppeteer headless browser",[80],{"id":81,"by":11,"text":82},"34973403","It's a tricky question. Part of it is looking at APIs as the main source of data for scheduled queries\u002Fdata feeds. Crul sort of operates as a text only browser when interacting…","unknown","launched","team",{"praise":87,"objections":103,"requests":140,"questions":164,"alternatives":176},[88,96],{"value":89,"source":90,"evidence":91},"Filter command is powerful for exploring data","commenter",[92],{"id":93,"by":94,"text":95},"34971106","dishwishy","I just started playing with CRUL recently to try to map out different media download links on various websites, in an effort to avoid clicking around looking for hidden content.…",{"value":97,"source":90,"evidence":98},"Slick tool",[99],{"id":100,"by":101,"text":102},"34975742","the_giver","That's a pretty slick tool you guys built!",[104,115,122,133],{"value":105,"source":90,"evidence":106},"Closed source, though open sourcing is planned",[107,111],{"id":108,"by":109,"text":110},"34977350","2h","looks like it is closed source: https:\u002F\u002Fgithub.com\u002Fcrul-oss",{"id":112,"by":113,"text":114},"34979124","imiric","This looks very cool. Congrats on the launch! Have you thought about open sourcing it? I realize you want to sustain a business, but an open core model with commercial features…",{"value":116,"source":90,"evidence":117},"Slow default throttle of one request per second per domain",[118],{"id":119,"by":120,"text":121},"34974690","KomoD","> status: complete (28.284 seconds \u002F 14 results) Why was it so slow? Is there a default delay between requests or something?",{"value":123,"source":90,"evidence":124},"Struggles with nested JSON and structural website changes",[125,129],{"id":126,"by":127,"text":128},"34979895","djhn","I played around with this trying it on a few tricky cases. At least for the initial step of getting the data in a tidy format, it wasn't immediately obvious to me how crul could…",{"id":130,"by":131,"text":132},"34972781","jawns","How does crul handle the dynamic nature of the web? Yes, content changes, but so does structure. If I'm interested in content that shows up in a news feed div, and that div is…",{"value":134,"source":90,"evidence":135},"Name is too close to curl",[136],{"id":137,"by":138,"text":139},"34986709","netsharc","Man, this looks cool, the name's closeness to \"curl\" irritates me..",[141,146,157],{"value":142,"source":90,"evidence":143},"Open source the project",[144,145],{"id":112,"by":113,"text":114},{"id":108,"by":109,"text":110},{"value":147,"source":90,"evidence":148},"Add a Crul datasource or FDW integration",[149,153],{"id":150,"by":151,"text":152},"34972765","jensneuse","Hey, just watched the video. This looks super useful! I'm the founder of WunderGraph (https:\u002F\u002Fwundergraph.com) and we allow our users to easily integrate multiple data sources…",{"id":154,"by":155,"text":156},"34976121","chatmasta","I'll make this same offer for Splitgraph :) If you feel like writing a Postgres FDW then we can add it to the engine on the backend, so that anyone with a Postgres client could…",{"value":158,"source":90,"evidence":159},"Support for HTML table extraction",[160],{"id":161,"by":162,"text":163},"34975858","throwaway_e9463","Do you have an example of how to turn an html table element into a CSV? I saw the open and the scrape commands, but wasn't sure where to go from there.",[165,172],{"value":166,"source":90,"evidence":167},"How to handle anti-bot blocking",[168],{"id":169,"by":170,"text":171},"34973272","asdadsdad","How do you plan on tackling anti-bot blocking?",{"value":173,"source":90,"evidence":174},"How it handles changes in web page structure",[175],{"id":130,"by":131,"text":132},[177,184,188,192],{"value":178,"source":90,"evidence":179},"curl and jq",[180],{"id":181,"by":182,"text":183},"34976006","1vuio0pswjnm7","Please share some data wrangling challenges that support this statement: \"The reality is that shit is hard, doesn't scale (classic blocking for-loop or async saturation), and…",{"value":185,"source":53,"evidence":186},"Puppeteer, Playwright, Selenium",[187],{"id":56,"by":11,"text":57},{"value":189,"source":53,"evidence":190},"Postman",[191],{"id":56,"by":11,"text":57},{"value":193,"source":90,"evidence":194},"Airbyte and Splitgraph",[195],{"id":154,"by":155,"text":156},{"comments":197,"answered":198,"key":199},28,14,[200,203,206],{"id":201,"text":202},"34972806","Will absolutely reach out! Our experience has been that just getting data is often really challenging, so we've really focused on that piece, and being able to easily share with…",{"id":204,"text":205},"34976262","We'll have to see how close our current postgres integration is! Would like to understand more and will reach out.",{"id":207,"text":208},"34977713","We hand wrote a number of integrations, sometimes it was a simple as reusing a schema with slightly different values, we are also using the awesome https:\u002F\u002Fwww.benthos.dev\u002F!",{"commenters":210,"max_depth":40,"maker_replies":197,"first_comment_mins":42,"lifespan_hours":211},36,33,[213,215,217,219,220,223,226,227,230,232,235,237,239,242,244,248,250,253,255,258,261,263,266,268,271,272,274,277,280,282,286,288,291,293,297,300,303,307,311,313,315,317,320,323,325,327,329,331,334,336,338,340,343,345,349,351,353,354,357,360,362,364,368,371,373,376,378,381,383,386,388,392,394,397,400,403,406,409,414,416],[150,42,48,214,151,42],"project\u002Fpositive\u002Fidea",[201,42,42,216,11,39],"maker\u002Fneutral\u002Fnone",[154,42,37,218,155,42],"project\u002Fneutral\u002Fidea",[204,42,42,216,11,39],[221,42,42,222,155,42],"34976439","commenter\u002Fneutral\u002Fnone",[224,42,42,222,225,42],"34977582","dan_rock_wilson",[207,42,42,216,11,39],[228,42,42,229,225,42],"34977828","other\u002Fneutral\u002Fnone",[93,39,48,231,94,42],"project\u002Fpositive\u002Fexecution",[233,39,42,234,11,39],"34971498","maker\u002Fneutral\u002Fdocs",[130,32,37,236,131,42],"project\u002Fneutral\u002Fexecution",[238,32,42,216,11,39],"34972855",[240,32,42,229,241,42],"34973960","robbs",[243,32,42,222,11,39],"34974053",[245,37,39,246,247,42],"34977290","project\u002Fcritical\u002Fexecution","ElijahLynn",[249,37,39,246,247,42],"34977302",[251,37,42,252,11,39],"34977581","maker\u002Fneutral\u002Fexecution",[254,37,37,236,247,42],"34985141",[256,37,37,236,257,42],"34978272","Brajeshwar",[181,259,39,260,182,42],4,"project\u002Fcritical\u002Fidea",[262,259,42,216,11,39],"34976227",[264,259,37,236,265,42],"34976576","nine_k",[267,259,37,236,182,42],"34977485",[269,259,37,270,265,42],"34977684","project\u002Fneutral\u002Fnone",[126,48,39,246,127,42],[273,48,42,234,11,39],"34980006",[275,48,37,276,127,42],"34980814","project\u002Fmixed\u002Fexecution",[278,48,37,236,279,42],"34980532","RobotToaster",[281,48,42,222,127,42],"34980645",[283,40,48,284,285,42],"34974612","maker\u002Fpositive\u002Fnone","Uptrenda",[287,40,42,216,11,39],"34974749",[289,18,37,270,290,42],"34973741","jwx48",[292,18,42,216,11,39],"34973797",[294,18,37,295,296,42],"34977597","project\u002Fneutral\u002Fdesign","orangutanemoji",[298,18,37,295,299,42],"34977803","docyes",[301,18,37,295,302,42],"34978411","MetaMonk",[304,305,37,236,306,42],"34980217",8,"yevpats",[308,309,48,214,310,42],"34974860",9,"hermitcrab",[312,309,42,216,11,39],"34974927",[314,309,37,270,310,42],"34974980",[63,309,42,316,11,39],"maker\u002Fneutral\u002Fpricing",[318,309,37,319,310,42],"34975118","project\u002Fneutral\u002Fpricing",[321,322,39,246,257,42],"34978263",10,[324,322,42,252,11,39],"34978297",[326,322,39,246,257,42],"34978322",[328,322,42,252,11,39],"34978390",[112,330,48,214,113,42],11,[332,330,42,333,11,39],"34979197","maker\u002Fneutral\u002Fidea",[335,330,42,222,113,42],"34980503",[161,33,37,337,162,42],"project\u002Fneutral\u002Fdocs",[339,33,42,234,11,39],"34976026",[100,341,48,342,101,42],13,"project\u002Fpositive\u002Fnone",[344,341,42,284,11,39],"34975774",[346,198,37,347,348,42],"34977090","project\u002Fneutral\u002Fprivacy","codetrotter",[350,198,42,216,11,39],"34977123",[169,352,37,236,170,42],15,[81,352,42,252,11,39],[355,352,42,229,356,42],"34974611","wefarrell",[358,352,42,229,359,42],"34975039","pocket_cheese",[361,352,42,229,356,42],"34975634",[363,352,42,222,170,42],"34976000",[365,352,42,366,367,42],"34973527","commenter\u002Fcritical\u002Fidea","is_true",[119,369,39,370,120,42],16,"project\u002Fcritical\u002Fperformance",[66,369,42,372,11,39],"project\u002Fneutral\u002Fperformance",[374,369,42,222,375,42],"34975465","mdaniel",[377,369,42,236,11,39],"34975629",[379,369,39,380,120,42],"34976087","project\u002Fcritical\u002Fpricing",[382,369,42,316,11,39],"34976491",[384,369,39,260,385,42],"34978218","curiousgeorgio",[387,369,42,222,11,39],"34978310",[389,390,37,218,391,42],"34978259",17,"zerop",[393,390,42,234,11,39],"34978286",[395,390,37,396,391,42],"34978346","project\u002Fmixed\u002Fidea",[398,390,42,399,11,39],"34978416","maker\u002Fpositive\u002Fdocs",[137,401,39,402,138,42],18,"project\u002Fcritical\u002Fdesign",[404,31,39,246,405,42],"34979088","demarq",[407,31,42,408,11,39],"34979161","maker\u002Fpositive\u002Fexecution",[410,411,48,412,413,42],"34990569",20,"project\u002Fpositive\u002Fdocs","markandrewj",[108,415,39,260,109,42],21,[56,417,42,216,11,39],22,[],[420,425,430,435,440,444],{"id":421,"name":422,"points":423,"category":424},"34908528","Mox",471,"infrastructure",{"id":426,"name":427,"points":428,"category":429},"34971579","Scribble Diffusion",373,"ai",{"id":431,"name":432,"points":433,"category":434},"34929968","Painted Ocean",359,"games",{"id":436,"name":437,"points":438,"category":439},"34913485","lwp",225,"creative",{"id":441,"name":442,"points":443,"category":434},"34987834","WeChatGPT+",195,{"id":445,"name":446,"points":447,"category":448},"34972839","Briefsky",172,"consumer_social",false,"v2",{"id":452,"name":5,"line":6,"ai":449,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":453,"top":18,"kw":454,"maker":11,"domain":10,"tier":19,"attempts":42,"strip":455,"url":9,"mix":456,"praise":89,"objection":105,"sub":21},34970917,-0.03,[24,25,26,27],"5300530131331353335331153533000101133151",[18,31,32,33],1793991061143]