[{"data":1,"prerenderedAt":768},["ShallowReactive",2],{"project-23255732":3},{"id":4,"name":5,"line":6,"line_source":7,"title":8,"url":9,"domain":10,"maker":11,"posted_at":12,"year":13,"points":14,"comments":15,"rank_in_year":16,"shown_in_year":17,"top":18,"tier":19,"category":20,"subcategory":21,"format":22,"keywords":23,"reception":28,"sentiment":29,"stance_mix":30,"scored_comments":35,"aspects":36,"strip":47,"post_text":50,"what":51,"said":84,"maker_in_thread":195,"shape":198,"grid":203,"attempts":731,"same_week":732,"maker_shows":40,"maker_hidden":760,"reader_version":761,"card":762},"23255732","newscatcher","Python package to collect normalized news from almost any website","title","Show HN: Python package to collect normalized news from almost any website","https:\u002F\u002Fgithub.com\u002Fkotartemiy\u002Fnewscatcher\u002Fblob\u002Fmaster\u002FREADME.md","github.com","kizy25",1590046051,2020,420,194,77,3028,3,"legend","productivity","news aggregation","library",[24,25,26,27],"news aggregator","python news scraper","rss feeds","news api","curious",0.017,{"positive":31,"neutral":32,"mixed":33,"critical":34},20,44,0,16,80,{"idea":37,"execution":39,"design":42,"performance":43,"docs":44,"pricing":45,"privacy":46},{"positive":38,"critical":38},9,{"positive":40,"critical":41},1,5,{"positive":18,"critical":33},{"positive":33,"critical":40},{"positive":40,"critical":33},{"positive":40,"critical":40},{"positive":33,"critical":33},[48,18,33,33,33,33,33,41,18,18,33,33,33,33,33,33,33,33,18,33,41,40,48,18,18,49,18,49,18,49,49,49,48,41,41,33,48,48,48,49],4,2,null,{"problem":50,"audience":50,"pricing":52,"motivation":59,"time_to_build":50,"stack":63,"openness":81,"stage":82,"team_size":83},{"value":53,"source":54,"evidence":55},"Python package is free with no limits; a separate paid newscatcher API is planned.","maker",[56],{"id":57,"by":11,"text":58},"23256208","Hey. Newscatcher (python) is an open-sourced package that we developed for users' side projects. There are no limits. You can even modify it for your own needs. Newscatcher API…",{"value":60,"source":54,"evidence":61},"Built for users' side projects; the paid API is the commercial product.",[62],{"id":57,"by":11,"text":58},[64,76],{"value":65,"source":66,"evidence":67},"Python","commenter",[68,72],{"id":69,"by":70,"text":71},"23256378","raziel2p","This doesn't \"collect\" anything, it's just a python package wrapped on top of a sqlite database. If you choose an arbitrary website that doesn't have information in said database…",{"id":73,"by":74,"text":75},"23257973","nojito","This is literally just a sqlite db with rss feeds and a python script with a very misleading description. The real purpose of this post is to get traffic for their paid news api.…",{"value":77,"source":66,"evidence":78},"SQLite",[79,80],{"id":69,"by":70,"text":71},{"id":73,"by":74,"text":75},"open_source","launched","unknown",{"praise":85,"objections":101,"requests":126,"questions":142,"alternatives":172},[86,97],{"value":87,"source":66,"evidence":88},"Useful metadata like language, topic and RSS URL.",[89,93],{"id":90,"by":91,"text":92},"23257725","tyingq","The metadata of language, topic, rss url, etc, is nice work. For use outside of python, here's a gist with a sorted list of sites and a sqlite dump of the site data:…",{"id":94,"by":95,"text":96},"23255962","anonytrary","The demo gif is really long (a few minutes), but it's well worth watching and summarizes the capabilities of the library well. It'd be cool if you could register additional…",{"value":98,"source":66,"evidence":99},"Demo gif summarizes the library well.",[100],{"id":94,"by":95,"text":96},[102,111,122],{"value":103,"source":66,"evidence":104},"Mostly a wrapper over an RSS sqlite database, not real collection from any website.",[105,109,110],{"id":106,"by":107,"text":108},"23257098","kingofpandora","> Programmatically collect normalized news from (almost) any website. By this you mean ... Fetch RSS feeds from the websites that we've hardcoded into a Python library? It seems…",{"id":69,"by":70,"text":71},{"id":73,"by":74,"text":75},{"value":112,"source":66,"evidence":113},"Only RSS summaries, not full article text.",[114,118],{"id":115,"by":116,"text":117},"23273262","charlesdaniels","As others have noted, this doesn't seem to collect the full article text, just stuff that you would get from an RSS fee. From the title, I expected something more like…",{"id":119,"by":120,"text":121},"23256359","nmstoker","How complete are the articles that are returned? Last time I looked into RSS news feeds a lot of sources would just put abbreviated \u002F teaser content in and then try to get you to…",{"value":123,"source":66,"evidence":124},"Description is misleading.",[125],{"id":73,"by":74,"text":75},[127,134],{"value":128,"source":66,"evidence":129},"Provide a list of supported websites.",[130],{"id":131,"by":132,"text":133},"23257112","gitgud","> \"Almost any website\" Is there a list of supported websites?",{"value":135,"source":66,"evidence":136},"Allow registering additional sources via API or config.",[137,138],{"id":94,"by":95,"text":96},{"id":139,"by":140,"text":141},"23257390","0x006A","Why did you choose to ship the list of RSS feeds as an SQL database? This makes it hard to keep it up to date and submit pull requests with additional sources. Would it not be…",[143,154,161],{"value":144,"source":66,"evidence":145},"How does it compare to newspaper3k?",[146,150],{"id":147,"by":148,"text":149},"23259243","zzuko","There is a similar library for this called newspaper which I had used in my undergrad thesis. Not dismissing this work, but I am curious about what it is offering on top of it?…",{"id":151,"by":152,"text":153},"23257836","jerzyt","First, kudos to the OP. If you don't mind, would you compare and contrast it to newspaper3k?",{"value":155,"source":66,"evidence":156},"How is content licensed and is RSS access free?",[157],{"id":158,"by":159,"text":160},"23258253","fuball63","This looks like a cool library, but I have a question about the newscatcher API. How does the licensing work for the content? Seems odd (but great) that I can just read the news…",{"value":162,"source":66,"evidence":163},"What does normalized news mean?",[164,168],{"id":165,"by":166,"text":167},"23259118","hitpointdrew","What is \"normalized news\"? What does that even mean?",{"id":169,"by":170,"text":171},"23258459","pheme1","What does the normalized news means? Able to query from a predefined RSS feed? Because I was expecting news coverage from different partisan source pointing to the same news event.",[173,181,188],{"value":174,"source":66,"evidence":175},"newspaper3k",[176,180],{"id":177,"by":178,"text":179},"23256389","pmyteh","Depending on what you're trying to do, there's also newspaper3k: https:\u002F\u002Fgithub.com\u002Fcodelucas\u002Fnewspaper It's quite easy to get \"good\" extraction for large numbers of…",{"id":147,"by":148,"text":149},{"value":182,"source":66,"evidence":183},"newsapi.org",[184],{"id":185,"by":186,"text":187},"23256052","permanent","Hi looks interesting and useful! What's the differences between newscatcher (python) vs. newscatcherapi.com? Is there any limit on using newscatcher (python) as shown in the…",{"value":189,"source":66,"evidence":190},"Feedity",[191],{"id":192,"by":193,"text":194},"23256874","nreece","This looks great! I read your blog post about how open-sourcing helped you find testers, which is a great step for such tools. * Shameless plug *: Our web service, Feedity -…",{"comments":40,"answered":40,"key":196},[197],{"id":57,"text":58},{"commenters":199,"max_depth":200,"maker_replies":40,"first_comment_mins":201,"lifespan_hours":202},123,8,37,151,[204,206,210,213,217,221,224,228,231,235,238,241,244,246,249,251,253,256,259,263,266,269,272,274,276,278,281,284,287,290,293,296,299,301,303,306,308,311,314,317,320,323,327,330,332,335,337,340,343,346,348,350,352,354,356,358,361,364,366,369,372,374,377,379,381,384,387,390,393,396,399,402,404,406,408,411,414,417,419,421,424,426,429,431,434,437,440,442,445,448,451,453,456,459,462,464,467,471,474,477,480,483,486,489,493,496,499,502,505,507,508,511,513,515,516,519,522,526,527,530,532,535,538,542,545,547,549,552,555,558,560,561,564,567,569,572,574,577,578,581,584,586,588,592,595,597,600,602,606,608,610,612,614,616,618,620,624,625,629,631,635,638,639,642,645,649,653,655,658,662,664,667,669,673,675,679,681,683,688,692,695,699,701,705,708,711,714,716,720,722,724,728],[106,33,40,205,107,33],"project\u002Fcritical\u002Fidea",[207,40,41,208,209,33],"23256002","project\u002Fpositive\u002Fidea","oceanbreeze83",[211,40,41,208,212,33],"23256098","jsilence",[214,40,18,215,216,33],"23256280","project\u002Fneutral\u002Fidea","heisenzombie",[218,40,33,219,220,33],"23256510","other\u002Fneutral\u002Fnone","scrollaway",[222,40,33,219,223,33],"23257628","eeZah7Ux",[225,40,41,226,227,33],"23256285","project\u002Fpositive\u002Fexecution","artembugara",[229,40,40,205,230,33],"23256310","doublesCs",[232,40,33,233,234,33],"23256392","commenter\u002Fneutral\u002Fnone","trabant00",[236,40,33,233,237,33],"23256409","p2detar",[239,40,33,233,240,33],"23256906","dragonwriter",[242,40,33,233,243,33],"23257451","eloisius",[245,40,33,233,240,33],"23258548",[247,40,33,219,248,33],"23256712","ekianjo",[250,40,33,219,230,33],"23256785",[252,40,33,219,248,33],"23282918",[254,40,33,219,255,33],"23256894","lucideer",[257,40,33,219,258,33],"23257026","luckylion",[260,40,33,261,262,33],"23256827","commenter\u002Fneutral\u002Fidea","the_duke",[264,40,33,219,265,33],"23257035","jeegsy",[267,40,33,219,268,33],"23257575","foobarbecue",[270,40,33,219,271,33],"23257139","yomly",[273,40,33,233,230,33],"23257398",[275,40,33,233,271,33],"23257620",[277,40,33,219,243,33],"23257409",[279,40,33,219,280,33],"23257853","wpietri",[282,40,33,219,283,33],"23258126","ineedasername",[285,40,33,219,286,33],"23259166","hunters_tryst",[288,40,33,261,289,33],"23257525","textgel",[291,40,33,219,292,33],"23256526","sabertoothed",[294,40,33,219,295,33],"23256550","freddie_mercury",[297,40,33,219,298,33],"23256668","mahalol",[300,40,33,233,292,33],"23256713",[302,40,33,219,295,33],"23257154",[304,40,33,233,305,33],"23257343","dd36",[307,40,33,219,292,33],"23256693",[309,40,33,219,310,33],"23257657","oblio",[312,40,41,208,313,33],"23256660","sdan",[315,40,40,205,316,33],"23259117","marcosdumay",[318,40,18,215,319,33],"23256857","swah",[321,40,33,219,322,33],"23256969","CuriousSkeptic",[324,40,18,325,326,33],"23256999","project\u002Fneutral\u002Fdesign","demirev",[328,40,41,329,209,33],"23260652","maker\u002Fpositive\u002Fdesign",[331,40,41,208,243,33],"23257476",[333,40,40,205,334,33],"23257603","SoylentOrange",[336,40,33,233,74,33],"23257820",[338,40,33,219,339,33],"23259625","PaulHoule",[341,40,33,233,342,33],"23257873","nautilus12",[344,40,33,233,345,33],"23258144","aidanlister",[347,40,33,233,289,33],"23258232",[349,40,33,233,283,33],"23258447",[351,40,33,233,289,33],"23258941",[353,40,33,233,283,33],"23264737",[355,40,33,233,334,33],"23258196",[357,40,33,233,283,33],"23258300",[359,40,33,233,360,33],"23257879","jumpman500",[362,40,33,233,363,33],"23258503","B4CKlash",[365,40,33,233,360,33],"23259890",[367,40,33,219,368,33],"23257972","Trumpi",[370,40,33,233,371,33],"23258140","johnday",[373,40,33,233,368,33],"23263142",[375,40,33,233,376,33],"23258023","mark_l_watson",[378,40,33,233,334,33],"23258422",[380,40,33,233,376,33],"23258533",[382,40,33,219,383,33],"23258948","Kuzutsukake",[385,40,33,233,386,33],"23263309","ComputerGuru",[388,40,33,233,389,33],"23260705","kkotak",[391,40,33,233,392,33],"23264651","jhelphenstine",[394,40,33,233,395,33],"23265985","deadbunny",[397,40,33,233,398,33],"23263032","billme",[400,40,33,233,401,33],"23263241","artificial",[403,40,33,233,398,33],"23263819",[405,40,33,233,401,33],"23265389",[407,40,33,219,386,33],"23263290",[409,40,33,219,410,33],"23258793","dgb23",[412,40,33,233,413,33],"23259025","ivanhoe",[415,40,33,233,416,33],"23259694","shuntress",[418,40,33,219,413,33],"23262761",[420,40,33,219,410,33],"23261079",[422,40,33,219,423,33],"23258164","Dansvidania",[425,40,33,233,413,33],"23258784",[427,40,33,233,428,33],"23259855","runawaybottle",[430,40,33,233,416,33],"23260026",[432,40,33,219,433,33],"23258798","fenwick67",[435,40,33,233,436,33],"23263222","Spivak",[438,40,33,233,439,33],"23259540","etangent",[441,40,33,233,416,33],"23259769",[443,40,33,233,444,33],"23260415","malinens",[446,40,18,215,447,33],"23260024","prox",[449,40,33,219,450,33],"23260141","tacon",[452,40,33,219,334,33],"23266955",[454,40,33,219,455,33],"23260301","tarkin2",[457,40,33,233,458,33],"23260918","froh",[460,40,33,233,461,33],"23261384","vadansky",[463,40,33,233,398,33],"23262952",[465,40,33,219,466,33],"23265650","rsynnott",[468,40,18,469,470,33],"23258213","project\u002Fneutral\u002Fnone","stronglikedan",[472,40,33,219,473,33],"23264103","totemandtoken",[475,40,41,208,476,33],"23258306","mguerville",[478,40,33,219,479,33],"23258513","pjc50",[481,40,33,219,482,33],"23258713","chrisweekly",[484,40,33,219,485,33],"23258957","chanmad29",[487,40,40,205,488,33],"23259895","mistermann",[490,40,40,491,492,33],"23259043","project\u002Fcritical\u002Fperformance","anaphor",[494,40,18,215,495,33],"23260396","chris_f",[497,40,41,498,209,33],"23260680","maker\u002Fpositive\u002Fidea",[500,40,41,329,501,33],"23265562","justinmiller",[503,40,33,219,504,33],"23266061","cloudwizard",[73,49,40,506,74,33],"project\u002Fcritical\u002Fexecution",[177,18,33,219,178,33],[509,18,18,510,152,33],"23257896","maker\u002Fneutral\u002Fexecution",[512,18,18,510,178,33],"23258317",[514,18,33,219,473,33],"23264008",[69,48,40,506,70,33],[517,48,41,518,227,33],"23256715","maker\u002Fpositive\u002Fnone",[520,48,18,215,521,33],"23257827","justaguyhere",[523,48,18,524,525,33],"23258043","project\u002Fneutral\u002Fexecution","slightwinder",[158,41,18,215,159,33],[528,41,33,219,529,33],"23258340","dunefox",[531,41,33,219,159,33],"23258710",[533,41,33,233,534,33],"23259930","carapace",[536,41,18,537,227,33],"23260481","maker\u002Fneutral\u002Fnone",[539,41,40,540,541,33],"23266191","maker\u002Fcritical\u002Fexecution","hundchenkatze",[543,41,18,544,227,33],"23270104","maker\u002Fneutral\u002Fdocs",[139,546,18,524,140,33],6,[548,546,18,510,227,33],"23257712",[550,546,18,510,551,33],"23265806","remram",[553,546,40,506,554,33],"23324066","pictuga",[90,556,41,557,91,33],7,"project\u002Fpositive\u002Fnone",[185,200,18,559,186,33],"project\u002Fneutral\u002Fpricing",[57,200,33,559,11,40],[562,200,18,215,563,33],"23257624","darwinwhy",[565,200,18,566,227,33],"23257737","maker\u002Fneutral\u002Fidea",[568,200,40,205,563,33],"23268118",[570,200,41,571,227,33],"23256299","project\u002Fpositive\u002Fpricing",[573,200,40,205,170,33],"23258612",[575,200,18,215,576,33],"23306559","bussyscorp",[119,38,18,524,120,33],[579,38,33,219,580,33],"23256575","k1m",[582,583,18,524,140,33],"23257399",10,[585,583,18,510,227,33],"23257718",[115,587,40,506,116,33],11,[589,590,33,219,591,33],"23257901",12,"uptown",[593,594,18,215,525,33],"23257091",13,[596,594,18,215,227,33],"23257104",[598,594,40,599,525,33],"23257141","project\u002Fcritical\u002Fpricing",[601,594,18,559,227,33],"23257306",[603,604,18,524,605,33],"23256755",14,"kuu",[607,604,18,524,605,33],"23256810",[609,604,18,524,227,33],"23256814",[611,604,40,205,525,33],"23257000",[613,604,33,233,227,33],"23257316",[615,604,33,261,525,33],"23257379",[617,604,18,537,227,33],"23257726",[619,604,41,518,605,33],"23269664",[621,622,18,469,623,33],"23258559",15,"polymorph1sm",[169,34,18,215,170,33],[626,627,41,208,628,33],"23256563",17,"forgingahead",[192,630,41,557,193,33],18,[632,633,33,219,634,33],"23261720",19,"ryanalam",[636,633,33,219,637,33],"23263402","modlinska",[147,31,18,215,148,33],[94,640,41,641,95,33],21,"project\u002Fpositive\u002Fdesign",[643,640,41,208,644,33],"23256642","pydry",[646,647,33,219,648,33],"23256172",22,"marban",[650,647,33,651,652,33],"23256474","other\u002Fcritical\u002Fdesign","rawoke083600",[654,647,33,219,648,33],"23257130",[656,647,33,651,657,33],"23256658","zwaps",[659,647,33,660,661,33],"23256858","other\u002Fneutral\u002Fdesign","zo1",[663,647,33,660,648,33],"23257932",[665,647,33,651,666,33],"23260187","allarm",[668,647,33,660,648,33],"23257143",[670,671,41,557,672,33],"23257933",23,"A4ET8a8uTh0",[151,674,18,215,152,33],24,[676,677,18,469,678,33],"23256199",25,"napolux",[680,677,18,469,227,33],"23256289",[682,677,33,233,678,33],"23256562",[684,685,18,686,687,33],"23257771",26,"project\u002Fneutral\u002Fprivacy","nickthemagicman",[689,690,18,215,691,33],"23258260",27,"amelius",[165,693,18,694,166,33],28,"project\u002Fneutral\u002Fdocs",[696,697,41,208,698,33],"23255922",29,"spiritplumber",[700,697,33,233,227,33],"23256306",[702,703,18,694,704,33],"23256599",30,"seemslegit",[706,703,18,694,707,33],"23256663","stevenjohns",[709,703,41,710,227,33],"23256677","maker\u002Fpositive\u002Fdocs",[712,703,33,713,704,33],"23256888","commenter\u002Fcritical\u002Fdocs",[715,703,18,544,227,33],"23256672",[717,718,18,215,719,33],"23258625",31,"unixhero",[721,718,33,219,719,33],"23287199",[131,723,18,694,132,33],32,[725,726,40,205,727,33],"23255970",33,"carbolymer",[729,726,33,730,227,33],"23256305","commenter\u002Fcritical\u002Fpricing",[],[733,737,742,746,751,756],{"id":734,"name":735,"points":736,"category":20},"23324598","Obsidian",1087,{"id":738,"name":739,"points":740,"category":741},"23312339","Bear",685,"web_saas",{"id":743,"name":744,"points":745,"category":20},"23295989","Wiper",574,{"id":747,"name":748,"points":749,"category":750},"23282207","LaTeX.css",515,"dev_tools",{"id":752,"name":753,"points":754,"category":755},"23283880","neut",456,"languages",{"id":757,"name":758,"points":759,"category":20},"23330228","Straight2Spam",318,false,"v2",{"id":763,"name":5,"line":6,"ai":760,"cat":20,"year":13,"points":14,"comments":15,"rec":28,"s":764,"top":18,"kw":765,"maker":11,"domain":10,"tier":19,"attempts":33,"strip":766,"url":9,"mix":767,"praise":87,"objection":103,"sub":21},23255732,0.02,[24,25,26,27],"4300000533000000003051433232322245504442",[31,32,33,34],1794036590785]