From 8b294a4b5ba7e9c7046677a48467ac79b8d42cfa Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 5 May 2017 18:59:50 +0200 Subject: [PATCH 01/79] manage feeds with missing id. fixes #13 --- feed2toot/main.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/feed2toot/main.py b/feed2toot/main.py index 2ddf17f..7869f0f 100755 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -99,20 +99,35 @@ class Main(object): # cache the ids of last rss feeds if not clioptions.all: for i in entries: - if 'id' in i and i['id'] not in cache.getdeque(): - totweet.append(i) + if 'id' in i: + if i['id'] not in cache.getdeque(): + totweet.append(i) + elif 'guid' in i: + if i['guid'] not in cache.getdeque(): + totweet.append(i) + else: + # if id or guid not in the entry, use link + if i['link'] not in cache.getdeque(): + totweet.append(i) else: totweet = entries for entry in totweet: - if 'id' not in entry: - # malformed feed entry, skip - continue - logging.debug('found feed entry {entryid}'.format(entryid=entry['id'])) - - rss = { - 'id': entry['id'], - } + if 'id' in entry: + logging.debug('found feed entry {entryid}'.format(entryid=entry['id'])) + rss = { + 'id': entry['id'], + } + elif 'guid' in entry: + logging.debug('found feed entry {entryid}'.format(entryid=entry['guid'])) + rss = { + 'id': entry['guid'], + } + else: + logging.debug('found feed entry {entryid}'.format(entryid=entry['link'])) + rss = { + 'id': entry['link'], + } severalwordsinhashtag = False # lets see if the rss feed has hashtag From 933d2e5a06f420ca6efab276d012c9496a784c49 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 5 May 2017 19:28:10 +0200 Subject: [PATCH 02/79] bump version --- README.md | 2 +- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 5f4b6e0..960f817 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.4.tar.gz + # tar zxvf feed2toot-0.5.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/install.rst b/docs/source/install.rst index 2339768..260d73b 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.4.tar.gz + $ tar zxvf feed2toot-0.5.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 431cd71..822ce95 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -23,7 +23,7 @@ import logging import os.path import sys -__version__ = '0.4' +__version__ = '0.5' class CliParse(object): '''CliParse class''' diff --git a/setup.py b/setup.py index 5ea1002..6f7c764 100755 --- a/setup.py +++ b/setup.py @@ -31,15 +31,15 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.4', + version='0.5', license='GNU GPL v3', description='Parse rss feed and tweet new posts to Mastodon', long_description='Parse rss feed and tweet new posts to the Mastodon social network', author = 'Carl Chenet', author_email = 'chaica@ohmytux.com', - url = 'https://github.com/chaica/feed2toot', + url = 'https://gitlab.com/chaica/feed2toot', classifiers=CLASSIFIERS, - download_url='https://github.com/chaica/feed2toot', + download_url='https://gitlab.com/chaica/feed2toot', packages=find_packages(), scripts=['scripts/feed2toot', 'scripts/register_feed2toot_app'], install_requires=['feedparser', 'Mastodon.py'], From 7b6d276606dffe5c97f07f6387e7452353cc5aba Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 5 May 2017 19:30:48 +0200 Subject: [PATCH 03/79] update for 0.5 --- CHANGELOG | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 6d47021..a1fc5ac 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +## [0.5] - 2017-05-05 +### Added +- manage rss feeds entries without id + ## [0.4] - 2017-04-28 ### Changed - [hashtaglist] section is not mandatory any more From e7d277e2399ce527fdb3e68aeb6ff0c06d6909ad Mon Sep 17 00:00:00 2001 From: Bastien Date: Sat, 13 May 2017 07:55:47 +0200 Subject: [PATCH 04/79] Small rephrasing and reformatting. Avoid prompt lines > 72 chars. --- scripts/register_feed2toot_app | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/register_feed2toot_app b/scripts/register_feed2toot_app index a989b93..1ea856e 100755 --- a/scripts/register_feed2toot_app +++ b/scripts/register_feed2toot_app @@ -21,10 +21,10 @@ from mastodon import Mastodon from mastodon.Mastodon import MastodonIllegalArgumentError import sys -print('\nThis app generates Mastodon app credentials needed by Feed2toot.\nfeed2toot_clientcred.txt and feed2toot_usercred.txt will be written in the current dir {cwd}.\nOne connection is initiated to create the app.\nYour password is *not* stored.\n'.format(cwd=getcwd())) +print('\nThis script generates the Mastodon application credentials for Feed2toot.\nfeed2toot_clientcred.txt and feed2toot_usercred.txt will be written\nin the current directory: {cwd}.\nA connection is initiated to create the application.\nYour password is *not* stored.\n'.format(cwd=getcwd())) # get the instance -instance = input('Mastodon instance url (defaults to https://mastodon.social):') +instance = input('Mastodon instance URL (defaults to https://mastodon.social): ') if not instance: instance = 'https://mastodon.social' elif not instance.startswith('http'): @@ -33,18 +33,18 @@ elif not instance.startswith('http'): # get the username userok = False while not userok: - user = input('Mastodon login:') + user = input('Mastodon login: ') if not user: - print('Your Mastodon username can not be empty') + print('Your Mastodon username can not be empty.') userok = False elif '@' not in user or '.' not in user: - print('Your Mastodon username should be an email') + print('Your Mastodon username should be an email.') userok = False else: userok = True # get the password -password = getpass(prompt='Mastodon password:') +password = getpass(prompt='Mastodon password: ') Mastodon.create_app( 'feed2toot', api_base_url=instance, @@ -60,6 +60,6 @@ try: ) except MastodonIllegalArgumentError as err: print(err) - sys.exit('\nMy guess is bad login/password\n') -print('The feed2toot app was added to your preferences=>authorized apps page') + sys.exit('\nI guess you entered a bad login or password.\n') +print('feed2toot was added to your preferences=>authorized apps page.') sys.exit(0) From 0e2418ea20eb81ab74bf5e41ba835b0a54471c5a Mon Sep 17 00:00:00 2001 From: The Dod Date: Wed, 28 Jun 2017 19:40:01 +0300 Subject: [PATCH 05/79] Add config option to change toot visibility Some instances allow bots only if their toots are unlisted, in order to avoid flooding the public timeline (makes sense). --- README.md | 2 ++ docs/source/configure.rst | 2 ++ feed2toot/main.py | 12 ++++++++++-- feed2toot/tootpost.py | 4 +++- 4 files changed, 17 insertions(+), 3 deletions(-) mode change 100755 => 100644 feed2toot/main.py diff --git a/README.md b/README.md index 960f817..c59105d 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ Alternatively you can donate cryptocurrencies: instance_url=https://mastodon.social user_credentials=feed2toot_usercred.txt client_credentials=feed2toot_clientcred.txt + ; Default visibility is public, but you can override it + toot_visibility=unlisted [cache] cachefile=cache.db diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 38c963e..db67c45 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -26,6 +26,8 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any ; Here you need the two files created by register_feed2toot_app user_credentials=/etc/feed2toot/credentials/feed2toot_usercred.txt client_credentials=/etc/feed2toot/credentials/feed2toot_clientcred.txt + ; Default visibility is public, but you can override it + toot_visibility=unlisted [cache] cachefile=/var/lib/feed2toot/feed2toot.db diff --git a/feed2toot/main.py b/feed2toot/main.py old mode 100755 new mode 100644 index 7869f0f..733bd49 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -189,13 +189,21 @@ class Main(object): if clioptions.dryrun: if entrytosend: - logging.warning('Tweet should have been sent: {tweet}'.format(tweet=finaltweet)) + logging.warning('Would toot with visibility "{visibility}": {toot}'.format( + toot=finaltweet, + visibility=config.get( + 'mastodon', 'toot_visibility', + fallback='public'))) else: logging.debug('This rss entry did not meet pattern criteria. Should have not been sent') else: storeit = True if entrytosend and not clioptions.populate: - logging.debug('sending the following tweet:{tweet}'.format(tweet=finaltweet)) + logging.debug('Tooting with visibility "{visibility}": {toot}'.format( + toot=finaltweet, + visibility=config.get( + 'mastodon', 'toot_visibility', + fallback='public'))) twp = TootPost(config, finaltweet) storeit = twp.storeit() else: diff --git a/feed2toot/tootpost.py b/feed2toot/tootpost.py index bad5684..c40c4ee 100644 --- a/feed2toot/tootpost.py +++ b/feed2toot/tootpost.py @@ -44,7 +44,9 @@ class TootPost: access_token = self.config.get('mastodon', 'user_credentials'), api_base_url = self.config.get('mastodon', 'instance_url') ) - mastodon.toot(self.toot) + mastodon.status_post(self.toot, + visibility=self.config.get( + 'mastodon', 'toot_visibility', fallback='public')) def storeit(self): '''Indicate if the tweet should be stored or not''' From 8673a648531d988ac2eb4e06c88beed541f578e1 Mon Sep 17 00:00:00 2001 From: The Dod Date: Wed, 28 Jun 2017 21:03:02 +0300 Subject: [PATCH 06/79] Document `toot_visibility` --- docs/source/configure.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index db67c45..231d738 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -49,6 +49,10 @@ For the [mastodon] section: - instance_url: the url of your Mastodon instance - user_credentials: a file with the user credentials, generated by the command register_feed2toot_app - client_credentials: a file with the client credentials, generated by the command register_feed2toot_app +- toot_visibility: any of the valid options for the `visibility` field + [here](https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md#posting-a-new-status). + Default is `public`, but `unlisted` prevents flooding + the instance's public timeline (which is more polite). For the [cache] section: From 5e0b915935c788c343490b17a9acfc2520976619 Mon Sep 17 00:00:00 2001 From: The Dod Date: Wed, 28 Jun 2017 21:10:10 +0300 Subject: [PATCH 07/79] =?UTF-8?q?D'Oh.=20It's=20RST=20(not=20MD)=20?= =?UTF-8?q?=F0=9F=98=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/configure.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 231d738..818e4fb 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -49,11 +49,13 @@ For the [mastodon] section: - instance_url: the url of your Mastodon instance - user_credentials: a file with the user credentials, generated by the command register_feed2toot_app - client_credentials: a file with the client credentials, generated by the command register_feed2toot_app -- toot_visibility: any of the valid options for the `visibility` field - [here](https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md#posting-a-new-status). - Default is `public`, but `unlisted` prevents flooding +- toot_visibility: any of the valid options for the *visibility* field + `here`__. + Default is *public*, but *unlisted* prevents flooding the instance's public timeline (which is more polite). +__ https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md#posting-a-new-status + For the [cache] section: - cachefile: the path to the cache file storing ids of already tweeted links. Absolute path is mandatory. This file should always use the .db extension. From 658b48571b4666955491f7a0ae6a3c9ba65fbd56 Mon Sep 17 00:00:00 2001 From: The Dod Date: Wed, 28 Jun 2017 21:21:27 +0300 Subject: [PATCH 08/79] Comment out `toot_visibility` in examples (to stress that it's optional) --- README.md | 4 ++-- docs/source/configure.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c59105d..033adf0 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,8 @@ Alternatively you can donate cryptocurrencies: instance_url=https://mastodon.social user_credentials=feed2toot_usercred.txt client_credentials=feed2toot_clientcred.txt - ; Default visibility is public, but you can override it - toot_visibility=unlisted + ; Default visibility is public, but you can override it: + ; toot_visibility=unlisted [cache] cachefile=cache.db diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 818e4fb..f354ce8 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -26,8 +26,8 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any ; Here you need the two files created by register_feed2toot_app user_credentials=/etc/feed2toot/credentials/feed2toot_usercred.txt client_credentials=/etc/feed2toot/credentials/feed2toot_clientcred.txt - ; Default visibility is public, but you can override it - toot_visibility=unlisted + ; Default visibility is public, but you can override it: + ; toot_visibility=unlisted [cache] cachefile=/var/lib/feed2toot/feed2toot.db From cdf99e3f0b4a2a957ed01305605e7f58a69eab57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Thu, 13 Jul 2017 20:19:04 +0200 Subject: [PATCH 09/79] Add the ability to use {feedname} in the tweet template --- docs/source/configure.rst | 11 ++++++++++- feed2toot/confparse.py | 27 +++++++++++++++++---------- feed2toot/filterentry.py | 13 +++++++++---- feed2toot/main.py | 6 +++--- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index f354ce8..607feda 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -5,7 +5,7 @@ As a prerequisite to use Feed2toot, you need to authorize a Mastodon app for you Just use the script register_feed2toot_app to register the feed2toot app for your account.:: - $ ./register_feed2toot_app + $ ./register_feed2toot_app This app generates Mastodon app credentials needed by Feed2toot. feed2toot_clientcred.txt and feed2toot_usercred.txt will be written in the current dir /home/chaica/progra/python/feed2toot. @@ -100,6 +100,15 @@ Now let's have a look at the =/home/john/feed2toot/rsslist.txt file:: Each line of this file is a url to a rss feed. Pretty simple. +Display the name of the feed in the toots +----------------------------------------- + +If you want to display the name of the feed in the resulting toot, you can do so by giving it a name with the following syntax:: + + Le journal du hacker + +Then in the `tweet` configuration, you can use the `{feedname}` syntax, which will be replaced by the actual name of the feed. + Match specific patterns of rss feeds in the uri_list files ---------------------------------------------------------- You can use specific pattern matching for uri in the uri_list file to filter some of the rss entries of a rss feed. Lets modify the previous file:: diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index 1611fb7..71ec749 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -23,6 +23,7 @@ import os import os.path import socket import sys +import re # 3rd party library imports import feedparser @@ -46,9 +47,9 @@ class ConfParse(object): if not config.read(os.path.expanduser(pathtoconfig)): sys.exit('Could not read config file') ########################### - # + # # the rss section - # + # ########################### section = 'rss' if config.has_section(section): @@ -101,6 +102,12 @@ class ConfParse(object): for line in rsslist: line = line.strip() # split each line in two parts, rss link and a string with the different patterns to look for + feedname = None + if '<' in line: + matches = re.match('(.*) <(.*)>', line) + if not matches: + sys.exit('This line in the list of uri to parse is not formatted correctly: {line}'.format(line)) + feedname, line = matches.groups() confobjects = line.split('|') if len(confobjects) > 3 or len(confobjects) == 2: sys.exit('This line in the list of uri to parse is not formatted correctly: {line}'.format(line)) @@ -124,7 +131,7 @@ class ConfParse(object): sys.exit('The rss object {rssobject} could not be found in the feed {rss}'.format(rssobject=rssobject, rss=rss)) else: sys.exit('The rss feed {rss} does not seem to be valid'.format(rss=rss)) - feeds.append({'feed': feed, 'patterns': patterns, 'rssobject': rssobject}) + feeds.append({'feed': feed, 'patterns': patterns, 'rssobject': rssobject, 'feedname': feedname}) # test if all feeds in the list were unsuccessfully retrieved and if so, leave if not feeds and bozoexception: sys.exit('No feed could be retrieved. Leaving.') @@ -153,9 +160,9 @@ class ConfParse(object): if config.has_option(section, currentoption): options['nopatternurinoglobalpattern'] = config.getboolean(section, currentoption) ########################### - # + # # the cache section - # + # ########################### section = 'cache' if not self.clioptions.cachefile: @@ -183,9 +190,9 @@ class ConfParse(object): else: options['cache_limit'] = 100 ########################### - # + # # the hashtag section - # + # ########################### section = 'hashtaglist' if not self.clioptions.hashtaglist: @@ -199,9 +206,9 @@ class ConfParse(object): else: options['hashtaglist'] = '' ########################### - # + # # the plugins section - # + # ########################### plugins = {} section = 'influxdb' @@ -228,7 +235,7 @@ class ConfParse(object): self.confs.append((options, config, self.tweetformat, feeds, plugins)) else: self.confs.append((options, config, self.tweetformat, [{'feed': feed, 'patterns': [], 'rssobject': ''}], plugins)) - + @property def confvalues(self): '''Return the values of the different configuration files''' diff --git a/feed2toot/filterentry.py b/feed2toot/filterentry.py index bf27ffe..3ede1c0 100644 --- a/feed2toot/filterentry.py +++ b/feed2toot/filterentry.py @@ -27,7 +27,7 @@ import feedparser class FilterEntry(object): '''FilterEntry class''' - def __init__(self, elements, entry, options, byrsspatterns, rssobject): + def __init__(self, elements, entry, options, byrsspatterns, rssobject, feedname): '''Constructor of the FilterEntry class''' self.matching = {} self.entry = entry @@ -35,15 +35,20 @@ class FilterEntry(object): self.options = options self.byrsspatterns = byrsspatterns self.rssobject = rssobject + self.feedname = feedname self.main() def main(self): '''Main of the FilterEntry class''' + authorized_elements = ['feedname', ] + authorized_elements.extend(self.entry.keys()) for i in self.elements: - if i not in self.entry: + if i not in authorized_elements: sys.exit('The element {} is not available in the RSS feed. The available ones are: {}'.format(i, [j for j in self.entry])) # for the case if no pattern at all is defined - if not self.options['patterns'] and not self.byrsspatterns and not self.rssobject: + if i == 'feedname': + self.matching[i] = self.feedname + elif not self.options['patterns'] and not self.byrsspatterns and not self.rssobject: self.matching[i] = self.entry[i] # global filter only elif self.options['patterns'] and not self.byrsspatterns and not self.rssobject: @@ -67,7 +72,7 @@ class FilterEntry(object): if not self.options['patternscasesensitive']['{}_case_sensitive'.format(patternlist)]: # not case sensitive, so we compare the lower case for pattern in self.options['patterns'][patternlist]: - finalpattern = pattern.lower() + finalpattern = pattern.lower() finaltitle = self.entry[patternlist.split('_')[0]].lower() if finalpattern in finaltitle: self.matching[i] = self.entry[i] diff --git a/feed2toot/main.py b/feed2toot/main.py index 733bd49..3da91b2 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -173,7 +173,7 @@ class Main(object): tmpelement = i.strip('{}') elements.append(tmpelement) # match elements of the tweet format string with available element in the RSS feed - fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject']) + fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject'], feed['feedname']) entrytosend = fe.finalentry if entrytosend: tweetwithnotag = tweetformat.format(**entrytosend) @@ -186,7 +186,7 @@ class Main(object): finaltweet = addtag.finaltweet else: finaltweet = dedup.finaltweet - + if clioptions.dryrun: if entrytosend: logging.warning('Would toot with visibility "{visibility}": {toot}'.format( @@ -219,7 +219,7 @@ class Main(object): pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower()) try: pluginmodule = importlib.import_module(pluginmodulename) - pluginclass = getattr(pluginmodule, pluginclassname) + pluginclass = getattr(pluginmodule, pluginclassname) pluginclass(plugins[plugin], finaltweet) except ImportError as err: print(err) From ff140ae2f7a4144c02f2662461b7fbcf6b2bfa6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Thu, 13 Jul 2017 20:42:16 +0200 Subject: [PATCH 10/79] Add a configuration option to accept bozo exceptions --- docs/source/configure.rst | 7 +++++++ feed2toot/confparse.py | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index f354ce8..7399f47 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -44,6 +44,9 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any [hashtaglist] several_words_hashtags_list=/etc/feed2toot/hashtags.txt + [feedparser] + accept_bozo_exceptions=true + For the [mastodon] section: - instance_url: the url of your Mastodon instance @@ -74,6 +77,10 @@ For the [hashtaglist] section: - several_words_hashtags_list: a path to the file containing hashtags in two or more words. Absolute path is mandatory. By default Feed2toot adds a # before every words of a hashtag. See documentation below for an example of this file. +for the [feedparser] section: + +- accept_bozo_exceptions: If set to true, feed2toot will accept malformed feeds, which are rejected by default. + Example of the list of hash tags ================================ The list of hash tags is a simple text file with one hash tag composed by several words on a single line:: diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index 1611fb7..d8d67f3 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -45,6 +45,13 @@ class ConfParse(object): config = SafeConfigParser() if not config.read(os.path.expanduser(pathtoconfig)): sys.exit('Could not read config file') + + # The feedparser section + if config.has_option('feedparser', 'accept_bozo_exceptions'): + self.accept_bozo_exceptions = config.getboolean('feedparser', 'accept_bozo_exceptions') + else: + self.accept_bozo_exceptions = False + ########################### # # the rss section @@ -117,7 +124,8 @@ class ConfParse(object): if 'bozo_exception' in feed: bozoexception = True logging.warning(feed['bozo_exception']) - continue + if not self.accept_bozo_exceptions: + continue # check if the rss feed and the rss entry are valid ones if 'entries' in feed: if rssobject and rssobject not in feed['entries'][0].keys(): From 37ee1e5a67abee5f33f431ecd2c5488cc2175a09 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 31 Jul 2017 15:58:30 +0200 Subject: [PATCH 11/79] cleaning some code (from MR #9) --- feed2toot/addtags.py | 6 +++--- feed2toot/cliparse.py | 6 +++--- feed2toot/confparse.py | 12 +++++------- feed2toot/feedcache.py | 4 ++-- feed2toot/plugins/influxdbplugin.py | 16 ++++++++-------- feed2toot/removeduplicates.py | 12 ++++++------ 6 files changed, 27 insertions(+), 29 deletions(-) diff --git a/feed2toot/addtags.py b/feed2toot/addtags.py index 6b63351..1859e07 100644 --- a/feed2toot/addtags.py +++ b/feed2toot/addtags.py @@ -20,7 +20,7 @@ # standard library imports from operator import itemgetter -class AddTags(object): +class AddTags: '''Add as many tags as possible depending on the tweet length''' def __init__(self, tweet, tags): '''Constructor of AddTags class''' @@ -34,14 +34,14 @@ class AddTags(object): tweetlength = len(self.tweet) # sort list of tags, the ones with the greater length first - tagswithindices = ({'text':i, 'length':len(i)} for i in self.tags) + tagswithindices = ({'text':i, 'length': len(i)} for i in self.tags) sortedtagswithindices = sorted(tagswithindices, key=itemgetter('length'), reverse=True) self.tags = (i['text'] for i in sortedtagswithindices) # add tags is space is available for tag in self.tags: taglength = len(tag) - if (tweetlength + (taglength +1)) <= maxlength: + if (tweetlength + (taglength + 1)) <= maxlength: self.tweet = ' '.join([self.tweet, tag]) tweetlength += (taglength + 1) diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 822ce95..1411ad8 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -25,7 +25,7 @@ import sys __version__ = '0.5' -class CliParse(object): +class CliParse: '''CliParse class''' def __init__(self): '''Constructor for the CliParse class''' @@ -34,7 +34,7 @@ class CliParse(object): def main(self): '''main of CliParse class''' feed2tootepilog = 'For more information: https://feed2toot.readhthedocs.org' - feed2tootdescription = 'Take rss feed and send it to Mastodon' + feed2tootdescription = 'Take rss feed and send it to Mastodon' parser = ArgumentParser(prog='feed2toot', description=feed2tootdescription, epilog=feed2tootepilog) @@ -63,7 +63,7 @@ class CliParse(object): action='store_const', const='debug', default='warning', help='enable debug output, work on log level DEBUG') levels = [i for i in logging._nameToLevel.keys() - if (type(i) == str and i != 'NOTSET')] + if (type(i) == str and i != 'NOTSET')] parser.add_argument('--syslog', nargs='?', default=None, type=str.upper, action='store', const='INFO', choices=levels, diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index 0a4edd9..d333547 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -17,18 +17,17 @@ '''Get values of the configuration file''' # standard library imports -from configparser import SafeConfigParser, NoOptionError, NoSectionError +from configparser import SafeConfigParser import logging import os import os.path -import socket import sys import re # 3rd party library imports import feedparser -class ConfParse(object): +class ConfParse: '''ConfParse class''' def __init__(self, clioptions): '''Constructor of the ConfParse class''' @@ -46,7 +45,6 @@ class ConfParse(object): config = SafeConfigParser() if not config.read(os.path.expanduser(pathtoconfig)): sys.exit('Could not read config file') - # The feedparser section if config.has_option('feedparser', 'accept_bozo_exceptions'): self.accept_bozo_exceptions = config.getboolean('feedparser', 'accept_bozo_exceptions') @@ -185,7 +183,7 @@ class ConfParse(object): sys.exit('The parent directory of the cache file does not exist: {cachefileparent}'.format(cachefileparent=cachefileparent)) else: options['cachefile'] = self.clioptions.cachefile - ### cache limit + # cache limit if config.has_section(section): confoption = 'cache_limit' if config.has_option(section, confoption): @@ -225,7 +223,7 @@ class ConfParse(object): # host, port, user, pass, database options ########################################## plugins[section] = {} - for currentoption in ['host','port','user','pass','database']: + for currentoption in ['host', 'port', 'user', 'pass', 'database']: if config.has_option(section, currentoption): plugins[section][currentoption] = config.get(section, currentoption) if 'host' not in plugins[section]: @@ -234,7 +232,7 @@ class ConfParse(object): plugins[section]['port'] = 8086 if 'measurement' not in plugins[section]: plugins[section]['measurement'] = 'tweets' - for field in ['user','pass','database']: + for field in ['user', 'pass', 'database']: if field not in plugins[section]: sys.exit('Parsing error for {field} in the [{section}] section: {field} is not defined'.format(field=field, section=section)) diff --git a/feed2toot/feedcache.py b/feed2toot/feedcache.py index 8b1f665..ca030f7 100644 --- a/feed2toot/feedcache.py +++ b/feed2toot/feedcache.py @@ -38,9 +38,9 @@ class FeedCache: with open(self.options['cachefile']) as dbdsc: dbfromfile = dbdsc.readlines() dblist = [i.strip() for i in dbfromfile] - self.dbfeed = deque(dblist, self.options['cache_limit'] ) + self.dbfeed = deque(dblist, self.options['cache_limit']) else: - self.dbfeed = deque([], self.options['cache_limit'] ) + self.dbfeed = deque([], self.options['cache_limit']) def append(self, rssid): '''Append a rss id to the cache''' diff --git a/feed2toot/plugins/influxdbplugin.py b/feed2toot/plugins/influxdbplugin.py index ccb8b5a..849cc8b 100644 --- a/feed2toot/plugins/influxdbplugin.py +++ b/feed2toot/plugins/influxdbplugin.py @@ -16,9 +16,6 @@ # Push values to a influxdb database '''Push values to a influxdb database''' -# standard libraries imports -import json - # 3rd party libraries imports from influxdb import InfluxDBClient @@ -30,13 +27,16 @@ class InfluxdbPlugin(object): self.data = data self.datatoinfluxdb = [] self.client = InfluxDBClient(self.plugininfo['host'], - self.plugininfo['port'], - self.plugininfo['user'], - self.plugininfo['pass'], - self.plugininfo['database']) + self.plugininfo['port'], + self.plugininfo['user'], + self.plugininfo['pass'], + self.plugininfo['database']) self.main() def main(self): '''Main of the PiwikModule class''' - self.datatoinfluxdb.append({'measurement': self.plugininfo['measurement'], 'fields': {'value': self.data}}) + self.datatoinfluxdb.append({ + 'measurement': self.plugininfo['measurement'], + 'fields': {'value': self.data} + }) self.client.write_points(self.datatoinfluxdb) diff --git a/feed2toot/removeduplicates.py b/feed2toot/removeduplicates.py index d45dfc8..6e857b1 100644 --- a/feed2toot/removeduplicates.py +++ b/feed2toot/removeduplicates.py @@ -17,7 +17,7 @@ # Remove duplicates from the final string before sending the tweet '''Remove duplicates from the final string before sending the tweet''' -class RemoveDuplicates(object): +class RemoveDuplicates: '''Remove duplicates from the final string before sending the tweet''' def __init__(self, tweet): '''Constructor of RemoveDuplicates class''' @@ -32,12 +32,12 @@ class RemoveDuplicates(object): if element != ' ' and (element.startswith('http://') or element.startswith('https://')): newlink = True # if we already found this link, increment the counter - for i,_ in enumerate(links): + for i, _ in enumerate(links): if links[i]['link'] == element: newlink = False links[i]['count'] += 1 if newlink: - links.append({'link': element, 'count': 1}) + links.append({'link': element, 'count': 1}) # remove duplicates validatedlinks = [] for i in range(len(links)): @@ -45,14 +45,14 @@ class RemoveDuplicates(object): validatedlinks.append(links[i]) wildcard = 'FEED2TOOTWILDCARD' for element in validatedlinks: - for i in range(element['count']): + for i in range(element['count']): # needed for not inversing the order of links if it is a duplicate # and the second link is not one if i == 0: - self.tweet = self.tweet.replace(element['link'], wildcard, 1 ) + self.tweet = self.tweet.replace(element['link'], wildcard, 1) else: self.tweet = self.tweet.replace(element['link'], '', 1) - # finally + # finally self.tweet = self.tweet.replace(wildcard, element['link'], 1) # remove all 2xspaces self.tweet = self.tweet.replace(' ', ' ') From f129d15f1106f77e051cebb6e3f72a056f7a3101 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 31 Jul 2017 17:21:48 +0200 Subject: [PATCH 12/79] ignore feedlist file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 65b3d74..7a3968d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ docs/build/ __pycache__ +feedlist *.swp *.pyc *.db From 3aef77acac75f211c9b90ac77adc28f9611a1e35 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 16:55:26 +0200 Subject: [PATCH 13/79] add support for feedname for uri parameter --- feed2toot/confparse.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index d333547..b266009 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -107,7 +107,7 @@ class ConfParse: for line in rsslist: line = line.strip() # split each line in two parts, rss link and a string with the different patterns to look for - feedname = None + feedname = '' if '<' in line: matches = re.match('(.*) <(.*)>', line) if not matches: @@ -147,7 +147,18 @@ class ConfParse: if not feeds and not self.clioptions.rss_uri: confoption = 'uri' if config.has_option(section, confoption): - options['rss_uri'] = config.get('rss', 'uri') + urifeed = config.get('rss', 'uri') + feedname = None + if '<' in urifeed: + matches = re.match('(.*) <(.*)>', urifeed) + if not matches: + sys.exit('This uri to parse is not formatted correctly: {urifeed}'.format(urifeed)) + feedname, finaluri = matches.groups() + print(feedname) + print(finaluri) + options['rss_uri'] = finaluri + else: + options['rss_uri'] = config.get('rss', 'uri') else: sys.exit('{confoption} parameter in the [{section}] section of the configuration file is mandatory. Exiting.'.format(section=section, confoption=confoption)) else: @@ -240,7 +251,7 @@ class ConfParse: if feeds: self.confs.append((options, config, self.tweetformat, feeds, plugins)) else: - self.confs.append((options, config, self.tweetformat, [{'feed': feed, 'patterns': [], 'rssobject': ''}], plugins)) + self.confs.append((options, config, self.tweetformat, [{'feed': feed, 'patterns': [], 'rssobject': '', 'feedname': feedname}], plugins)) @property def confvalues(self): From 65242b02cfc90ea9f2e838390091bff94437100d Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 16:57:08 +0200 Subject: [PATCH 14/79] cleaning some code (from MR #9) --- feed2toot/filterentry.py | 8 +------- feed2toot/main.py | 8 ++------ feed2toot/tootpost.py | 20 +++++--------------- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/feed2toot/filterentry.py b/feed2toot/filterentry.py index 3ede1c0..4d9d793 100644 --- a/feed2toot/filterentry.py +++ b/feed2toot/filterentry.py @@ -17,15 +17,9 @@ '''Filter an entry of the RSS feeds''' # standard library imports -from configparser import SafeConfigParser, NoOptionError, NoSectionError -import os -import os.path import sys -# 3rd party library imports -import feedparser - -class FilterEntry(object): +class FilterEntry: '''FilterEntry class''' def __init__(self, elements, entry, options, byrsspatterns, rssobject, feedname): '''Constructor of the FilterEntry class''' diff --git a/feed2toot/main.py b/feed2toot/main.py index 3da91b2..0b379c7 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -21,12 +21,8 @@ import codecs import importlib import logging import logging.handlers -import os import sys -# 3rd party libraries imports -import feedparser - # app libraries imports from feed2toot.addtags import AddTags from feed2toot.cliparse import CliParse @@ -36,7 +32,7 @@ from feed2toot.removeduplicates import RemoveDuplicates from feed2toot.tootpost import TootPost from feed2toot.feedcache import FeedCache -class Main(object): +class Main: '''Main class of Feed2toot''' def __init__(self): @@ -164,7 +160,7 @@ class Main(object): nospace = nospace.replace(" ", "") rss['hashtags'].append('#{}'.format(nospace)) - elements=[] + elements = [] for i in tweetformat.split(' '): tmpelement = '' # if i is not an empty string diff --git a/feed2toot/tootpost.py b/feed2toot/tootpost.py index c40c4ee..7b2f358 100644 --- a/feed2toot/tootpost.py +++ b/feed2toot/tootpost.py @@ -15,16 +15,7 @@ """Checks an RSS feed and posts new entries to Mastodon.""" -# standard libraires imports -from configparser import SafeConfigParser, NoOptionError, NoSectionError -from argparse import ArgumentParser -import codecs -import logging -import os -import sys - # 3rd party libraries imports -import feedparser from mastodon import Mastodon class TootPost: @@ -40,13 +31,12 @@ class TootPost: def main(self): '''Main of the TweetPost class''' mastodon = Mastodon( - client_id = self.config.get('mastodon', 'client_credentials'), - access_token = self.config.get('mastodon', 'user_credentials'), - api_base_url = self.config.get('mastodon', 'instance_url') + client_id=self.config.get('mastodon', 'client_credentials'), + access_token=self.config.get('mastodon', 'user_credentials'), + api_base_url=self.config.get('mastodon', 'instance_url') ) - mastodon.status_post(self.toot, - visibility=self.config.get( - 'mastodon', 'toot_visibility', fallback='public')) + toot_visibility = self.config.get('mastodon', 'toot_visibility', fallback='public') + mastodon.status_post(self.toot, visibility=toot_visibility) def storeit(self): '''Indicate if the tweet should be stored or not''' From 239700d123950d9b3d6c01c1171e9e55c115213c Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 17:20:52 +0200 Subject: [PATCH 15/79] deprecation warning for tweet parameter --- feed2toot/confparse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index b266009..2f2110a 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -50,7 +50,6 @@ class ConfParse: self.accept_bozo_exceptions = config.getboolean('feedparser', 'accept_bozo_exceptions') else: self.accept_bozo_exceptions = False - ########################### # # the rss section @@ -63,7 +62,9 @@ class ConfParse: ############################ oldconfoption = 'tweet' confoption = 'toot' + # manage 'tweet' for compatibility reason with first versions if config.has_option(section, oldconfoption): + logging.warn("Your configuration file uses a 'tweet' parameter instead of 'toot'. 'tweet' is deprecated and will be removed in Feed2toot 0.7") self.tweetformat = config.get(section, oldconfoption) elif config.has_option(section, confoption): self.tweetformat = config.get(section, confoption) From a9fadd6ccf77ea151b495784958aca5f97e1603e Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 18:29:24 +0200 Subject: [PATCH 16/79] verify the path to the hashtaglist --- feed2toot/cliparse.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 1411ad8..a67cba3 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -98,6 +98,11 @@ class CliParse: # verify if a configuration file is provided if not self.opts.configs: sys.exit('no configuration file was found at the specified path(s) with the option -c') + # verify the path to the hashtaglist + if self.opts.hashtaglist: + hashtaglist = os.path.expanduser(self.opts.hashtaglist) + if not os.path.exists(hashtaglist): + sys.exit('the {hashtaglist} file does not seem to exist, please provide a valid path'.format(hashtaglist=hashtaglist)) @property def options(self): From fe0c47cdf23024b795c004d58f5585743d0fe5a2 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 18:30:24 +0200 Subject: [PATCH 17/79] split configuration parsing --- feed2toot/confparse.py | 74 ++++++++++------------------ feed2toot/confparsers/hashtaglist.py | 38 ++++++++++++++ feed2toot/confparsers/plugins.py | 43 ++++++++++++++++ 3 files changed, 107 insertions(+), 48 deletions(-) create mode 100644 feed2toot/confparsers/hashtaglist.py create mode 100644 feed2toot/confparsers/plugins.py diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index 2f2110a..868d001 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -27,6 +27,10 @@ import re # 3rd party library imports import feedparser +# feed2toot library imports +from feed2toot.confparsers.hashtaglist import parsehashtaglist +from feed2toot.confparsers.plugins import parseplugins + class ConfParse: '''ConfParse class''' def __init__(self, clioptions): @@ -70,9 +74,9 @@ class ConfParse: self.tweetformat = config.get(section, confoption) else: sys.exit('You should define a format for your tweet with the parameter "{confoption}" in the [{section}] section'.format(confoption=confoption, section=section)) - ############################ + ####################### # pattern format option - ############################ + ####################### options['patterns'] = {} options['patternscasesensitive'] = {} for pattern in ['summary_detail', 'published_parsed', 'guidislink', 'authors', 'links', 'title_detail', 'author', 'author_detail', 'comments', 'published', 'summary', 'tags', 'title', 'link', 'id']: @@ -84,20 +88,22 @@ class ConfParse: else: options['patterns'][currentoption] = [tmppattern] - # pattern_case_sensitive format + ############################### + # pattern_case_sensitive option + ############################### currentoption = '{}_pattern_case_sensitive'.format(pattern) if config.has_option(section, currentoption): try: options['patternscasesensitive'][currentoption] = config.getboolean(section, currentoption) except ValueError as err: - print(err) + logging.warn(err) options['patternscasesensitive'][currentoption] = True - ############################ - # rsslist - ############################ bozoexception = False feeds = [] patterns = [] + ################ + # uri_list option + ################ currentoption = 'uri_list' if config.has_option(section, currentoption): rssfile = config.get(section, currentoption) @@ -142,9 +148,9 @@ class ConfParse: # test if all feeds in the list were unsuccessfully retrieved and if so, leave if not feeds and bozoexception: sys.exit('No feed could be retrieved. Leaving.') - ############################ - # uri - ############################ + ############ + # uri option + ############ if not feeds and not self.clioptions.rss_uri: confoption = 'uri' if config.has_option(section, confoption): @@ -155,8 +161,6 @@ class ConfParse: if not matches: sys.exit('This uri to parse is not formatted correctly: {urifeed}'.format(urifeed)) feedname, finaluri = matches.groups() - print(feedname) - print(finaluri) options['rss_uri'] = finaluri else: options['rss_uri'] = config.get('rss', 'uri') @@ -184,6 +188,9 @@ class ConfParse: ########################### section = 'cache' if not self.clioptions.cachefile: + ################## + # cachefile option + ################## confoption = 'cachefile' if config.has_section(section): options['cachefile'] = config.get(section, confoption) @@ -208,47 +215,18 @@ class ConfParse: else: options['cache_limit'] = 100 ########################### - # # the hashtag section - # ########################### - section = 'hashtaglist' - if not self.clioptions.hashtaglist: - confoption = 'several_words_hashtags_list' - if config.has_section(section): - if config.has_option(section, confoption): - options['hashtaglist'] = config.get(section, confoption) - options['hashtaglist'] = os.path.expanduser(options['hashtaglist']) - if not os.path.exists(options['hashtaglist']) or not os.path.isfile(options['hashtaglist']): - sys.exit('The path to the several_words_hashtags_list parameter is not valid: {hashtaglist}'.format(hashtaglist=options['hashtaglist'])) - else: - options['hashtaglist'] = '' + options['hashtaglist'] = parsehashtaglist(self.clioptions.hashtaglist, config) ########################### - # # the plugins section - # ########################### - plugins = {} - section = 'influxdb' - if config.has_section(section): - ########################################## - # host, port, user, pass, database options - ########################################## - plugins[section] = {} - for currentoption in ['host', 'port', 'user', 'pass', 'database']: - if config.has_option(section, currentoption): - plugins[section][currentoption] = config.get(section, currentoption) - if 'host' not in plugins[section]: - plugins[section]['host'] = '127.0.0.1' - if 'port' not in plugins[section]: - plugins[section]['port'] = 8086 - if 'measurement' not in plugins[section]: - plugins[section]['measurement'] = 'tweets' - for field in ['user', 'pass', 'database']: - if field not in plugins[section]: - sys.exit('Parsing error for {field} in the [{section}] section: {field} is not defined'.format(field=field, section=section)) - - # create the returned object with previously parsed data + plugins = parseplugins(config) + ######################################## + # + # return the final configurations values + # + ######################################## if feeds: self.confs.append((options, config, self.tweetformat, feeds, plugins)) else: diff --git a/feed2toot/confparsers/hashtaglist.py b/feed2toot/confparsers/hashtaglist.py new file mode 100644 index 0000000..e7ba0ee --- /dev/null +++ b/feed2toot/confparsers/hashtaglist.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Tue, 1 Aug 2017 23:12:33 +0200 Subject: [PATCH 18/79] add conf parsing for urilist option of rss section --- feed2toot/confparsers/rss/urilist.py | 82 ++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 feed2toot/confparsers/rss/urilist.py diff --git a/feed2toot/confparsers/rss/urilist.py b/feed2toot/confparsers/rss/urilist.py new file mode 100644 index 0000000..6b71c51 --- /dev/null +++ b/feed2toot/confparsers/rss/urilist.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see ', line) + if not matches: + sys.exit('This line in the list of uri to parse is not formatted correctly: {line}'.format(line)) + feedname, line = matches.groups() + confobjects = line.split('|') + if len(confobjects) > 3 or len(confobjects) == 2: + sys.exit('This line in the list of uri to parse is not formatted correctly: {line}'.format(line)) + if len(confobjects) == 3: + rss, rssobject, patternstring = line.split('|') + if len(confobjects) == 1: + rss = confobjects[0] + rssobject = '' + patternstring = '' + # split different searched patterns + patterns = [i for i in patternstring.split(stringsep) if i] + # retrieve the content of the rss + feed = feedparser.parse(rss) + if 'bozo_exception' in feed: + bozoexception = True + logging.warning(feed['bozo_exception']) + if not accept_bozo_exceptions: + continue + # check if the rss feed and the rss entry are valid ones + if 'entries' in feed: + if rssobject and rssobject not in feed['entries'][0].keys(): + sys.exit('The rss object {rssobject} could not be found in the feed {rss}'.format(rssobject=rssobject, rss=rss)) + else: + sys.exit('The rss feed {rss} does not seem to be valid'.format(rss=rss)) + feeds.append({'feed': feed, 'patterns': patterns, 'rssobject': rssobject, 'feedname': feedname}) + # test if all feeds in the list were unsuccessfully retrieved and if so, leave + if not feeds and bozoexception: + sys.exit('No feed could be retrieved. Leaving.') + return feeds From 77123c585c5bb30007914e41b910cd09b080fc3c Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 23:12:50 +0200 Subject: [PATCH 19/79] add conf parsing for toot option of rss section --- feed2toot/confparsers/rss/toot.py | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 feed2toot/confparsers/rss/toot.py diff --git a/feed2toot/confparsers/rss/toot.py b/feed2toot/confparsers/rss/toot.py new file mode 100644 index 0000000..0c82220 --- /dev/null +++ b/feed2toot/confparsers/rss/toot.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Tue, 1 Aug 2017 23:13:33 +0200 Subject: [PATCH 20/79] add conf parsing for pattern pattern pattern_case_sensitive options of the rss section --- feed2toot/confparsers/rss/pattern.py | 52 ++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 feed2toot/confparsers/rss/pattern.py diff --git a/feed2toot/confparsers/rss/pattern.py b/feed2toot/confparsers/rss/pattern.py new file mode 100644 index 0000000..273ee25 --- /dev/null +++ b/feed2toot/confparsers/rss/pattern.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Tue, 1 Aug 2017 23:13:53 +0200 Subject: [PATCH 21/79] init modules --- feed2toot/confparsers/__init__.py | 15 +++++++++++++++ feed2toot/confparsers/rss/__init__.py | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 feed2toot/confparsers/__init__.py create mode 100644 feed2toot/confparsers/rss/__init__.py diff --git a/feed2toot/confparsers/__init__.py b/feed2toot/confparsers/__init__.py new file mode 100644 index 0000000..d19bf9f --- /dev/null +++ b/feed2toot/confparsers/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see diff --git a/feed2toot/confparsers/rss/__init__.py b/feed2toot/confparsers/rss/__init__.py new file mode 100644 index 0000000..d19bf9f --- /dev/null +++ b/feed2toot/confparsers/rss/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see From c2f01927ce04c988716a832f03da1b8b4e557ef3 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 23:14:22 +0200 Subject: [PATCH 22/79] conf parsing for the cache section --- feed2toot/confparsers/cache.py | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 feed2toot/confparsers/cache.py diff --git a/feed2toot/confparsers/cache.py b/feed2toot/confparsers/cache.py new file mode 100644 index 0000000..5490b1a --- /dev/null +++ b/feed2toot/confparsers/cache.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Tue, 1 Aug 2017 23:14:41 +0200 Subject: [PATCH 23/79] conf parsing for the feedparser section --- feed2toot/confparsers/feedparser.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 feed2toot/confparsers/feedparser.py diff --git a/feed2toot/confparsers/feedparser.py b/feed2toot/confparsers/feedparser.py new file mode 100644 index 0000000..e3c192a --- /dev/null +++ b/feed2toot/confparsers/feedparser.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Tue, 1 Aug 2017 23:15:50 +0200 Subject: [PATCH 24/79] add support for cache,feedparser,pattern,toot,urilist options and sections --- feed2toot/confparse.py | 146 ++++++----------------------------------- 1 file changed, 20 insertions(+), 126 deletions(-) diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index 868d001..fb49ee2 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -28,8 +28,13 @@ import re import feedparser # feed2toot library imports +from feed2toot.confparsers.cache import parsecache from feed2toot.confparsers.hashtaglist import parsehashtaglist +from feed2toot.confparsers.feedparser import parsefeedparser from feed2toot.confparsers.plugins import parseplugins +from feed2toot.confparsers.rss.pattern import parsepattern +from feed2toot.confparsers.rss.toot import parsetoot +from feed2toot.confparsers.rss.urilist import parseurilist class ConfParse: '''ConfParse class''' @@ -49,105 +54,25 @@ class ConfParse: config = SafeConfigParser() if not config.read(os.path.expanduser(pathtoconfig)): sys.exit('Could not read config file') - # The feedparser section - if config.has_option('feedparser', 'accept_bozo_exceptions'): - self.accept_bozo_exceptions = config.getboolean('feedparser', 'accept_bozo_exceptions') - else: - self.accept_bozo_exceptions = False + #################### + # feedparser section + #################### + accept_bozo_exceptions = parsefeedparser(config) ########################### - # # the rss section - # ########################### + self.tweetformat = parsetoot(config) + ################################################# + # pattern and patter_case_sensitive format option + ################################################# + options['patterns'], options['patternscasesensitive'] = parsepattern(config) + ################# + # uri_list option + ################# + feeds = [] + feeds = parseurilist(config, accept_bozo_exceptions) section = 'rss' if config.has_section(section): - ############################ - # tweet option - ############################ - oldconfoption = 'tweet' - confoption = 'toot' - # manage 'tweet' for compatibility reason with first versions - if config.has_option(section, oldconfoption): - logging.warn("Your configuration file uses a 'tweet' parameter instead of 'toot'. 'tweet' is deprecated and will be removed in Feed2toot 0.7") - self.tweetformat = config.get(section, oldconfoption) - elif config.has_option(section, confoption): - self.tweetformat = config.get(section, confoption) - else: - sys.exit('You should define a format for your tweet with the parameter "{confoption}" in the [{section}] section'.format(confoption=confoption, section=section)) - ####################### - # pattern format option - ####################### - options['patterns'] = {} - options['patternscasesensitive'] = {} - for pattern in ['summary_detail', 'published_parsed', 'guidislink', 'authors', 'links', 'title_detail', 'author', 'author_detail', 'comments', 'published', 'summary', 'tags', 'title', 'link', 'id']: - currentoption = '{}_pattern'.format(pattern) - if config.has_option(section, currentoption): - tmppattern = config.get(section, currentoption) - if self.stringsep in tmppattern: - options['patterns'][currentoption] = [i for i in tmppattern.split(self.stringsep) if i] - else: - options['patterns'][currentoption] = [tmppattern] - - ############################### - # pattern_case_sensitive option - ############################### - currentoption = '{}_pattern_case_sensitive'.format(pattern) - if config.has_option(section, currentoption): - try: - options['patternscasesensitive'][currentoption] = config.getboolean(section, currentoption) - except ValueError as err: - logging.warn(err) - options['patternscasesensitive'][currentoption] = True - bozoexception = False - feeds = [] - patterns = [] - ################ - # uri_list option - ################ - currentoption = 'uri_list' - if config.has_option(section, currentoption): - rssfile = config.get(section, currentoption) - rssfile = os.path.expanduser(rssfile) - if not os.path.exists(rssfile) or not os.path.isfile(rssfile): - sys.exit('The path to the uri_list parameter is not valid: {rssfile}'.format(rssfile=rssfile)) - rsslist = open(rssfile, 'r').readlines() - for line in rsslist: - line = line.strip() - # split each line in two parts, rss link and a string with the different patterns to look for - feedname = '' - if '<' in line: - matches = re.match('(.*) <(.*)>', line) - if not matches: - sys.exit('This line in the list of uri to parse is not formatted correctly: {line}'.format(line)) - feedname, line = matches.groups() - confobjects = line.split('|') - if len(confobjects) > 3 or len(confobjects) == 2: - sys.exit('This line in the list of uri to parse is not formatted correctly: {line}'.format(line)) - if len(confobjects) == 3: - rss, rssobject, patternstring = line.split('|') - if len(confobjects) == 1: - rss = confobjects[0] - rssobject = '' - patternstring = '' - # split different searched patterns - patterns = [i for i in patternstring.split(self.stringsep) if i] - # retrieve the content of the rss - feed = feedparser.parse(rss) - if 'bozo_exception' in feed: - bozoexception = True - logging.warning(feed['bozo_exception']) - if not self.accept_bozo_exceptions: - continue - # check if the rss feed and the rss entry are valid ones - if 'entries' in feed: - if rssobject and rssobject not in feed['entries'][0].keys(): - sys.exit('The rss object {rssobject} could not be found in the feed {rss}'.format(rssobject=rssobject, rss=rss)) - else: - sys.exit('The rss feed {rss} does not seem to be valid'.format(rss=rss)) - feeds.append({'feed': feed, 'patterns': patterns, 'rssobject': rssobject, 'feedname': feedname}) - # test if all feeds in the list were unsuccessfully retrieved and if so, leave - if not feeds and bozoexception: - sys.exit('No feed could be retrieved. Leaving.') ############ # uri option ############ @@ -182,38 +107,9 @@ class ConfParse: if config.has_option(section, currentoption): options['nopatternurinoglobalpattern'] = config.getboolean(section, currentoption) ########################### - # # the cache section - # ########################### - section = 'cache' - if not self.clioptions.cachefile: - ################## - # cachefile option - ################## - confoption = 'cachefile' - if config.has_section(section): - options['cachefile'] = config.get(section, confoption) - else: - sys.exit('You should provide a {confoption} parameter in the [{section}] section'.format(section=section, confoption=confoption)) - options['cachefile'] = os.path.expanduser(options['cachefile']) - cachefileparent = os.path.dirname(options['cachefile']) - if cachefileparent and not os.path.exists(cachefileparent): - sys.exit('The parent directory of the cache file does not exist: {cachefileparent}'.format(cachefileparent=cachefileparent)) - else: - options['cachefile'] = self.clioptions.cachefile - # cache limit - if config.has_section(section): - confoption = 'cache_limit' - if config.has_option(section, confoption): - try: - options['cache_limit'] = int(config.get(section, confoption)) - except ValueError as err: - sys.exit('Error in configuration with the {confoption} parameter in [{section}]: {err}'.format(confoption=confoption, section=section, err=err)) - else: - options['cache_limit'] = 100 - else: - options['cache_limit'] = 100 + options['cachefile'], options['cache_limit'] = parsecache(self.clioptions.cachefile, config) ########################### # the hashtag section ########################### @@ -223,9 +119,7 @@ class ConfParse: ########################### plugins = parseplugins(config) ######################################## - # # return the final configurations values - # ######################################## if feeds: self.confs.append((options, config, self.tweetformat, feeds, plugins)) From ad5f3ec9404079567206d65b6784efd8e87e8b2f Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 1 Aug 2017 23:34:11 +0200 Subject: [PATCH 25/79] conf parsing for uri option of the rss section --- feed2toot/confparse.py | 40 +++------------------ feed2toot/confparsers/rss/uri.py | 62 ++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 feed2toot/confparsers/rss/uri.py diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index fb49ee2..c6b9e20 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -34,6 +34,7 @@ from feed2toot.confparsers.feedparser import parsefeedparser from feed2toot.confparsers.plugins import parseplugins from feed2toot.confparsers.rss.pattern import parsepattern from feed2toot.confparsers.rss.toot import parsetoot +from feed2toot.confparsers.rss.uri import parseuri from feed2toot.confparsers.rss.urilist import parseurilist class ConfParse: @@ -71,41 +72,10 @@ class ConfParse: ################# feeds = [] feeds = parseurilist(config, accept_bozo_exceptions) - section = 'rss' - if config.has_section(section): - ############ - # uri option - ############ - if not feeds and not self.clioptions.rss_uri: - confoption = 'uri' - if config.has_option(section, confoption): - urifeed = config.get('rss', 'uri') - feedname = None - if '<' in urifeed: - matches = re.match('(.*) <(.*)>', urifeed) - if not matches: - sys.exit('This uri to parse is not formatted correctly: {urifeed}'.format(urifeed)) - feedname, finaluri = matches.groups() - options['rss_uri'] = finaluri - else: - options['rss_uri'] = config.get('rss', 'uri') - else: - sys.exit('{confoption} parameter in the [{section}] section of the configuration file is mandatory. Exiting.'.format(section=section, confoption=confoption)) - else: - options['rss_uri'] = self.clioptions.rss_uri - # get the rss feed for rss parameter of [rss] section - feed = feedparser.parse(options['rss_uri']) - if not feed: - sys.exit('Unable to parse the feed at the following url: {rss}'.format(rss=rss)) - - ######################################### - # no_uri_pattern_no_global_pattern option - ######################################### - currentoption = 'no_uri_pattern_no_global_pattern' - # default value - options['nopatternurinoglobalpattern'] = False - if config.has_option(section, currentoption): - options['nopatternurinoglobalpattern'] = config.getboolean(section, currentoption) + ############ + # uri option + ############ + options['rss_uri'], feed, feedname, options['nopatternurinoglobalpattern'] = parseuri(config, self.clioptions.rss_uri, feeds) ########################### # the cache section ########################### diff --git a/feed2toot/confparsers/rss/uri.py b/feed2toot/confparsers/rss/uri.py new file mode 100644 index 0000000..d574849 --- /dev/null +++ b/feed2toot/confparsers/rss/uri.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see ', urifeed) + if not matches: + sys.exit('This uri to parse is not formatted correctly: {urifeed}'.format(urifeed)) + feedname, finaluri = matches.groups() + rssuri = finaluri + else: + rssuri = config.get('rss', 'uri') + else: + sys.exit('{confoption} parameter in the [{section}] section of the configuration file is mandatory. Exiting.'.format(section=section, confoption=confoption)) + else: + rssuri = clioption + # get the rss feed for rss parameter of [rss] section + feed = feedparser.parse(rssuri) + if not feed: + sys.exit('Unable to parse the feed at the following url: {rss}'.format(rss=rss)) + ######################################### + # no_uri_pattern_no_global_pattern option + ######################################### + currentoption = 'no_uri_pattern_no_global_pattern' + # default value + if config.has_option(section, currentoption): + nopatternurinoglobalpattern = config.getboolean(section, currentoption) + return rssuri, feed, feedname, nopatternurinoglobalpattern From 23b6beee207116bf5a08eb99b3c94b6c9a828a5a Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 11:51:18 +0200 Subject: [PATCH 26/79] add Alexis Metaireau --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 0ed4805..96af06f 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,2 +1,3 @@ Antoine Beaupré Carl Chenet +Alexis Metaireau From 3be1a8c94376c1b9d1561a953aaed89f7a8e3eff Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 15:20:12 +0200 Subject: [PATCH 27/79] bump version and update documentation --- CHANGELOG | 11 +++++++++++ README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/configure.rst | 6 +++--- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 6 +++--- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a1fc5ac..acf0bed 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +## [0.6] - 2017-08-02 +### Added +- define a name for a feed, accessible with {feedname}. Contributed by Alexis Metaireau. +- switch the toot visibility by The Dod (@thedod) +- new accept_bozo_exceptions option to allow malformed rss feeds. Contributed by Alexis Metaireau. + +### Changed +- configuration parser was split into much smaller chunks +- remove useless imports and coding style. Contributed by Alexis Metaireau. +- rephrasing and reformatting of the script register_feed2toot_app. Contributed by Bastien Guerry. + ## [0.5] - 2017-05-05 ### Added - manage rss feeds entries without id diff --git a/README.md b/README.md index 033adf0..cbe6a16 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.5.tar.gz + # tar zxvf feed2toot-0.6.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 84c7fef..1f0a11a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2017, Carl Chenet ' # built documents. # # The short X.Y version. -version = '1.0' +version = '0.6' # The full version, including alpha/beta/rc tags. -release = '1.0' +release = '0.6' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 4c88c7a..ecf2b62 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -137,9 +137,9 @@ It is possible to get all entries from a rss feed available in the uri_list file In you rsslist.txt, just don't give anything else than the needed feed url to get all the entries:: -https://www.journalduhacker.net/rss|title|hacker,psql -https://carlchenet.com/feed|title|gitlab -https://blog.linuxjobs.fr/feed.php?rss + https://www.journalduhacker.net/rss|title|hacker,psql + https://carlchenet.com/feed|title|gitlab + https://blog.linuxjobs.fr/feed.php?rss The last line of the file above only has the url of a rss feed. All entries from this feed will be tweeted. diff --git a/docs/source/install.rst b/docs/source/install.rst index 260d73b..848dbe4 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.5.tar.gz + $ tar zxvf feed2toot-0.6.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index a67cba3..a71c6d4 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -23,7 +23,7 @@ import logging import os.path import sys -__version__ = '0.5' +__version__ = '0.6' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index 6f7c764..4567863 100755 --- a/setup.py +++ b/setup.py @@ -31,10 +31,10 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.5', + version='0.6', license='GNU GPL v3', - description='Parse rss feed and tweet new posts to Mastodon', - long_description='Parse rss feed and tweet new posts to the Mastodon social network', + description='Parse rss feeds and tweet new posts to Mastodon', + long_description='Parse rss feeds and tweet new posts to the Mastodon social network', author = 'Carl Chenet', author_email = 'chaica@ohmytux.com', url = 'https://gitlab.com/chaica/feed2toot', From bdaa166211c2143a3640256543a2d16be20b7af9 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 15:30:58 +0200 Subject: [PATCH 28/79] correctly crediting The Dod --- AUTHORS | 1 + CHANGELOG | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index 96af06f..e0ce7ee 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,3 +1,4 @@ Antoine Beaupré Carl Chenet Alexis Metaireau +The Dod diff --git a/CHANGELOG b/CHANGELOG index acf0bed..3e86421 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,7 @@ ## [0.6] - 2017-08-02 ### Added - define a name for a feed, accessible with {feedname}. Contributed by Alexis Metaireau. -- switch the toot visibility by The Dod (@thedod) +- switch the toot visibility. Contributed by The Dod. - new accept_bozo_exceptions option to allow malformed rss feeds. Contributed by Alexis Metaireau. ### Changed From 208b5881b86fb18aa7e3d84106924fc6c092d5ca Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:11:01 +0200 Subject: [PATCH 29/79] fix issue with default name of the measurements for the influx plugin --- feed2toot/confparsers/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feed2toot/confparsers/plugins.py b/feed2toot/confparsers/plugins.py index b92269d..7600dd2 100644 --- a/feed2toot/confparsers/plugins.py +++ b/feed2toot/confparsers/plugins.py @@ -28,7 +28,7 @@ def parseplugins(config): # host, port, user, pass, database options ########################################## plugins[section] = {} - for currentoption in ['host', 'port', 'user', 'pass', 'database']: + for currentoption in ['host', 'port', 'user', 'pass', 'database', 'measurement']: if config.has_option(section, currentoption): plugins[section][currentoption] = config.get(section, currentoption) if 'host' not in plugins[section]: @@ -36,7 +36,7 @@ def parseplugins(config): if 'port' not in plugins[section]: plugins[section]['port'] = 8086 if 'measurement' not in plugins[section]: - plugins[section]['measurement'] = 'tweets' + plugins[section]['measurement'] = 'toots' for field in ['user', 'pass', 'database']: if field not in plugins[section]: sys.exit('Parsing error for {field} in the [{section}] section: {field} is not defined'.format(field=field, section=section)) From 8d24597e842de7c9cebfd652cc31044424e80886 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:16:28 +0200 Subject: [PATCH 30/79] remove useless linebreak --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 4567863..89f670b 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.6' ] - setup( name='feed2toot', version='0.6', From cf1d404d54279156e9340899533b889abd28f3fd Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:21:01 +0200 Subject: [PATCH 31/79] replace newly deprecated tweet option by toot option --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cbe6a16..7737524 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Alternatively you can donate cryptocurrencies: [rss] uri=https://www.journalduhacker.net/rss - tweet={title} {link} + toot={title} {link} [hashtaglist] several_words_hashtags_list=hashtags.txt From 8dfd51754f22312b88911a98b595644f64225edf Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:22:53 +0200 Subject: [PATCH 32/79] replace tweet by toot --- docs/source/configure.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index ecf2b62..513493e 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -36,7 +36,7 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any [rss] uri=https://www.journalduhacker.net/rss uri_list=/etc/feed2toot//rsslist.txt - tweet={title} {link} + toot={title} {link} title_pattern=Open Source title_pattern_case_sensitive=true no_uri_pattern_no_global_pattern=true @@ -61,14 +61,14 @@ __ https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md#p For the [cache] section: -- cachefile: the path to the cache file storing ids of already tweeted links. Absolute path is mandatory. This file should always use the .db extension. +- cachefile: the path to the cache file storing ids of already tooted links. Absolute path is mandatory. This file should always use the .db extension. - cache_limit: length of the cache queue. defaults to 100. For the [rss] section: - uri: the url of the rss feed to parse - uri_list: a path to a file with several adresses of rss feeds, one by line. Absolute path is mandatory. -- tweet: format of the tweet you want to post. It should use existing entries of the RSS fields like {title} or {link}. Launch it with this field empty to display all available entries. +- toot: format of the toot you want to post. It should use existing entries of the RSS fields like {title} or {link}. Launch it with this field empty to display all available entries. - {one field of the rss feed}_pattern: takes a string representing a pattern to match for a specified field of each rss entry of the rss feed, like title_pattern or summary_pattern. - {one field of the rss feed}_pattern_case_sensitive: either the pattern matching for the specified field should be case sensitive or not. Default to true if not specified. - no_uri_pattern_no_global_pattern: don't apply global pattern (see above) when no pattern-by-uri is defined in the uri_list. Allows to get all entries of a rss in the uri_list because no pattern is defined so we match them all. Defaults to false, meaning the global patterns will be tried on every rss in the uri_list NOT HAVING specific patterns and so ONLY entries from the specific uri in the uri_list matching the global patterns will be considered. @@ -98,7 +98,7 @@ With the parameter **uri_list**, you can define a list of uri to use. Feed2toot [rss] uri_list=/home/john/feed2toot/rsslist.txt - tweet={title} {link} + toot={title} {link} Now let's have a look at the =/home/john/feed2toot/rsslist.txt file:: @@ -114,7 +114,7 @@ If you want to display the name of the feed in the resulting toot, you can do so Le journal du hacker -Then in the `tweet` configuration, you can use the `{feedname}` syntax, which will be replaced by the actual name of the feed. +Then in the `toot` configuration, you can use the `{feedname}` syntax, which will be replaced by the actual name of the feed. Match specific patterns of rss feeds in the uri_list files ---------------------------------------------------------- @@ -141,7 +141,7 @@ In you rsslist.txt, just don't give anything else than the needed feed url to ge https://carlchenet.com/feed|title|gitlab https://blog.linuxjobs.fr/feed.php?rss -The last line of the file above only has the url of a rss feed. All entries from this feed will be tweeted. +The last line of the file above only has the url of a rss feed. All entries from this feed will be tooted. How to display available sections of the rss feed ================================================= From 68015a133b7a70a37b372c5874e8e255ec964dc9 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:26:29 +0200 Subject: [PATCH 33/79] replaced tweet by toot --- docs/source/plugins.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index b885c89..2c1e510 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -4,7 +4,7 @@ Feed2toot supports plugins. Plugins offer optional features, not supported by de InfluxDB -------- -The InfluxDB plugin allows to store already published tweets in a InfluxDB database. +The InfluxDB plugin allows to store already published toots in a InfluxDB database. Install the InfluxDB plugin ^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -28,7 +28,7 @@ Below is the block of configuration to add in your feed2toot.ini:: user=influxuser pass=V3ryS3cr3t database=influxdb - measurement=tweets + measurement=toots - host: the host where the influxdb instance is. Defaults to 127.0.0.1 - port: the port where the influxdb instance is listening to. Defaults to 8086 From a9c88f3cd1d5d5452a3ff21cd046af42327391c2 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:30:28 +0200 Subject: [PATCH 34/79] improve short and long description --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 89f670b..2b01f37 100755 --- a/setup.py +++ b/setup.py @@ -32,8 +32,8 @@ setup( name='feed2toot', version='0.6', license='GNU GPL v3', - description='Parse rss feeds and tweet new posts to Mastodon', - long_description='Parse rss feeds and tweet new posts to the Mastodon social network', + description='Parse rss feeds and send new posts to Mastodon', + long_description='Parse rss feeds and send new posts to the Mastodon social network', author = 'Carl Chenet', author_email = 'chaica@ohmytux.com', url = 'https://gitlab.com/chaica/feed2toot', From 485cff0af660e0dcc6a35e52d66e0517710dfb76 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 2 Aug 2017 17:32:00 +0200 Subject: [PATCH 35/79] add missing os module import --- feed2toot/cliparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index a71c6d4..437f121 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -20,6 +20,7 @@ from argparse import ArgumentParser import glob import logging +import os import os.path import sys From c811653d317e92cd4a2d5a8501855d7266b4a623 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 4 Sep 2017 09:55:38 +0200 Subject: [PATCH 36/79] change bitcoin address --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7737524..21cb7fa 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ For the full documentation, [read it online](https://feed2toot.readthedocs.org/e If you would like, you can [support the development of this project on Liberapay](https://liberapay.com/carlchenet/). Alternatively you can donate cryptocurrencies: -- BTC: 1MaFaUbmJcTVN9wsm5bzqFyq6zWu434X3A +- BTC: 1AW12Zw93rx4NzWn5evcG7RNNEM2RSLmAC - XMR: 43GGv8KzVhxehv832FWPTF7FSVuWjuBarFd17QP163uxMaFyoqwmDf1aiRtS5jWgCiRsi73yqedNJJ6V1La2joznKHGAhDi ### Quick Install From e8ed25027c8183a37207dca381a98aacdf491b4b Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sat, 23 Sep 2017 15:20:46 +0200 Subject: [PATCH 37/79] fix wrong variable name. fixes #23 --- feed2toot/confparsers/rss/urilist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feed2toot/confparsers/rss/urilist.py b/feed2toot/confparsers/rss/urilist.py index 6b71c51..65a8943 100644 --- a/feed2toot/confparsers/rss/urilist.py +++ b/feed2toot/confparsers/rss/urilist.py @@ -23,7 +23,7 @@ import os.path import sys import re -def parseurilist(config, accept_bozo_exception): +def parseurilist(config, accept_bozo_exceptions): '''Parse configuration value of the uri_list option of the rss section''' bozoexception = False feeds = [] From 5af950ee306e362734048da4c27ceaafc490925c Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 26 Sep 2017 18:55:51 +0200 Subject: [PATCH 38/79] bump version and update changelog --- CHANGELOG | 4 ++++ README.md | 2 +- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 3e86421..52cf501 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +## [0.7] - 2017-09-26 +### Changed +- fix issue while using uri_list + ## [0.6] - 2017-08-02 ### Added - define a name for a feed, accessible with {feedname}. Contributed by Alexis Metaireau. diff --git a/README.md b/README.md index 21cb7fa..25bfec3 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.6.tar.gz + # tar zxvf feed2toot-0.7.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/install.rst b/docs/source/install.rst index 848dbe4..943428d 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.6.tar.gz + $ tar zxvf feed2toot-0.7.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 437f121..cbacd11 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.6' +__version__ = '0.7' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index 2b01f37..524b6b5 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.6', + version='0.7', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From d66415e2f4714628617912d88ab178278050dd50 Mon Sep 17 00:00:00 2001 From: Matthew Lorentz Date: Mon, 26 Feb 2018 18:52:25 -0500 Subject: [PATCH 39/79] Added code to strip html tags from RSS fields --- feed2toot/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/feed2toot/main.py b/feed2toot/main.py index 0b379c7..ff1bc22 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -31,6 +31,7 @@ from feed2toot.filterentry import FilterEntry from feed2toot.removeduplicates import RemoveDuplicates from feed2toot.tootpost import TootPost from feed2toot.feedcache import FeedCache +from bs4 import BeautifulSoup class Main: '''Main class of Feed2toot''' @@ -183,6 +184,9 @@ class Main: else: finaltweet = dedup.finaltweet + # strip html tags + finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text() + if clioptions.dryrun: if entrytosend: logging.warning('Would toot with visibility "{visibility}": {toot}'.format( From c548e0cecb0c90f2498b86401b5feb7c91377fd8 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 30 May 2018 23:24:47 +0200 Subject: [PATCH 40/79] add [media] parameter and bump to 0.8 --- .gitignore | 1 + CHANGELOG | 4 ++++ README.md | 2 +- docs/source/configure.rst | 7 ++++++ docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- feed2toot/confparse.py | 5 +++++ feed2toot/confparsers/media.py | 39 ++++++++++++++++++++++++++++++++++ feed2toot/main.py | 2 +- feed2toot/tootpost.py | 9 ++++++-- setup.py | 2 +- 11 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 feed2toot/confparsers/media.py diff --git a/.gitignore b/.gitignore index 7a3968d..f5189cf 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ feedlist *.ini *.txt *.bck +*.png diff --git a/CHANGELOG b/CHANGELOG index 52cf501..87edf46 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +## [0.8] - 2018-05-30 +### Added +- add the custom parameter in [media] section to join a custom media with every toots + ## [0.7] - 2017-09-26 ### Changed - fix issue while using uri_list diff --git a/README.md b/README.md index 25bfec3..f468e58 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.7.tar.gz + # tar zxvf feed2toot-0.8.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 513493e..1953f4d 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -47,6 +47,9 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any [feedparser] accept_bozo_exceptions=true + [media] + custom=/var/lib/feed2toot/media/logo.png + For the [mastodon] section: - instance_url: the url of your Mastodon instance @@ -81,6 +84,10 @@ for the [feedparser] section: - accept_bozo_exceptions: If set to true, feed2toot will accept malformed feeds, which are rejected by default. +For the [media] section: + +- custom: the path to a media (should be supported by Mastodon) to be posted with every Mastodon post. + Example of the list of hash tags ================================ The list of hash tags is a simple text file with one hash tag composed by several words on a single line:: diff --git a/docs/source/install.rst b/docs/source/install.rst index 943428d..966a647 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.7.tar.gz + $ tar zxvf feed2toot-0.8.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index cbacd11..e196382 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.7' +__version__ = '0.8' class CliParse: '''CliParse class''' diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index c6b9e20..ae5a723 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -31,6 +31,7 @@ import feedparser from feed2toot.confparsers.cache import parsecache from feed2toot.confparsers.hashtaglist import parsehashtaglist from feed2toot.confparsers.feedparser import parsefeedparser +from feed2toot.confparsers.media import parsemedia from feed2toot.confparsers.plugins import parseplugins from feed2toot.confparsers.rss.pattern import parsepattern from feed2toot.confparsers.rss.toot import parsetoot @@ -85,6 +86,10 @@ class ConfParse: ########################### options['hashtaglist'] = parsehashtaglist(self.clioptions.hashtaglist, config) ########################### + # the media section + ########################### + options['media'] = parsemedia(config) + ########################### # the plugins section ########################### plugins = parseplugins(config) diff --git a/feed2toot/confparsers/media.py b/feed2toot/confparsers/media.py new file mode 100644 index 0000000..4bd577f --- /dev/null +++ b/feed2toot/confparsers/media.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Thu, 7 Jun 2018 00:34:49 +0200 Subject: [PATCH 41/79] update changelog and bump version --- CHANGELOG | 4 ++++ README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 4 ++-- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 87edf46..bc29821 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +## [0.9] - 2018-06-07 +### Added +- remove html characters from toots. Contributed by Matthew Lorentz and Simounet + ## [0.8] - 2018-05-30 ### Added - add the custom parameter in [media] section to join a custom media with every toots diff --git a/README.md b/README.md index f468e58..366167c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.8.tar.gz + # tar zxvf feed2toot-0.9.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 1f0a11a..4cd8695 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2017, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.6' +version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.6' +release = '0.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index 966a647..cab5edf 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.8.tar.gz + $ tar zxvf feed2toot-0.9.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index e196382..f25dcf7 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.8' +__version__ = '0.9' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index dafcdd4..a270ea0 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.8', + version='0.9', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', @@ -41,7 +41,7 @@ setup( download_url='https://gitlab.com/chaica/feed2toot', packages=find_packages(), scripts=['scripts/feed2toot', 'scripts/register_feed2toot_app'], - install_requires=['feedparser', 'Mastodon.py'], + install_requires=['beautifulsoup4', 'feedparser', 'Mastodon.py'], extras_require={ 'influxdb': ["influxdb"] } From affc3b6d2821773e72c2596fcf257bcb497eb395 Mon Sep 17 00:00:00 2001 From: Matthias Henze Date: Wed, 12 Sep 2018 10:53:13 +0200 Subject: [PATCH 42/79] allowed toot formating and made tags optional --- feed2toot/confparse.py | 5 ++++ feed2toot/confparsers/rss/addtags.py | 34 ++++++++++++++++++++++++++++ feed2toot/main.py | 20 +++++++--------- 3 files changed, 47 insertions(+), 12 deletions(-) create mode 100644 feed2toot/confparsers/rss/addtags.py diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index ae5a723..a44d5e3 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -37,6 +37,7 @@ from feed2toot.confparsers.rss.pattern import parsepattern from feed2toot.confparsers.rss.toot import parsetoot from feed2toot.confparsers.rss.uri import parseuri from feed2toot.confparsers.rss.urilist import parseurilist +from feed2toot.confparsers.rss.addtags import parseaddtags class ConfParse: '''ConfParse class''' @@ -68,6 +69,10 @@ class ConfParse: # pattern and patter_case_sensitive format option ################################################# options['patterns'], options['patternscasesensitive'] = parsepattern(config) + ############################### + # addtags option, default: True + ############################### + options['addtags'] = parseaddtags(config) ################# # uri_list option ################# diff --git a/feed2toot/confparsers/rss/addtags.py b/feed2toot/confparsers/rss/addtags.py new file mode 100644 index 0000000..55bedf2 --- /dev/null +++ b/feed2toot/confparsers/rss/addtags.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2017 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Sat, 22 Sep 2018 17:06:32 +0200 Subject: [PATCH 43/79] add addtags in [rss] section and use of new syntax {field.xx} --- docs/source/configure.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 1953f4d..0203427 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -71,10 +71,11 @@ For the [rss] section: - uri: the url of the rss feed to parse - uri_list: a path to a file with several adresses of rss feeds, one by line. Absolute path is mandatory. -- toot: format of the toot you want to post. It should use existing entries of the RSS fields like {title} or {link}. Launch it with this field empty to display all available entries. +- toot: format of the toot you want to post. It should use existing entries of the RSS fields like {title} or {link}. Launch it with this field empty to display all available entries. If you want to shorten the size of a field, you can use the syntax {summary:.100} to cut the field "summary" of the rss feed after the first 100 characters (starting from Feed2toot 0.10). - {one field of the rss feed}_pattern: takes a string representing a pattern to match for a specified field of each rss entry of the rss feed, like title_pattern or summary_pattern. - {one field of the rss feed}_pattern_case_sensitive: either the pattern matching for the specified field should be case sensitive or not. Default to true if not specified. - no_uri_pattern_no_global_pattern: don't apply global pattern (see above) when no pattern-by-uri is defined in the uri_list. Allows to get all entries of a rss in the uri_list because no pattern is defined so we match them all. Defaults to false, meaning the global patterns will be tried on every rss in the uri_list NOT HAVING specific patterns and so ONLY entries from the specific uri in the uri_list matching the global patterns will be considered. + addtags: add the tags from the rss feed at the end of the toot. Defaults to true. For the [hashtaglist] section: From 5137eae3111804f2e212bc7e5eecc8fcb8a03b33 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sat, 22 Sep 2018 17:12:36 +0200 Subject: [PATCH 44/79] add 0.10 description --- CHANGELOG | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index bc29821..de9336a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +## [0.10] - 2018-09-22 +### Added +- new syntax for the toot parameter of [rss] section. Use {summary:.100} to cut the rss field summary after the first 100 characters. +- add the addtags parameter of the [rss] section + ## [0.9] - 2018-06-07 ### Added - remove html characters from toots. Contributed by Matthew Lorentz and Simounet From 1f33975a90a34956a41b8014deaa1aa72ac4b536 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sat, 22 Sep 2018 17:12:52 +0200 Subject: [PATCH 45/79] bump version --- README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 366167c..d87c7af 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.9.tar.gz + # tar zxvf feed2toot-0.10.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 4cd8695..57f7e23 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2017, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.9' +version = '0.10' # The full version, including alpha/beta/rc tags. -release = '0.9' +release = '0.10' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index cab5edf..2b0ca17 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.9.tar.gz + $ tar zxvf feed2toot-0.10.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index f25dcf7..349edcc 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.9' +__version__ = '0.10' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index a270ea0..bce9f9d 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.9', + version='0.10', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From 31831f0934c6c11e1ce42b0921e5ba647a933eaa Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sat, 22 Sep 2018 17:17:54 +0200 Subject: [PATCH 46/79] add Matthias Henze to the list of authors --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index e0ce7ee..bddf2e0 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,3 +2,4 @@ Antoine Beaupré Carl Chenet Alexis Metaireau The Dod +Matthias Henze From 04d5441be49520fccbb6437205c57eb1e73f3d23 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sat, 22 Sep 2018 17:19:08 +0200 Subject: [PATCH 47/79] mention contributions by Matthias Henze --- CHANGELOG | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index de9336a..a568522 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,7 @@ ## [0.10] - 2018-09-22 ### Added -- new syntax for the toot parameter of [rss] section. Use {summary:.100} to cut the rss field summary after the first 100 characters. -- add the addtags parameter of the [rss] section +- new syntax for the toot parameter of [rss] section. Use {summary:.100} to cut the rss field summary after the first 100 characters. Contributed by Matthias Henze. +- add the addtags parameter of the [rss] section. Contributed by Matthias Henze. ## [0.9] - 2018-06-07 ### Added From 8dbf986b342328cbf0dee195650628abf1fef6b7 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sat, 17 Aug 2019 15:26:49 +0200 Subject: [PATCH 48/79] use a lock file to check if only one feed2toot process runs at a given time --- feed2toot/cliparse.py | 6 ++++ feed2toot/confparse.py | 5 +++ feed2toot/confparsers/lock.py | 48 ++++++++++++++++++++++++++ feed2toot/lock.py | 63 +++++++++++++++++++++++++++++++++++ feed2toot/main.py | 5 +++ 5 files changed, 127 insertions(+) create mode 100644 feed2toot/confparsers/lock.py create mode 100644 feed2toot/lock.py diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 349edcc..2e47476 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -52,8 +52,14 @@ class CliParse: help='tweet all RSS items, regardless of cache') parser.add_argument('-l', '--limit', dest='limit', default=10, type=int, help='tweet only LIMIT items (default: %(default)s)') + parser.add_argument('-t', '--lock-timeout', dest='locktimeout', default=3600, type=int, + help='lock timeout in seconds after which feed2toot can removes the lock itself') parser.add_argument('--cachefile', dest='cachefile', help='location of the cache file (default: %(default)s)') + parser.add_argument('--lockfile', dest='lockfile', + default=os.path.join(os.getenv('XDG_CONFIG_HOME', '~/.config'), + 'feed2toot.lock'), + help='location of the lock file (default: %(default)s)') parser.add_argument('-n', '--dry-run', dest='dryrun', action='store_true', default=False, help='Do not actually post tweets') diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index a44d5e3..21d3a86 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -31,6 +31,7 @@ import feedparser from feed2toot.confparsers.cache import parsecache from feed2toot.confparsers.hashtaglist import parsehashtaglist from feed2toot.confparsers.feedparser import parsefeedparser +from feed2toot.confparsers.lock import parselock from feed2toot.confparsers.media import parsemedia from feed2toot.confparsers.plugins import parseplugins from feed2toot.confparsers.rss.pattern import parsepattern @@ -69,6 +70,10 @@ class ConfParse: # pattern and patter_case_sensitive format option ################################################# options['patterns'], options['patternscasesensitive'] = parsepattern(config) + ################################################# + # lock file options + ################################################# + options['lockfile'], options['locktimeout'] = parselock(self.clioptions.lockfile, self.clioptions.locktimeout, config) ############################### # addtags option, default: True ############################### diff --git a/feed2toot/confparsers/lock.py b/feed2toot/confparsers/lock.py new file mode 100644 index 0000000..06f478a --- /dev/null +++ b/feed2toot/confparsers/lock.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import datetime +import logging +import os +import os.path +import sys + +class LockFile: + '''LockFile object''' + def __init__(self, lockfile, locktimeout): + '''check the lockfile and the locktimeout''' + self.lockfile = lockfile + ltimeout = datetime.timedelta(seconds=locktimeout) + self.lfdateformat = '%Y-%m-%d_%H-%M-%S' + # if a lock file exists + if os.path.exists(self.lockfile): + if os.path.isfile(self.lockfile): + with open(self.lockfile, 'r') as lf: + lfcontent = lf.read().rstrip() + # lfcontent should be a datetime + logging.debug('Check if lock file is older than timeout ({timeout} secs)'.format(timeout=locktimeout)) + locktime = datetime.datetime.strptime(lfcontent, self.lfdateformat) + if locktime < (datetime.datetime.now() - ltimeout): + # remove the lock file + logging.debug('Found an expired lock file') + self.release() + self.create_lock() + else: + # quit because another feed2toot process is running + logging.debug('Found a valid lock file. Exiting immediately.') + sys.exit(0) + else: + # no lock file. Creating one + self.create_lock() + + def create_lock(self): + '''Create a lock file''' + with open(self.lockfile, 'w') as lf: + currentdatestring = datetime.datetime.now().strftime(self.lfdateformat) + lf.write(currentdatestring) + logging.debug('lockfile {lockfile} created.'.format(lockfile=self.lockfile)) + + def release(self): + '''Release the lockfile''' + os.remove(self.lockfile) + logging.debug('Removed lock file.') diff --git a/feed2toot/main.py b/feed2toot/main.py index c573651..625589e 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -32,6 +32,7 @@ from feed2toot.filterentry import FilterEntry from feed2toot.removeduplicates import RemoveDuplicates from feed2toot.tootpost import TootPost from feed2toot.feedcache import FeedCache +from feed2toot.lock import LockFile from bs4 import BeautifulSoup class Main: @@ -71,6 +72,8 @@ class Main: tweetformat = conf[2] feeds = conf[3] plugins = conf[4] + # check the logfile and logtimeout + lockfile = LockFile(options['lockfile'], options['locktimeout']) # create link to the persistent list cache = FeedCache(options) if 'hashtaglist' in options and options['hashtaglist']: @@ -221,3 +224,5 @@ class Main: print(err) # do not forget to close cache (shelf object) cache.close() + # release the lock file + lockfile.release() From 898ff9b7f1464c9916bf26a476e30cca83ccda96 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sun, 18 Aug 2019 12:08:27 +0200 Subject: [PATCH 49/79] update copyright and supported python versions --- docs/source/conf.py | 2 +- feed2toot.py | 2 +- feed2toot/addtags.py | 2 +- feed2toot/cliparse.py | 2 +- feed2toot/confparse.py | 2 +- feed2toot/confparsers/__init__.py | 2 +- feed2toot/confparsers/cache.py | 2 +- feed2toot/confparsers/feedparser.py | 2 +- feed2toot/confparsers/hashtaglist.py | 2 +- feed2toot/confparsers/media.py | 2 +- feed2toot/confparsers/plugins.py | 2 +- feed2toot/confparsers/rss/__init__.py | 2 +- feed2toot/confparsers/rss/addtags.py | 2 +- feed2toot/confparsers/rss/pattern.py | 2 +- feed2toot/confparsers/rss/toot.py | 2 +- feed2toot/confparsers/rss/uri.py | 2 +- feed2toot/confparsers/rss/urilist.py | 2 +- feed2toot/feedcache.py | 2 +- feed2toot/filterentry.py | 2 +- feed2toot/main.py | 2 +- feed2toot/removeduplicates.py | 2 +- feed2toot/tootpost.py | 2 +- scripts/feed2toot | 2 +- scripts/register_feed2toot_app | 2 +- setup.py | 5 +++-- 25 files changed, 27 insertions(+), 26 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 57f7e23..4c756fa 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,7 +47,7 @@ master_doc = 'index' # General information about the project. project = 'feed2toot' -copyright = '2017, Carl Chenet ' +copyright = '2015-2019, Carl Chenet ' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/feed2toot.py b/feed2toot.py index d7ce8ba..1767e41 100755 --- a/feed2toot.py +++ b/feed2toot.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/addtags.py b/feed2toot/addtags.py index 1859e07..936895e 100644 --- a/feed2toot/addtags.py +++ b/feed2toot/addtags.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 349edcc..af5af7b 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index a44d5e3..10950e6 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/__init__.py b/feed2toot/confparsers/__init__.py index d19bf9f..d77b4df 100644 --- a/feed2toot/confparsers/__init__.py +++ b/feed2toot/confparsers/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2017 Carl Chenet +# Copyright © 2017-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/cache.py b/feed2toot/confparsers/cache.py index 5490b1a..c6a7d52 100644 --- a/feed2toot/confparsers/cache.py +++ b/feed2toot/confparsers/cache.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/feedparser.py b/feed2toot/confparsers/feedparser.py index e3c192a..af2fddc 100644 --- a/feed2toot/confparsers/feedparser.py +++ b/feed2toot/confparsers/feedparser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtaglist.py b/feed2toot/confparsers/hashtaglist.py index e7ba0ee..63e5854 100644 --- a/feed2toot/confparsers/hashtaglist.py +++ b/feed2toot/confparsers/hashtaglist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/media.py b/feed2toot/confparsers/media.py index 4bd577f..a150768 100644 --- a/feed2toot/confparsers/media.py +++ b/feed2toot/confparsers/media.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/plugins.py b/feed2toot/confparsers/plugins.py index 7600dd2..d811f4a 100644 --- a/feed2toot/confparsers/plugins.py +++ b/feed2toot/confparsers/plugins.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/__init__.py b/feed2toot/confparsers/rss/__init__.py index d19bf9f..d77b4df 100644 --- a/feed2toot/confparsers/rss/__init__.py +++ b/feed2toot/confparsers/rss/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2017 Carl Chenet +# Copyright © 2017-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/addtags.py b/feed2toot/confparsers/rss/addtags.py index 55bedf2..9a2f8a7 100644 --- a/feed2toot/confparsers/rss/addtags.py +++ b/feed2toot/confparsers/rss/addtags.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/pattern.py b/feed2toot/confparsers/rss/pattern.py index 273ee25..b573035 100644 --- a/feed2toot/confparsers/rss/pattern.py +++ b/feed2toot/confparsers/rss/pattern.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/toot.py b/feed2toot/confparsers/rss/toot.py index 0c82220..2fcf0be 100644 --- a/feed2toot/confparsers/rss/toot.py +++ b/feed2toot/confparsers/rss/toot.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/uri.py b/feed2toot/confparsers/rss/uri.py index d574849..eb979e0 100644 --- a/feed2toot/confparsers/rss/uri.py +++ b/feed2toot/confparsers/rss/uri.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/urilist.py b/feed2toot/confparsers/rss/urilist.py index 65a8943..ca9f381 100644 --- a/feed2toot/confparsers/rss/urilist.py +++ b/feed2toot/confparsers/rss/urilist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/feedcache.py b/feed2toot/feedcache.py index ca030f7..1f457df 100644 --- a/feed2toot/feedcache.py +++ b/feed2toot/feedcache.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/filterentry.py b/feed2toot/filterentry.py index 4d9d793..3372bdc 100644 --- a/feed2toot/filterentry.py +++ b/feed2toot/filterentry.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/main.py b/feed2toot/main.py index c573651..dc5f4a3 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/removeduplicates.py b/feed2toot/removeduplicates.py index 6e857b1..3c609d3 100644 --- a/feed2toot/removeduplicates.py +++ b/feed2toot/removeduplicates.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/tootpost.py b/feed2toot/tootpost.py index ddf0a0a..a6bd8cb 100644 --- a/feed2toot/tootpost.py +++ b/feed2toot/tootpost.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/scripts/feed2toot b/scripts/feed2toot index cb5f300..8ba5242 100755 --- a/scripts/feed2toot +++ b/scripts/feed2toot @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/scripts/register_feed2toot_app b/scripts/register_feed2toot_app index 1ea856e..4b0eb0d 100755 --- a/scripts/register_feed2toot_app +++ b/scripts/register_feed2toot_app @@ -1,7 +1,7 @@ #!/usr/bin/env python3 #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2017 Carl Chenet +# Copyright © 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/setup.py b/setup.py index bce9f9d..e4f5b5d 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Copyright 2015-2017 Carl Chenet +# Copyright 2015-2019 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -25,7 +25,8 @@ CLASSIFIERS = [ 'Operating System :: POSIX :: Linux', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6' + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7' ] setup( From 4048bd9abd44a2a8c28eb2ace3cc80a4c28b7b9b Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Thu, 22 Aug 2019 00:41:51 +0200 Subject: [PATCH 50/79] bump version, write doc, write changelog --- CHANGELOG | 20 +++++++++++++------- README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/configure.rst | 9 +++++++++ docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 7 files changed, 28 insertions(+), 13 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a568522..44b7a01 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,7 +1,13 @@ +## [0.11] - 2019-08-24 +### Added +- command line options --lock-file to define a lock file +- command line options --lock-timeout to remove this lock file automatically +- lock section in configuration with lock_file andd lock_time parameters + ## [0.10] - 2018-09-22 ### Added -- new syntax for the toot parameter of [rss] section. Use {summary:.100} to cut the rss field summary after the first 100 characters. Contributed by Matthias Henze. -- add the addtags parameter of the [rss] section. Contributed by Matthias Henze. +- new syntax for the toot parameter of [rss] section. Use {summary:.100} to cut the rss field summary after the first 100 characters. Contributed by Matthias Henze +- add the addtags parameter of the [rss] section. Contributed by Matthias Henze ## [0.9] - 2018-06-07 ### Added @@ -17,14 +23,14 @@ ## [0.6] - 2017-08-02 ### Added -- define a name for a feed, accessible with {feedname}. Contributed by Alexis Metaireau. -- switch the toot visibility. Contributed by The Dod. -- new accept_bozo_exceptions option to allow malformed rss feeds. Contributed by Alexis Metaireau. +- define a name for a feed, accessible with {feedname}, contributed by Alexis Metaireau +- switch the toot visibility. Contributed by The Dod +- new accept_bozo_exceptions option to allow malformed rss feeds, contributed by Alexis Metaireau ### Changed - configuration parser was split into much smaller chunks -- remove useless imports and coding style. Contributed by Alexis Metaireau. -- rephrasing and reformatting of the script register_feed2toot_app. Contributed by Bastien Guerry. +- remove useless imports and coding style, contributed by Alexis Metaireau +- rephrasing and reformatting of the script register_feed2toot_app, contributed by Bastien Guerry ## [0.5] - 2017-05-05 ### Added diff --git a/README.md b/README.md index d87c7af..46e6e3b 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.10.tar.gz + # tar zxvf feed2toot-0.11.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 4c756fa..a6641c7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2015-2019, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.10' +version = '0.11' # The full version, including alpha/beta/rc tags. -release = '0.10' +release = '0.11' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 0203427..25677da 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -33,6 +33,10 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any cachefile=/var/lib/feed2toot/feed2toot.db cache_limit=10000 + [lock] + lock_file=/var/lock/feed2toot.lock + lock_timeout=3600 + [rss] uri=https://www.journalduhacker.net/rss uri_list=/etc/feed2toot//rsslist.txt @@ -67,6 +71,11 @@ For the [cache] section: - cachefile: the path to the cache file storing ids of already tooted links. Absolute path is mandatory. This file should always use the .db extension. - cache_limit: length of the cache queue. defaults to 100. +For the [lock] section (starting from version 0.11): + +- lock_file: lock to stop any other feed2toot instance to run at the same time. Default is ~/.config/feed2toot.lock +- lock_timeout: automatically remove the lock if the datetime in the lock file is greater than n seconds. Default is 3600 seconds. + For the [rss] section: - uri: the url of the rss feed to parse diff --git a/docs/source/install.rst b/docs/source/install.rst index 2b0ca17..ef0a3f4 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.10.tar.gz + $ tar zxvf feed2toot-0.11.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 9862c4e..7575a48 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.10' +__version__ = '0.11' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index e4f5b5d..69d9870 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.10', + version='0.11', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From b7f3b20f57ef372053b6c76eccbcd7af470f655e Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 23 Aug 2019 01:06:12 +0200 Subject: [PATCH 51/79] add 0.11 in changelog --- CHANGELOG | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 44b7a01..99e1e5e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,8 @@ ## [0.11] - 2019-08-24 ### Added -- command line options --lock-file to define a lock file -- command line options --lock-timeout to remove this lock file automatically -- lock section in configuration with lock_file andd lock_time parameters +- MAJOR CHANGE: command line options --lockfile to define a lock file +- MAJOR CHANGE: command line options --lock-timeout to remove this lock file automatically +- MAJOR CHANGE: lock section in configuration with lock_file andd lock_time parameters ## [0.10] - 2018-09-22 ### Added From ec664e24d344275ec6954f2518aa0747028db692 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 23 Aug 2019 01:10:20 +0200 Subject: [PATCH 52/79] fix typo in the warning message when the parent directory of the cache file does not exist --- feed2toot/confparsers/lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feed2toot/confparsers/lock.py b/feed2toot/confparsers/lock.py index 06f478a..9588d9c 100644 --- a/feed2toot/confparsers/lock.py +++ b/feed2toot/confparsers/lock.py @@ -34,7 +34,7 @@ def parselock(lockfile, locktimeout, config): lockfile = os.path.expanduser(lockfile) lockfileparent = os.path.dirname(lockfile) if lockfileparent and not os.path.exists(lockfileparent): - sys.exit('The parent directory of the cache file does not exist: {lockfileparent}'.format(lockfileparent=lockfileparent)) + sys.exit('The parent directory of the lock file does not exist: {lockfileparent}'.format(lockfileparent=lockfileparent)) ###################### # lock_timeout option ###################### From 3b5e66af40b46460de55e9fb85be794e5c811593 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 26 Aug 2019 15:34:37 +0200 Subject: [PATCH 53/79] push code from main to new modules --- feed2toot/hashtags.py | 66 +++++++++++++++ feed2toot/main.py | 147 ++++++---------------------------- feed2toot/message.py | 72 +++++++++++++++++ feed2toot/plugins/__init__.py | 16 ++++ feed2toot/rss.py | 42 ++++++++++ feed2toot/sortentries.py | 42 ++++++++++ 6 files changed, 262 insertions(+), 123 deletions(-) create mode 100644 feed2toot/hashtags.py create mode 100644 feed2toot/message.py create mode 100644 feed2toot/rss.py create mode 100644 feed2toot/sortentries.py diff --git a/feed2toot/hashtags.py b/feed2toot/hashtags.py new file mode 100644 index 0000000..88405b0 --- /dev/null +++ b/feed2toot/hashtags.py @@ -0,0 +1,66 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import codecs + +def extract_hashtags_from_list(options): + '''extract hashtags from the the list''' + if 'hashtaglist' in options and options['hashtaglist']: + severalwordshashtags = codecs.open(options['hashtaglist'], + encoding='utf-8').readlines() + severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags] + else: + severalwordshashtags = [] + return severalwordshashtags + +def build_hashtags(entry, rss, options, severalwordshashtags): + '''build hashtags''' + severalwordsinhashtag = False + # has the the rss feed hashtag + if 'tags' in entry and options['addtags']: + hastags = True + else: + hastags = False + if hastags: + rss['hashtags'] = [] + for i, _ in enumerate(entry['tags']): + if 'hashtaglist' in options: + prehashtags = entry['tags'][i]['term'] + tmphashtags = entry['tags'][i]['term'] + for element in severalwordshashtags: + if element in prehashtags: + severalwordsinhashtag = True + tmphashtags = prehashtags.replace(element, + ''.join(element.split())) + # replace characters stopping a word from being a hashtag + if severalwordsinhashtag: + # remove ' from hashtag + tmphashtags = tmphashtags.replace("'", "") + # remove - from hashtag + tmphashtags = tmphashtags.replace("-", "") + # remove . from hashtag + tmphashtags = tmphashtags.replace(".", "") + # remove space from hashtag + finalhashtags = tmphashtags.replace(" ", "") + rss['hashtags'].append('#{}'.format(finalhashtags)) + else: + nospace = ''.join(entry['tags'][i]['term']) + # remove space from hashtag + nospace = nospace.replace(" ", "") + rss['hashtags'].append('#{}'.format(nospace)) + return rss diff --git a/feed2toot/main.py b/feed2toot/main.py index cebb434..ff2041c 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -24,16 +24,25 @@ import logging.handlers import sys import re +# external liraries imports +from bs4 import BeautifulSoup + # app libraries imports from feed2toot.addtags import AddTags from feed2toot.cliparse import CliParse from feed2toot.confparse import ConfParse -from feed2toot.filterentry import FilterEntry -from feed2toot.removeduplicates import RemoveDuplicates -from feed2toot.tootpost import TootPost from feed2toot.feedcache import FeedCache +from feed2toot.filterentry import FilterEntry +from feed2toot.hashtags import build_hashtags +from feed2toot.hashtags import extract_hashtags_from_list from feed2toot.lock import LockFile -from bs4 import BeautifulSoup +from feed2toot.message import build_message +from feed2toot.message import send_message_dry_run +from feed2toot.message import send_message +from feed2toot.removeduplicates import RemoveDuplicates +from feed2toot.rss import populate_rss +from feed2toot.sortentries import sort_entries +from feed2toot.tootpost import TootPost class Main: '''Main class of Feed2toot''' @@ -59,7 +68,7 @@ class Main: logging.debug('configured stdout level %s' % sh.level) def main(self): - """The main function.""" + '''The main function''' clip = CliParse() clioptions = clip.options self.setup_logging(clioptions) @@ -76,12 +85,7 @@ class Main: lockfile = LockFile(options['lockfile'], options['locktimeout']) # create link to the persistent list cache = FeedCache(options) - if 'hashtaglist' in options and options['hashtaglist']: - severalwordshashtags = codecs.open(options['hashtaglist'], - encoding='utf-8').readlines() - severalwordshashtags = [i.rstrip('\n') for i in severalwordshashtags] - else: - severalwordshashtags = [] + severalwordshashtags = extract_hashtags_from_list(options) # reverse feed entries because most recent one should be sent as the last one in Mastodon for feed in feeds: # store the patterns by rss @@ -96,73 +100,12 @@ class Main: sys.exit(0) else: sys.exit('Could not parse the section of the rss feed') - totweet = [] - # cache the ids of last rss feeds - if not clioptions.all: - for i in entries: - if 'id' in i: - if i['id'] not in cache.getdeque(): - totweet.append(i) - elif 'guid' in i: - if i['guid'] not in cache.getdeque(): - totweet.append(i) - else: - # if id or guid not in the entry, use link - if i['link'] not in cache.getdeque(): - totweet.append(i) - else: - totweet = entries - + # sort entries and check if they were not previously sent + totweet = sort_entries(clioptions.all, cache, entries) for entry in totweet: - if 'id' in entry: - logging.debug('found feed entry {entryid}'.format(entryid=entry['id'])) - rss = { - 'id': entry['id'], - } - elif 'guid' in entry: - logging.debug('found feed entry {entryid}'.format(entryid=entry['guid'])) - rss = { - 'id': entry['guid'], - } - else: - logging.debug('found feed entry {entryid}'.format(entryid=entry['link'])) - rss = { - 'id': entry['link'], - } - - severalwordsinhashtag = False - # lets see if the rss feed has hashtag - if 'tags' in entry and options['addtags']: - hastags = True - else: - hastags = False - if hastags: - rss['hashtags'] = [] - for i, _ in enumerate(entry['tags']): - if 'hashtaglist' in options: - prehashtags = entry['tags'][i]['term'] - tmphashtags = entry['tags'][i]['term'] - for element in severalwordshashtags: - if element in prehashtags: - severalwordsinhashtag = True - tmphashtags = prehashtags.replace(element, - ''.join(element.split())) - # replace characters stopping a word from being a hashtag - if severalwordsinhashtag: - # remove ' from hashtag - tmphashtags = tmphashtags.replace("'", "") - # remove - from hashtag - tmphashtags = tmphashtags.replace("-", "") - # remove . from hashtag - tmphashtags = tmphashtags.replace(".", "") - # remove space from hashtag - finalhashtags = tmphashtags.replace(" ", "") - rss['hashtags'].append('#{}'.format(finalhashtags)) - else: - nospace = ''.join(entry['tags'][i]['term']) - # remove space from hashtag - nospace = nospace.replace(" ", "") - rss['hashtags'].append('#{}'.format(nospace)) + # populate rss with new entry to send + rss = populate_rss(entry) + rss = build_hashtags(entry, rss, options, severalwordshashtags) # parse tweetfomat to elements elements = re.findall(r"\{(.*?)\}",tweetformat) # strip : from elements to allow string formating, eg. {title:.20} @@ -172,56 +115,14 @@ class Main: fe = FilterEntry(elements, entry, options, feed['patterns'], feed['rssobject'], feed['feedname']) entrytosend = fe.finalentry if entrytosend: - tweetwithnotag = tweetformat.format(**entrytosend) - # remove duplicates from the final tweet - dedup = RemoveDuplicates(tweetwithnotag) - # only append hashtags if they exist - # remove last tags if tweet too long - if 'hashtags' in rss: - addtag = AddTags(dedup.finaltweet, rss['hashtags']) - finaltweet = addtag.finaltweet - else: - finaltweet = dedup.finaltweet - - # strip html tags - finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text() - + finaltweet = build_message(entrytosend, tweetformat, rss) if clioptions.dryrun: - if entrytosend: - logging.warning('Would toot with visibility "{visibility}": {toot}'.format( - toot=finaltweet, - visibility=config.get( - 'mastodon', 'toot_visibility', - fallback='public'))) - else: - logging.debug('This rss entry did not meet pattern criteria. Should have not been sent') + send_message_dry_run(config, entrytosend, finaltweet) else: - storeit = True - if entrytosend and not clioptions.populate: - logging.debug('Tooting with visibility "{visibility}": {toot}'.format( - toot=finaltweet, - visibility=config.get( - 'mastodon', 'toot_visibility', - fallback='public'))) - twp = TootPost(config, options, finaltweet) - storeit = twp.storeit() - else: - logging.debug('populating RSS entry {}'.format(rss['id'])) - # in both cas we store the id of the sent tweet - if storeit: - cache.append(rss['id']) + send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss) # plugins if plugins and entrytosend: - for plugin in plugins: - capitalizedplugin = plugin.title() - pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin) - pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower()) - try: - pluginmodule = importlib.import_module(pluginmodulename) - pluginclass = getattr(pluginmodule, pluginclassname) - pluginclass(plugins[plugin], finaltweet) - except ImportError as err: - print(err) + activate_plugins(plugins) # do not forget to close cache (shelf object) cache.close() # release the lock file diff --git a/feed2toot/message.py b/feed2toot/message.py new file mode 100644 index 0000000..8ccbad0 --- /dev/null +++ b/feed2toot/message.py @@ -0,0 +1,72 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Build the message''' + +# standard libraires imports +import logging + +# external liraries imports +from bs4 import BeautifulSoup + +# app libraries imports +from feed2toot.addtags import AddTags +from feed2toot.removeduplicates import RemoveDuplicates +from feed2toot.tootpost import TootPost + +def build_message(entrytosend, tweetformat, rss): + '''populate the rss dict with the new entry''' + tweetwithnotag = tweetformat.format(**entrytosend) + # remove duplicates from the final tweet + dedup = RemoveDuplicates(tweetwithnotag) + # only append hashtags if they exist + # remove last tags if tweet too long + if 'hashtags' in rss: + addtag = AddTags(dedup.finaltweet, rss['hashtags']) + finaltweet = addtag.finaltweet + else: + finaltweet = dedup.finaltweet + + # strip html tags + finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text() + return finaltweet + +def send_message_dry_run(config, entrytosend, finaltweet): + '''simulate sending message using dry run mode''' + if entrytosend: + logging.warning('Would toot with visibility "{visibility}": {toot}'.format( + toot=finaltweet, + visibility=config.get( + 'mastodon', 'toot_visibility', + fallback='public'))) + else: + logging.debug('This rss entry did not meet pattern criteria. Should have not been sent') + +def send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss): + '''send message''' + storeit = True + if entrytosend and not clioptions.populate: + logging.debug('Tooting with visibility "{visibility}": {toot}'.format( + toot=finaltweet, + visibility=config.get( + 'mastodon', 'toot_visibility', + fallback='public'))) + twp = TootPost(config, options, finaltweet) + storeit = twp.storeit() + else: + logging.debug('populating RSS entry {}'.format(rss['id'])) + # in both cas we store the id of the sent tweet + if storeit: + cache.append(rss['id']) diff --git a/feed2toot/plugins/__init__.py b/feed2toot/plugins/__init__.py index d19bf9f..bcccc6a 100644 --- a/feed2toot/plugins/__init__.py +++ b/feed2toot/plugins/__init__.py @@ -13,3 +13,19 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see + +# standard libraires imports +import importlib + +def activate_plugins(entrytosend): + '''activate plugins''' + for plugin in plugins: + capitalizedplugin = plugin.title() + pluginclassname = '{plugin}Plugin'.format(plugin=capitalizedplugin) + pluginmodulename = 'feed2toot.plugins.{pluginmodule}'.format(pluginmodule=pluginclassname.lower()) + try: + pluginmodule = importlib.import_module(pluginmodulename) + pluginclass = getattr(pluginmodule, pluginclassname) + pluginclass(plugins[plugin], finaltweet) + except ImportError as err: + print(err) diff --git a/feed2toot/rss.py b/feed2toot/rss.py new file mode 100644 index 0000000..3edbbd7 --- /dev/null +++ b/feed2toot/rss.py @@ -0,0 +1,42 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import datetime +import logging +import os +import os.path +import sys + +def populate_rss(entry): + '''populate the rss dict with the new entry''' + if 'id' in entry: + logging.debug('found feed entry {entryid}'.format(entryid=entry['id'])) + rss = { + 'id': entry['id'], + } + elif 'guid' in entry: + logging.debug('found feed entry {entryid}'.format(entryid=entry['guid'])) + rss = { + 'id': entry['guid'], + } + else: + logging.debug('found feed entry {entryid}'.format(entryid=entry['link'])) + rss = { + 'id': entry['link'], + } + return rss diff --git a/feed2toot/sortentries.py b/feed2toot/sortentries.py new file mode 100644 index 0000000..09d02ba --- /dev/null +++ b/feed2toot/sortentries.py @@ -0,0 +1,42 @@ +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see + +'''Manage a lock file''' + +# standard libraires imports +import datetime +import logging +import os +import os.path +import sys + +def sort_entries(is_all, cache, entries): + '''sort entries before sending''' + totweet = [] + if not is_all: + for i in entries: + if 'id' in i: + if i['id'] not in cache.getdeque(): + totweet.append(i) + elif 'guid' in i: + if i['guid'] not in cache.getdeque(): + totweet.append(i) + else: + # if id or guid not in the entry, use link + if i['link'] not in cache.getdeque(): + totweet.append(i) + else: + totweet = entries + return totweet From 5c29dfed8d0871466dd439cc611e9a85c78bd6c0 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 26 Aug 2019 16:47:53 +0200 Subject: [PATCH 54/79] fix plugins --- feed2toot/main.py | 15 ++++++++------- feed2toot/plugins/__init__.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/feed2toot/main.py b/feed2toot/main.py index ff2041c..8e7b52a 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -39,6 +39,7 @@ from feed2toot.lock import LockFile from feed2toot.message import build_message from feed2toot.message import send_message_dry_run from feed2toot.message import send_message +from feed2toot.plugins import activate_plugins from feed2toot.removeduplicates import RemoveDuplicates from feed2toot.rss import populate_rss from feed2toot.sortentries import sort_entries @@ -116,13 +117,13 @@ class Main: entrytosend = fe.finalentry if entrytosend: finaltweet = build_message(entrytosend, tweetformat, rss) - if clioptions.dryrun: - send_message_dry_run(config, entrytosend, finaltweet) - else: - send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss) - # plugins - if plugins and entrytosend: - activate_plugins(plugins) + if clioptions.dryrun: + send_message_dry_run(config, entrytosend, finaltweet) + else: + send_message(config, clioptions, options, entrytosend, finaltweet, cache, rss) + # plugins + if plugins and entrytosend: + activate_plugins(plugins, finaltweet) # do not forget to close cache (shelf object) cache.close() # release the lock file diff --git a/feed2toot/plugins/__init__.py b/feed2toot/plugins/__init__.py index bcccc6a..f6ac8a4 100644 --- a/feed2toot/plugins/__init__.py +++ b/feed2toot/plugins/__init__.py @@ -17,7 +17,7 @@ # standard libraires imports import importlib -def activate_plugins(entrytosend): +def activate_plugins(plugins, finaltweet): '''activate plugins''' for plugin in plugins: capitalizedplugin = plugin.title() From 5243ac4ea726c085e4c20124eb2755e955dc093c Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 27 Aug 2019 18:58:39 +0200 Subject: [PATCH 55/79] fix missing default vale for case sensitve pattern matching --- feed2toot/confparsers/rss/pattern.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/feed2toot/confparsers/rss/pattern.py b/feed2toot/confparsers/rss/pattern.py index b573035..183917b 100644 --- a/feed2toot/confparsers/rss/pattern.py +++ b/feed2toot/confparsers/rss/pattern.py @@ -49,4 +49,7 @@ def parsepattern(config): except ValueError as err: logging.warn(err) patternscasesensitive[currentoption] = True + else: + # default value + patternscasesensitive[currentoption] = False return patterns, patternscasesensitive From b0de8058ba776422e742d7289ab64dd38121f064 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 27 Aug 2019 19:08:33 +0200 Subject: [PATCH 56/79] bump version, update changelog --- CHANGELOG | 5 +++++ README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 99e1e5e..caca498 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +## [0.12] - 2019-08-27 +### Changed +- simplify code, mostly in the main function +- fix a bug when title_pattern is used and *_pattern_case_sensitive is not defined + ## [0.11] - 2019-08-24 ### Added - MAJOR CHANGE: command line options --lockfile to define a lock file diff --git a/README.md b/README.md index 46e6e3b..2ee0bf4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* - # tar zxvf feed2toot-0.11.tar.gz + # tar zxvf feed2toot-0.12.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index a6641c7..84d188e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2015-2019, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.11' +version = '0.12' # The full version, including alpha/beta/rc tags. -release = '0.11' +release = '0.12' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index ef0a3f4..749be21 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.11.tar.gz + $ tar zxvf feed2toot-0.12.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 7575a48..545c906 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.11' +__version__ = '0.12' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index 69d9870..e4c9a6a 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.11', + version='0.12', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From bae51858e6121d2e37df004f60b90bfb6688e2cb Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 8 Nov 2019 09:28:39 +0100 Subject: [PATCH 57/79] fix typos in readthedocs url. fixes #46 --- README.md | 4 ++-- feed2toot/cliparse.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2ee0bf4..29169f5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ ### Feed2toot Feed2toot automatically parses rss feeds, identifies new posts and posts them on the [Mastodon](https://mastodon.social) social network. -For the full documentation, [read it online](https://feed2toot.readthedocs.org/en/latest/). +For the full documentation, [read it online](https://feed2toot.readthedocs.io/en/latest/). If you would like, you can [support the development of this project on Liberapay](https://liberapay.com/carlchenet/). Alternatively you can donate cryptocurrencies: @@ -17,7 +17,7 @@ Alternatively you can donate cryptocurrencies: * Install Feed2toot from sources *(see the installation guide for full details) - [Installation Guide](http://feed2toot.readthedocs.org/en/latest/install.html)* + [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* # tar zxvf feed2toot-0.12.tar.gz diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 545c906..971d2f3 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -34,7 +34,7 @@ class CliParse: def main(self): '''main of CliParse class''' - feed2tootepilog = 'For more information: https://feed2toot.readhthedocs.org' + feed2tootepilog = 'For more information: https://feed2toot.readthedocs.io' feed2tootdescription = 'Take rss feed and send it to Mastodon' parser = ArgumentParser(prog='feed2toot', description=feed2tootdescription, From 15415e369a7d07517af318d3f66d8c690d06e250 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 25 Dec 2019 23:27:44 +0100 Subject: [PATCH 58/79] bypass ssl security while fetching invalid https url --- docs/source/configure.rst | 2 ++ feed2toot/cliparse.py | 3 +++ feed2toot/confparse.py | 9 ++++++-- feed2toot/confparsers/rss/ignoressl.py | 30 ++++++++++++++++++++++++++ feed2toot/confparsers/rss/uri.py | 7 +++++- feed2toot/confparsers/rss/urilist.py | 7 +++++- 6 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 feed2toot/confparsers/rss/ignoressl.py diff --git a/docs/source/configure.rst b/docs/source/configure.rst index 25677da..a6f1170 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -44,6 +44,7 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any title_pattern=Open Source title_pattern_case_sensitive=true no_uri_pattern_no_global_pattern=true + ; ignore_ssl=false [hashtaglist] several_words_hashtags_list=/etc/feed2toot/hashtags.txt @@ -85,6 +86,7 @@ For the [rss] section: - {one field of the rss feed}_pattern_case_sensitive: either the pattern matching for the specified field should be case sensitive or not. Default to true if not specified. - no_uri_pattern_no_global_pattern: don't apply global pattern (see above) when no pattern-by-uri is defined in the uri_list. Allows to get all entries of a rss in the uri_list because no pattern is defined so we match them all. Defaults to false, meaning the global patterns will be tried on every rss in the uri_list NOT HAVING specific patterns and so ONLY entries from the specific uri in the uri_list matching the global patterns will be considered. addtags: add the tags from the rss feed at the end of the toot. Defaults to true. +- ignore_ssl: when the uri or uri_list contains an https url with an invalid certificate (e.g an expired one), feed2toot will be unable to get rss content. This option allows to bypass the ssl security to catch the rss content. Defaults to false. For the [hashtaglist] section: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 971d2f3..7e36d46 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -50,6 +50,9 @@ class CliParse: parser.add_argument('-a', '--all', action='store_true', default=False, dest='all', help='tweet all RSS items, regardless of cache') + parser.add_argument('--ignore-ssl', action='store_true', default=False, + dest='ignore_ssl', + help='ignore ssl errors while fetching rss feeds') parser.add_argument('-l', '--limit', dest='limit', default=10, type=int, help='tweet only LIMIT items (default: %(default)s)') parser.add_argument('-t', '--lock-timeout', dest='locktimeout', default=3600, type=int, diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index f592bb1..dcf6ee1 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -34,6 +34,7 @@ from feed2toot.confparsers.feedparser import parsefeedparser from feed2toot.confparsers.lock import parselock from feed2toot.confparsers.media import parsemedia from feed2toot.confparsers.plugins import parseplugins +from feed2toot.confparsers.rss.ignoressl import parseignoressl from feed2toot.confparsers.rss.pattern import parsepattern from feed2toot.confparsers.rss.toot import parsetoot from feed2toot.confparsers.rss.uri import parseuri @@ -78,15 +79,19 @@ class ConfParse: # addtags option, default: True ############################### options['addtags'] = parseaddtags(config) + ################### + # ignore_ssl option + ################### + ignore_ssl = parseignoressl(config, self.clioptions.ignore_ssl) ################# # uri_list option ################# feeds = [] - feeds = parseurilist(config, accept_bozo_exceptions) + feeds = parseurilist(config, accept_bozo_exceptions, ignore_ssl) ############ # uri option ############ - options['rss_uri'], feed, feedname, options['nopatternurinoglobalpattern'] = parseuri(config, self.clioptions.rss_uri, feeds) + options['rss_uri'], feed, feedname, options['nopatternurinoglobalpattern'] = parseuri(config, self.clioptions.rss_uri, feeds, ignore_ssl) ########################### # the cache section ########################### diff --git a/feed2toot/confparsers/rss/ignoressl.py b/feed2toot/confparsers/rss/ignoressl.py new file mode 100644 index 0000000..269b19c --- /dev/null +++ b/feed2toot/confparsers/rss/ignoressl.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see Date: Fri, 27 Dec 2019 19:05:54 +0100 Subject: [PATCH 59/79] release the lockfile before exiting when using --rss-sections option. fixes #47 --- feed2toot/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/feed2toot/main.py b/feed2toot/main.py index 8e7b52a..641a14a 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -98,9 +98,11 @@ class Main: if clioptions.rsssections: if entries: print('The following sections are available in this RSS feed: {}'.format([j for j in entries[0]])) - sys.exit(0) else: - sys.exit('Could not parse the section of the rss feed') + print('Could not parse the section of the rss feed') + # release the lock file + lockfile.release() + sys.exit(0) # sort entries and check if they were not previously sent totweet = sort_entries(clioptions.all, cache, entries) for entry in totweet: From b4ab6a84b64393a5e397a5bf7920f1a79ca680d4 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 27 Dec 2019 22:39:45 +0100 Subject: [PATCH 60/79] update CHANGELOG and bump version --- CHANGELOG | 8 ++++++++ README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index caca498..7c945d4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,8 +1,16 @@ +## [0.13] - 2019-12-27 +### Added +- new cli option --ignore-ssl and parameter ignore_ssl to ignore ssl error in the rss feeds + +### Changed +- delete the lock file when using --rss-sections cli option + ## [0.12] - 2019-08-27 ### Changed - simplify code, mostly in the main function - fix a bug when title_pattern is used and *_pattern_case_sensitive is not defined + ## [0.11] - 2019-08-24 ### Added - MAJOR CHANGE: command line options --lockfile to define a lock file diff --git a/README.md b/README.md index 29169f5..25ea89e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* - # tar zxvf feed2toot-0.12.tar.gz + # tar zxvf feed2toot-0.13.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 84d188e..01634bd 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2015-2019, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.12' +version = '0.13' # The full version, including alpha/beta/rc tags. -release = '0.12' +release = '0.13' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index 749be21..6fd1624 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.12.tar.gz + $ tar zxvf feed2toot-0.13.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 7e36d46..790f2e2 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.12' +__version__ = '0.13' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index e4c9a6a..2cb1511 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.12', + version='0.13', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From 267d5cd49610f223bff57fe6df95ae6b2fc65bfd Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sun, 29 Dec 2019 16:13:34 +0100 Subject: [PATCH 61/79] document --limit CLI option. fixes #30 --- docs/source/use.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/use.rst b/docs/source/use.rst index 0066532..cb571d6 100644 --- a/docs/source/use.rst +++ b/docs/source/use.rst @@ -54,3 +54,11 @@ Using syslog Feed2toot is able to send its log to syslog. You can use it with the following command:: $ feed2toot --syslog=WARN -c /path/to/feed2toot.ini + +Limit number of rss entries published at each execution +======================================================= +If you want to limit the number of rss entries published at each execution, you can use the --limit CLI option. + + $ feed2toot --limit 5 -c /path/to/feed2toot.ini + +The number of posts to Mastodon will be at 5 posts top with this CLI option. From 3c0412a4ff67af26caa2cf851a65e31ff1fe4a5a Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Sun, 29 Dec 2019 17:08:39 +0100 Subject: [PATCH 62/79] remove useless imports in main --- feed2toot/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/feed2toot/main.py b/feed2toot/main.py index 641a14a..975124e 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -40,10 +40,8 @@ from feed2toot.message import build_message from feed2toot.message import send_message_dry_run from feed2toot.message import send_message from feed2toot.plugins import activate_plugins -from feed2toot.removeduplicates import RemoveDuplicates from feed2toot.rss import populate_rss from feed2toot.sortentries import sort_entries -from feed2toot.tootpost import TootPost class Main: '''Main class of Feed2toot''' From 9249e0015226ab3782b797e0f76f0144c9b34dcd Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 30 Dec 2019 18:57:02 +0100 Subject: [PATCH 63/79] interpret linebreak. fixes #42 --- feed2toot/message.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/feed2toot/message.py b/feed2toot/message.py index 8ccbad0..b7f4ea9 100644 --- a/feed2toot/message.py +++ b/feed2toot/message.py @@ -29,8 +29,10 @@ from feed2toot.tootpost import TootPost def build_message(entrytosend, tweetformat, rss): '''populate the rss dict with the new entry''' tweetwithnotag = tweetformat.format(**entrytosend) + # replace line breaks + tootwithlinebreaks = tweetwithnotag.replace('\\n', '\n') # remove duplicates from the final tweet - dedup = RemoveDuplicates(tweetwithnotag) + dedup = RemoveDuplicates(tootwithlinebreaks) # only append hashtags if they exist # remove last tags if tweet too long if 'hashtags' in rss: From 86ca0f14b6aa05fe3661adee8d54fc062fc1f605 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 30 Dec 2019 22:11:19 +0100 Subject: [PATCH 64/79] new conf parameter toot_max_len to truncate toot max length --- feed2toot/confparse.py | 2 ++ feed2toot/confparsers/rss/tootmaxlen.py | 37 +++++++++++++++++++++++++ feed2toot/main.py | 2 +- feed2toot/message.py | 9 ++++-- 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 feed2toot/confparsers/rss/tootmaxlen.py diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index dcf6ee1..af31575 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -40,6 +40,7 @@ from feed2toot.confparsers.rss.toot import parsetoot from feed2toot.confparsers.rss.uri import parseuri from feed2toot.confparsers.rss.urilist import parseurilist from feed2toot.confparsers.rss.addtags import parseaddtags +from feed2toot.confparsers.rss.tootmaxlen import parsetootmaxlen class ConfParse: '''ConfParse class''' @@ -67,6 +68,7 @@ class ConfParse: # the rss section ########################### self.tweetformat = parsetoot(config) + options['tootmaxlen'] = parsetootmaxlen(config) ################################################# # pattern and patter_case_sensitive format option ################################################# diff --git a/feed2toot/confparsers/rss/tootmaxlen.py b/feed2toot/confparsers/rss/tootmaxlen.py new file mode 100644 index 0000000..f975469 --- /dev/null +++ b/feed2toot/confparsers/rss/tootmaxlen.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see tootmaxlen: + return ''.join([finaltweet[0:-3], '...']) + else: + return finaltweet def send_message_dry_run(config, entrytosend, finaltweet): '''simulate sending message using dry run mode''' From b7430f3b10cc7087ae1519c0d185fc31e89f839f Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 30 Dec 2019 22:35:05 +0100 Subject: [PATCH 65/79] add no_tags_in_toot configuration parameter to remove hashtags from toot --- feed2toot/confparse.py | 4 ++- feed2toot/confparsers/hashtags/__init__.py | 15 +++++++++++ feed2toot/confparsers/hashtags/nohashtags.py | 26 ++++++++++++++++++++ feed2toot/main.py | 2 +- feed2toot/message.py | 18 ++++++++------ 5 files changed, 56 insertions(+), 9 deletions(-) create mode 100644 feed2toot/confparsers/hashtags/__init__.py create mode 100644 feed2toot/confparsers/hashtags/nohashtags.py diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index af31575..efc1dfb 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -30,6 +30,7 @@ import feedparser # feed2toot library imports from feed2toot.confparsers.cache import parsecache from feed2toot.confparsers.hashtaglist import parsehashtaglist +from feed2toot.confparsers.hashtags.nohashtags import parsenotagsintoot from feed2toot.confparsers.feedparser import parsefeedparser from feed2toot.confparsers.lock import parselock from feed2toot.confparsers.media import parsemedia @@ -99,9 +100,10 @@ class ConfParse: ########################### options['cachefile'], options['cache_limit'] = parsecache(self.clioptions.cachefile, config) ########################### - # the hashtag section + # the hashtaglist section ########################### options['hashtaglist'] = parsehashtaglist(self.clioptions.hashtaglist, config) + options['notagsintoot'] = parsenotagsintoot(config) ########################### # the media section ########################### diff --git a/feed2toot/confparsers/hashtags/__init__.py b/feed2toot/confparsers/hashtags/__init__.py new file mode 100644 index 0000000..d77b4df --- /dev/null +++ b/feed2toot/confparsers/hashtags/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2017-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see diff --git a/feed2toot/confparsers/hashtags/nohashtags.py b/feed2toot/confparsers/hashtags/nohashtags.py new file mode 100644 index 0000000..adcf448 --- /dev/null +++ b/feed2toot/confparsers/hashtags/nohashtags.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Copyright © 2015-2019 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see tootmaxlen: return ''.join([finaltweet[0:-3], '...']) else: From 976fc0322435af4324fea3c6d3a116a698736e0a Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 31 Dec 2019 12:10:50 +0100 Subject: [PATCH 66/79] fix truncate messages --- feed2toot/message.py | 1 + 1 file changed, 1 insertion(+) diff --git a/feed2toot/message.py b/feed2toot/message.py index da4f64d..6062d2e 100644 --- a/feed2toot/message.py +++ b/feed2toot/message.py @@ -48,6 +48,7 @@ def build_message(entrytosend, tweetformat, rss, tootmaxlen, notagsintoot): finaltweet = BeautifulSoup(finaltweet, 'html.parser').get_text() # truncate toot to user-defined value whatever the content is if len(finaltweet) > tootmaxlen: + finaltweet = finaltweet[0:tootmaxlen-1] return ''.join([finaltweet[0:-3], '...']) else: return finaltweet From 4ed27c721f1ac611828b7d51e645db26fc1cb9f9 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 31 Dec 2019 12:26:43 +0100 Subject: [PATCH 67/79] add explainations for no_tags_in_toot, toot_max_len and line break in toot --- docs/source/configure.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/source/configure.rst b/docs/source/configure.rst index a6f1170..e18a5eb 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -41,6 +41,7 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any uri=https://www.journalduhacker.net/rss uri_list=/etc/feed2toot//rsslist.txt toot={title} {link} + ; toot_max_len=125 title_pattern=Open Source title_pattern_case_sensitive=true no_uri_pattern_no_global_pattern=true @@ -48,6 +49,7 @@ In order to configure Feed2toot, you need to create a feed2toot.ini file (or any [hashtaglist] several_words_hashtags_list=/etc/feed2toot/hashtags.txt + ; no_tags_in_toot=false [feedparser] accept_bozo_exceptions=true @@ -81,7 +83,8 @@ For the [rss] section: - uri: the url of the rss feed to parse - uri_list: a path to a file with several adresses of rss feeds, one by line. Absolute path is mandatory. -- toot: format of the toot you want to post. It should use existing entries of the RSS fields like {title} or {link}. Launch it with this field empty to display all available entries. If you want to shorten the size of a field, you can use the syntax {summary:.100} to cut the field "summary" of the rss feed after the first 100 characters (starting from Feed2toot 0.10). +- toot: format of the toot you want to post. It should use existing entries of the RSS fields like {title} or {link}. Launch it with this field empty to display all available entries. If you want to shorten the size of a field, you can use the syntax {summary:.100} to cut the field "summary" of the rss feed after the first 100 characters (starting from version 0.10). To add new lines you can use \n (starting from version 0.14) +- toot_max_len: the max length of a toot can be defined here. If the toot size is longer, the toot is truncated and "..." added at the end. Defaults is 500 characters. - {one field of the rss feed}_pattern: takes a string representing a pattern to match for a specified field of each rss entry of the rss feed, like title_pattern or summary_pattern. - {one field of the rss feed}_pattern_case_sensitive: either the pattern matching for the specified field should be case sensitive or not. Default to true if not specified. - no_uri_pattern_no_global_pattern: don't apply global pattern (see above) when no pattern-by-uri is defined in the uri_list. Allows to get all entries of a rss in the uri_list because no pattern is defined so we match them all. Defaults to false, meaning the global patterns will be tried on every rss in the uri_list NOT HAVING specific patterns and so ONLY entries from the specific uri in the uri_list matching the global patterns will be considered. @@ -91,6 +94,7 @@ For the [rss] section: For the [hashtaglist] section: - several_words_hashtags_list: a path to the file containing hashtags in two or more words. Absolute path is mandatory. By default Feed2toot adds a # before every words of a hashtag. See documentation below for an example of this file. +- no_tags_in_toot: stop hash tags to be added at the toot. Defaults to false. for the [feedparser] section: From 4602d4d9dc4c33ecee8e7b0e571bb216daf54d72 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 31 Dec 2019 12:35:00 +0100 Subject: [PATCH 68/79] bump version and update CHANGELOG --- CHANGELOG | 6 ++++++ README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 7c945d4..6151a87 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +## [0.14] - 2019-12-31 +### Added +- support for line breaks in toots +- new configuration parameter toot_max_len to define the maximum length of a toot +- new configuration parameter no_tags_in_toot to stop hash tags to be added in toots + ## [0.13] - 2019-12-27 ### Added - new cli option --ignore-ssl and parameter ignore_ssl to ignore ssl error in the rss feeds diff --git a/README.md b/README.md index 25ea89e..0eb8e1a 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* - # tar zxvf feed2toot-0.13.tar.gz + # tar zxvf feed2toot-0.14.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 01634bd..2d1153a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2015-2019, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.13' +version = '0.14' # The full version, including alpha/beta/rc tags. -release = '0.13' +release = '0.14' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index 6fd1624..7716a2a 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.13.tar.gz + $ tar zxvf feed2toot-0.14.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 790f2e2..ac397ff 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.13' +__version__ = '0.14' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index 2cb1511..0e45678 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.13', + version='0.14', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From cfd3f1716218a2897509d9b4068e224a30ec0ec2 Mon Sep 17 00:00:00 2001 From: Procyonid Date: Wed, 24 Jun 2020 17:33:03 +0000 Subject: [PATCH 69/79] Fixed minor writing errors. --- docs/source/use.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/use.rst b/docs/source/use.rst index cb571d6..80e4b05 100644 --- a/docs/source/use.rst +++ b/docs/source/use.rst @@ -6,7 +6,7 @@ After the configuration of Feed2toot, just launch the following command:: Run Feed2toot on a regular basis ================================= -Feed2toot should be launche on a regular basis in order to efficiently send your new RSS entries to Mastodon. It is quite easy to achieve with adding a line to your user crontab, as described below:: +Feed2toot should be launched on a regular basis in order to efficiently send your new RSS entries to Mastodon. It is quite easy to achieve by adding a line to your user crontab, as described below:: @hourly feed2toot -c /path/to/feed2toot.ini From 45bb623f86c3e0c363fb472ef29ae5f7929478f2 Mon Sep 17 00:00:00 2001 From: Christoph Miksche Date: Sun, 29 Nov 2020 16:47:14 +0000 Subject: [PATCH 70/79] fix issue when only uri_list and no uri is configured Fixes #58 --- feed2toot/confparse.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index efc1dfb..3ad42b7 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -94,7 +94,11 @@ class ConfParse: ############ # uri option ############ - options['rss_uri'], feed, feedname, options['nopatternurinoglobalpattern'] = parseuri(config, self.clioptions.rss_uri, feeds, ignore_ssl) + if config.has_option('rss', 'uri') or self.clioptions.rss_uri: + options['rss_uri'], feed, feedname, options['nopatternurinoglobalpattern'] = parseuri(config, self.clioptions.rss_uri, feeds, ignore_ssl) + else: + if config.has_option('rss', 'no_uri_pattern_no_global_pattern'): + options['nopatternurinoglobalpattern'] = config.getboolean('rss', 'no_uri_pattern_no_global_pattern') ########################### # the cache section ########################### From 32fd92e9e25f24dfbaf9acdbc81a2b0bf96e2a48 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 7 Dec 2020 20:13:39 +0100 Subject: [PATCH 71/79] bump to 0.15 --- README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0eb8e1a..f0b1ce6 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* - # tar zxvf feed2toot-0.14.tar.gz + # tar zxvf feed2toot-0.15.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 2d1153a..d1d8311 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -54,9 +54,9 @@ copyright = '2015-2019, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.14' +version = '0.15' # The full version, including alpha/beta/rc tags. -release = '0.14' +release = '0.15' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index 7716a2a..2e75782 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.14.tar.gz + $ tar zxvf feed2toot-0.15.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index ac397ff..5a1f248 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.14' +__version__ = '0.15' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index 0e45678..91b8c4c 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.14', + version='0.15', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From 22cf7b4208ed4d2dc023693afdf6d63ebe307233 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 7 Dec 2020 20:33:30 +0100 Subject: [PATCH 72/79] update author email and copyright --- AUTHORS | 2 +- README.md | 2 +- docs/source/authors.rst | 2 +- docs/source/conf.py | 8 ++++---- feed2toot.py | 2 +- feed2toot/__init__.py | 2 +- feed2toot/addtags.py | 2 +- feed2toot/cliparse.py | 2 +- feed2toot/confparse.py | 2 +- feed2toot/confparsers/__init__.py | 2 +- feed2toot/confparsers/cache.py | 2 +- feed2toot/confparsers/feedparser.py | 2 +- feed2toot/confparsers/hashtaglist.py | 2 +- feed2toot/confparsers/hashtags/__init__.py | 2 +- feed2toot/confparsers/hashtags/nohashtags.py | 2 +- feed2toot/confparsers/lock.py | 2 +- feed2toot/confparsers/media.py | 2 +- feed2toot/confparsers/plugins.py | 2 +- feed2toot/confparsers/rss/__init__.py | 2 +- feed2toot/confparsers/rss/addtags.py | 2 +- feed2toot/confparsers/rss/ignoressl.py | 2 +- feed2toot/confparsers/rss/pattern.py | 2 +- feed2toot/confparsers/rss/toot.py | 2 +- feed2toot/confparsers/rss/tootmaxlen.py | 2 +- feed2toot/confparsers/rss/uri.py | 2 +- feed2toot/confparsers/rss/urilist.py | 2 +- feed2toot/feedcache.py | 2 +- feed2toot/filterentry.py | 2 +- feed2toot/hashtags.py | 2 +- feed2toot/lock.py | 2 +- feed2toot/main.py | 2 +- feed2toot/message.py | 2 +- feed2toot/removeduplicates.py | 2 +- feed2toot/rss.py | 2 +- feed2toot/sortentries.py | 2 +- feed2toot/tootpost.py | 2 +- scripts/feed2toot | 2 +- scripts/register_feed2toot_app | 3 +-- setup.py | 4 ++-- 39 files changed, 43 insertions(+), 44 deletions(-) diff --git a/AUTHORS b/AUTHORS index bddf2e0..ec96c0b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,5 +1,5 @@ Antoine Beaupré -Carl Chenet +Carl Chenet Alexis Metaireau The Dod Matthias Henze diff --git a/README.md b/README.md index f0b1ce6..86ea752 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Alternatively you can donate cryptocurrencies: ### Authors -* Carl Chenet +* Carl Chenet * Antoine Beaupré * First developed by Todd Eddy diff --git a/docs/source/authors.rst b/docs/source/authors.rst index 68bc3db..13b4749 100644 --- a/docs/source/authors.rst +++ b/docs/source/authors.rst @@ -1,4 +1,4 @@ Authors ======= -Carl Chenet +Carl Chenet diff --git a/docs/source/conf.py b/docs/source/conf.py index d1d8311..1edf696 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -47,7 +47,7 @@ master_doc = 'index' # General information about the project. project = 'feed2toot' -copyright = '2015-2019, Carl Chenet ' +copyright = '2015-2020, Carl Chenet ' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -200,7 +200,7 @@ latex_elements = { # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'feed2toot.tex', 'feed2toot Documentation', - 'Carl Chenet \\textless{}chaica@ohmytux.com.org\\textgreater{}', 'manual'), + 'Carl Chenet \\textless{}carl.chenet@ohmytux.com.org\\textgreater{}', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -230,7 +230,7 @@ latex_documents = [ # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'feed2toot', 'feed2toot Documentation', - ['Carl Chenet '], 1) + ['Carl Chenet '], 1) ] # If true, show URL addresses after external links. @@ -244,7 +244,7 @@ man_pages = [ # dir menu entry, description, category) texinfo_documents = [ ('index', 'feed2toot', 'feed2toot Documentation', - 'Carl Chenet ', 'feed2toot', 'One line description of project.', + 'Carl Chenet ', 'feed2toot', 'One line description of project.', 'Miscellaneous'), ] diff --git a/feed2toot.py b/feed2toot.py index 1767e41..416e53b 100755 --- a/feed2toot.py +++ b/feed2toot.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/__init__.py b/feed2toot/__init__.py index d19bf9f..dad5d70 100644 --- a/feed2toot/__init__.py +++ b/feed2toot/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2017 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/addtags.py b/feed2toot/addtags.py index 936895e..2cd8668 100644 --- a/feed2toot/addtags.py +++ b/feed2toot/addtags.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 5a1f248..cdeb811 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index 3ad42b7..baa67c3 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/__init__.py b/feed2toot/confparsers/__init__.py index d77b4df..dad5d70 100644 --- a/feed2toot/confparsers/__init__.py +++ b/feed2toot/confparsers/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2017-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/cache.py b/feed2toot/confparsers/cache.py index c6a7d52..2ca2515 100644 --- a/feed2toot/confparsers/cache.py +++ b/feed2toot/confparsers/cache.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/feedparser.py b/feed2toot/confparsers/feedparser.py index af2fddc..c7467ba 100644 --- a/feed2toot/confparsers/feedparser.py +++ b/feed2toot/confparsers/feedparser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtaglist.py b/feed2toot/confparsers/hashtaglist.py index 63e5854..d965cfa 100644 --- a/feed2toot/confparsers/hashtaglist.py +++ b/feed2toot/confparsers/hashtaglist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtags/__init__.py b/feed2toot/confparsers/hashtags/__init__.py index d77b4df..dad5d70 100644 --- a/feed2toot/confparsers/hashtags/__init__.py +++ b/feed2toot/confparsers/hashtags/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2017-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtags/nohashtags.py b/feed2toot/confparsers/hashtags/nohashtags.py index adcf448..42a985b 100644 --- a/feed2toot/confparsers/hashtags/nohashtags.py +++ b/feed2toot/confparsers/hashtags/nohashtags.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/lock.py b/feed2toot/confparsers/lock.py index 9588d9c..c3ed485 100644 --- a/feed2toot/confparsers/lock.py +++ b/feed2toot/confparsers/lock.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/media.py b/feed2toot/confparsers/media.py index a150768..47e2708 100644 --- a/feed2toot/confparsers/media.py +++ b/feed2toot/confparsers/media.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/plugins.py b/feed2toot/confparsers/plugins.py index d811f4a..05f3b80 100644 --- a/feed2toot/confparsers/plugins.py +++ b/feed2toot/confparsers/plugins.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/__init__.py b/feed2toot/confparsers/rss/__init__.py index d77b4df..dad5d70 100644 --- a/feed2toot/confparsers/rss/__init__.py +++ b/feed2toot/confparsers/rss/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2017-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/addtags.py b/feed2toot/confparsers/rss/addtags.py index 9a2f8a7..4fc611d 100644 --- a/feed2toot/confparsers/rss/addtags.py +++ b/feed2toot/confparsers/rss/addtags.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/ignoressl.py b/feed2toot/confparsers/rss/ignoressl.py index 269b19c..da67dcb 100644 --- a/feed2toot/confparsers/rss/ignoressl.py +++ b/feed2toot/confparsers/rss/ignoressl.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/pattern.py b/feed2toot/confparsers/rss/pattern.py index 183917b..cbe635f 100644 --- a/feed2toot/confparsers/rss/pattern.py +++ b/feed2toot/confparsers/rss/pattern.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/toot.py b/feed2toot/confparsers/rss/toot.py index 2fcf0be..5f8b96c 100644 --- a/feed2toot/confparsers/rss/toot.py +++ b/feed2toot/confparsers/rss/toot.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/tootmaxlen.py b/feed2toot/confparsers/rss/tootmaxlen.py index f975469..88af10c 100644 --- a/feed2toot/confparsers/rss/tootmaxlen.py +++ b/feed2toot/confparsers/rss/tootmaxlen.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/uri.py b/feed2toot/confparsers/rss/uri.py index 22e16ec..7e0a1b2 100644 --- a/feed2toot/confparsers/rss/uri.py +++ b/feed2toot/confparsers/rss/uri.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/urilist.py b/feed2toot/confparsers/rss/urilist.py index 5cc489d..9027498 100644 --- a/feed2toot/confparsers/rss/urilist.py +++ b/feed2toot/confparsers/rss/urilist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/feedcache.py b/feed2toot/feedcache.py index 1f457df..8b85edd 100644 --- a/feed2toot/feedcache.py +++ b/feed2toot/feedcache.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/filterentry.py b/feed2toot/filterentry.py index 3372bdc..44608d5 100644 --- a/feed2toot/filterentry.py +++ b/feed2toot/filterentry.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/hashtags.py b/feed2toot/hashtags.py index 88405b0..893ba8c 100644 --- a/feed2toot/hashtags.py +++ b/feed2toot/hashtags.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/lock.py b/feed2toot/lock.py index 90cbfd4..e58ff3e 100644 --- a/feed2toot/lock.py +++ b/feed2toot/lock.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/main.py b/feed2toot/main.py index 3ac48ce..da76c73 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/message.py b/feed2toot/message.py index 6062d2e..a71eb20 100644 --- a/feed2toot/message.py +++ b/feed2toot/message.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/removeduplicates.py b/feed2toot/removeduplicates.py index 3c609d3..f7f76c2 100644 --- a/feed2toot/removeduplicates.py +++ b/feed2toot/removeduplicates.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/rss.py b/feed2toot/rss.py index 3edbbd7..27e63f4 100644 --- a/feed2toot/rss.py +++ b/feed2toot/rss.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/sortentries.py b/feed2toot/sortentries.py index 09d02ba..d88920b 100644 --- a/feed2toot/sortentries.py +++ b/feed2toot/sortentries.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/tootpost.py b/feed2toot/tootpost.py index a6bd8cb..43eae3f 100644 --- a/feed2toot/tootpost.py +++ b/feed2toot/tootpost.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/scripts/feed2toot b/scripts/feed2toot index 8ba5242..0391cdb 100755 --- a/scripts/feed2toot +++ b/scripts/feed2toot @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/scripts/register_feed2toot_app b/scripts/register_feed2toot_app index 4b0eb0d..6a5446a 100755 --- a/scripts/register_feed2toot_app +++ b/scripts/register_feed2toot_app @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -#!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2019 Carl Chenet +# Copyright © 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/setup.py b/setup.py index 91b8c4c..c1fa29b 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Copyright 2015-2019 Carl Chenet +# Copyright 2015-2020 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -36,7 +36,7 @@ setup( description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', author = 'Carl Chenet', - author_email = 'chaica@ohmytux.com', + author_email = 'carl.chenet@ohmytux.com', url = 'https://gitlab.com/chaica/feed2toot', classifiers=CLASSIFIERS, download_url='https://gitlab.com/chaica/feed2toot', From 38f6ddb355e43838b67b9bfcdf58f68d6880f07b Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Mon, 7 Dec 2020 20:36:43 +0100 Subject: [PATCH 73/79] update changelog --- CHANGELOG | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 6151a87..ce71f4a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +## [0.15] - 2020-12-07 +### Changed +- maintenance version +- fix bug while using uri_list parameter alone following feedparser upgrade +- update copyrigth +- update author email + ## [0.14] - 2019-12-31 ### Added - support for line breaks in toots From 72100f4cac7811437f3230bcb0a487b75d8822f0 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Tue, 8 Dec 2020 19:35:56 +0100 Subject: [PATCH 74/79] register_feed2toot_app: --name, --user-credentials-file and client-credentials-file CLI options and documentation --- docs/source/conf.py | 1 + docs/source/configure.rst | 41 +++++++++++++++------------- docs/source/use.rst | 49 ++++++++++++++++++++++++++++------ scripts/register_feed2toot_app | 40 ++++++++++++++++++++++----- 4 files changed, 98 insertions(+), 33 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1edf696..7894211 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -31,6 +31,7 @@ import os # ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.autosectionlabel' ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/source/configure.rst b/docs/source/configure.rst index e18a5eb..a16fd82 100644 --- a/docs/source/configure.rst +++ b/docs/source/configure.rst @@ -1,23 +1,34 @@ Configure Feed2toot =================== +Create credentials for Mastodon +------------------------------- As a prerequisite to use Feed2toot, you need to authorize a Mastodon app for your account. Just use the script register_feed2toot_app to register the feed2toot app for your account.:: $ ./register_feed2toot_app - - This app generates Mastodon app credentials needed by Feed2toot. - feed2toot_clientcred.txt and feed2toot_usercred.txt will be written in the current dir /home/chaica/progra/python/feed2toot. - One connection is initiated to create the app. + + This script generates the Mastodon application credentials for Feed2toot. + feed2toot_clientcred.txt and feed2toot_usercred.txt will be written + in the current directory: /home/me/feed2toot. + WARNING: previous files with the same names will be overwritten. + + A connection is also initiated to create the application. Your password is *not* stored. + + Mastodon instance URL (defaults to https://mastodon.social): https://framapiaf.org + Mastodon login: toto@titi.com + Mastodon password: + + The app feed2toot was added to your Preferences=>Accounts=>Authorized apps page. + The file feed2toot_clientcred.txt and feed2toot_usercred.txt were created in the current directory. - Mastodon instance url (defaults to https://mastodon.social): - Mastodon login:chaica@ohmytux.com - Mastodon password: - The feed2toot app was added to your preferences=>authorized apps page +As described above, two files were created. See the :ref:`Use register_feed2toot_app` section for more options for register_feed2toot_app. -As described above, two files were created. You'll need them in the feed2toot configuration. +Create Feed2toot configuration +------------------------------ +After using register_feed2toot_app, you'll need the credentials in the feed2toot configuration. In order to configure Feed2toot, you need to create a feed2toot.ini file (or any name you prefer, finishing with the extension .ini) with the following parameters:: @@ -88,7 +99,7 @@ For the [rss] section: - {one field of the rss feed}_pattern: takes a string representing a pattern to match for a specified field of each rss entry of the rss feed, like title_pattern or summary_pattern. - {one field of the rss feed}_pattern_case_sensitive: either the pattern matching for the specified field should be case sensitive or not. Default to true if not specified. - no_uri_pattern_no_global_pattern: don't apply global pattern (see above) when no pattern-by-uri is defined in the uri_list. Allows to get all entries of a rss in the uri_list because no pattern is defined so we match them all. Defaults to false, meaning the global patterns will be tried on every rss in the uri_list NOT HAVING specific patterns and so ONLY entries from the specific uri in the uri_list matching the global patterns will be considered. - addtags: add the tags from the rss feed at the end of the toot. Defaults to true. +- addtags: add the tags from the rss feed at the end of the toot. Defaults to true. - ignore_ssl: when the uri or uri_list contains an https url with an invalid certificate (e.g an expired one), feed2toot will be unable to get rss content. This option allows to bypass the ssl security to catch the rss content. Defaults to false. For the [hashtaglist] section: @@ -143,8 +154,8 @@ Match specific patterns of rss feeds in the uri_list files ---------------------------------------------------------- You can use specific pattern matching for uri in the uri_list file to filter some of the rss entries of a rss feed. Lets modify the previous file:: -https://www.journalduhacker.net/rss|title|hacker,psql -https://carlchenet.com/feed|title|gitlab + https://www.journalduhacker.net/rss|title|hacker,psql + https://carlchenet.com/feed|title|gitlab Each line of this file starts with an uri, followed by a pipe (|), followed by the name of the available section to parse (see below), again followed by a pipe (|), followed by patterns, each pattern being separated from the other one by a semi-colon (,). @@ -166,9 +177,3 @@ In you rsslist.txt, just don't give anything else than the needed feed url to ge The last line of the file above only has the url of a rss feed. All entries from this feed will be tooted. -How to display available sections of the rss feed -================================================= -Feed2toot offers the **--rss-sections** command line option to display the available section of the rss feed and exits:: - - $ feed2toot --rss-sections -c feed2toot.ini - The following sections are available in this RSS feed: ['title', 'comments', 'authors', 'link', 'author', 'summary', 'links', 'tags', id', 'author_detail', 'published']. diff --git a/docs/source/use.rst b/docs/source/use.rst index 80e4b05..d757345 100644 --- a/docs/source/use.rst +++ b/docs/source/use.rst @@ -5,7 +5,7 @@ After the configuration of Feed2toot, just launch the following command:: $ feed2toot -c /path/to/feed2toot.ini Run Feed2toot on a regular basis -================================= +--------------------------------- Feed2toot should be launched on a regular basis in order to efficiently send your new RSS entries to Mastodon. It is quite easy to achieve by adding a line to your user crontab, as described below:: @hourly feed2toot -c /path/to/feed2toot.ini @@ -15,19 +15,19 @@ will execute feed2toot every hour. Or without the syntactic sugar in the global 0 * * * * johndoe feed2toot -c /path/to/feed2toot.ini Test option -=========== +----------- In order to know what's going to be sent to Mastodon without actually doing it, use the **--dry-run** option:: $ feed2toot --dry-run -c /path/to/feed2toot.ini Debug option -============ +------------ In order to increase the verbosity of what's Feed2toot is doing, use the **--debug** option followed by the level of verbosity see [the the available different levels](https://docs.python.org/3/library/logging.html):: $ feed2toot --debug -c /path/to/feed2toot.ini -Populate the cache file without posting tweets -============================================== +Populate the cache file without posting toots +--------------------------------------------- Starting from 0.8, Feed2toot offers the **--populate-cache** command line option to populate the cache file without posting to Mastodon:: $ feed2toot --populate-cache -c feed2toot.ini @@ -43,22 +43,55 @@ Starting from 0.8, Feed2toot offers the **--populate-cache** command line option populating RSS entry https://www.journalduhacker.net/s/lqswmz How to display available sections of the rss feed -================================================= +------------------------------------------------- Starting from 0.8, Feed2toot offers the **--rss-sections** command line option to display the available section of the rss feed and exits:: $ feed2toot --rss-sections -c feed2toot.ini The following sections are available in this RSS feed: ['title', 'comments', 'authors', 'link', 'author', 'summary', 'links', 'tags', id', 'author_detail', 'published']. Using syslog -============ +------------ Feed2toot is able to send its log to syslog. You can use it with the following command:: $ feed2toot --syslog=WARN -c /path/to/feed2toot.ini Limit number of rss entries published at each execution -======================================================= +------------------------------------------------------- If you want to limit the number of rss entries published at each execution, you can use the --limit CLI option. $ feed2toot --limit 5 -c /path/to/feed2toot.ini The number of posts to Mastodon will be at 5 posts top with this CLI option. + +Use register_feed2toot_app +========================== +You need a Mastodon app associated to a user on the Mastodon instance. The script register_feed2toot_app will create an app for Feed2toot and upload it on the specified Mastodon instance. + +Primary usage :: + + $ register_feed2toot_app + +Possible CLI options: + +- use the **--client-credentials-file** option to change the filename in which the client credentials are stored (defaults to feed2toot_clientcred.txt) +- use the **--user-credentials-file** option to change the filename in which the user credentials are stored (defaults to feed2toot_usercred.txt) +- use the **--name** to change the Mastodon app name (defaults to feed2toot) + +Example with full options and full output:: + + $ ./register_feed2toot_app --user-credentials-file f2tusercreds.txt --client-credentials-file f2tclientcreds.txt --name f2t + + This script generates the Mastodon application credentials for Feed2toot. + f2tclientcreds.txt and f2tusercreds.txt will be written + in the current directory: /home/me/feed2toot/scripts. + WARNING: previous files with the same names will be overwritten. + + A connection is also initiated to create the application. + Your password is *not* stored. + + Mastodon instance URL (defaults to https://mastodon.social): https://framapiaf.org + Mastodon login: toto@titi.com + Mastodon password: + + The app f2t was added to your preferences=>authorized apps page. + The file f2tclientcreds.txt and f2tusercreds.txt were created in the current directory. diff --git a/scripts/register_feed2toot_app b/scripts/register_feed2toot_app index 6a5446a..5096c08 100755 --- a/scripts/register_feed2toot_app +++ b/scripts/register_feed2toot_app @@ -14,13 +14,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see +from argparse import ArgumentParser from getpass import getpass from os import getcwd +from os import linesep +from os import sep from mastodon import Mastodon from mastodon.Mastodon import MastodonIllegalArgumentError import sys -print('\nThis script generates the Mastodon application credentials for Feed2toot.\nfeed2toot_clientcred.txt and feed2toot_usercred.txt will be written\nin the current directory: {cwd}.\nA connection is initiated to create the application.\nYour password is *not* stored.\n'.format(cwd=getcwd())) +__version__ = '0.2' + +epilog = 'For more information: https://feed2toot.readthedocs.io' +description = 'Create a Mastodon app for Feed2toot' +parser = ArgumentParser(prog='register_feed2toot_app', + description=description, + epilog=epilog) +parser.add_argument('--version', action='version', version=__version__) +parser.add_argument('--client-credentials-file', dest='clientcredfile', help='the name of the client credentials for the Mastodon app', default='feed2toot_clientcred.txt') +parser.add_argument('--user-credentials-file', dest='usercredfile', help='the name of the user credentials for the Mastodon app', default='feed2toot_usercred.txt') +parser.add_argument('--name', help='the name of the Mastodon app', default='feed2toot') +opts = parser.parse_args() + +clientcredfile=opts.clientcredfile +usercredfile=opts.usercredfile + +headline = '{linesep}This script generates the Mastodon application credentials for Feed2toot.{linesep}{clientcredfile} and {usercredfile} will be written{linesep}in the current directory: {cwd}.{linesep}WARNING: previous files with the same names will be overwritten.{linesep}{linesep}A connection is also initiated to create the application.{linesep}Your password is *not* stored.{linesep}'.format(linesep=linesep, clientcredfile=clientcredfile, usercredfile=usercredfile, cwd=getcwd()) +print(headline) + # get the instance instance = input('Mastodon instance URL (defaults to https://mastodon.social): ') @@ -45,20 +66,25 @@ while not userok: # get the password password = getpass(prompt='Mastodon password: ') Mastodon.create_app( - 'feed2toot', + opts.name, api_base_url=instance, - to_file = '{cwd}/feed2toot_clientcred.txt'.format(cwd=getcwd()) + to_file = '{cwd}{sep}{clientcredfile}'.format(cwd=getcwd(), sep=sep, clientcredfile=clientcredfile) ) -mastodon = Mastodon(client_id = '{cwd}/feed2toot_clientcred.txt'.format(cwd=getcwd()), +mastodon = Mastodon(client_id = '{cwd}{sep}{clientcredfile}'.format(cwd=getcwd(), sep=sep, clientcredfile=clientcredfile), api_base_url=instance) try: mastodon.log_in( user, password, - to_file = '{cwd}/feed2toot_usercred.txt'.format(cwd=getcwd()) + to_file = '{cwd}{sep}{usercredfile}'.format(cwd=getcwd(), sep=sep, usercredfile=usercredfile) ) except MastodonIllegalArgumentError as err: print(err) - sys.exit('\nI guess you entered a bad login or password.\n') -print('feed2toot was added to your preferences=>authorized apps page.') + sys.exit('{linesep}I guess you entered a bad login or password.{linesep}'.format(linesep=linesep)) + +summary = '{linesep}The app {appname} was added to your Preferences=>Accounts=>Authorized apps page.{linesep}The file {clientcredfile} and {usercredfile} were created in the current directory.{linesep}'.format(appname=opts.name, + linesep=linesep, + clientcredfile=clientcredfile, + usercredfile=usercredfile) +print(summary) sys.exit(0) From a8958f914b5b2fb7faaf2a8633d94ae3d39221ef Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 9 Dec 2020 01:29:25 +0100 Subject: [PATCH 75/79] bump version --- README.md | 2 +- docs/source/conf.py | 4 ++-- docs/source/install.rst | 2 +- feed2toot/cliparse.py | 2 +- setup.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 86ea752..e53e052 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* - # tar zxvf feed2toot-0.15.tar.gz + # tar zxvf feed2toot-0.16.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index 7894211..ddef383 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -55,9 +55,9 @@ copyright = '2015-2020, Carl Chenet ' # built documents. # # The short X.Y version. -version = '0.15' +version = '0.16' # The full version, including alpha/beta/rc tags. -release = '0.15' +release = '0.16' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index 2e75782..cd00e3a 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.15.tar.gz + $ tar zxvf feed2toot-0.16.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index cdeb811..663a1ba 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.15' +__version__ = '0.16' class CliParse: '''CliParse class''' diff --git a/setup.py b/setup.py index c1fa29b..7fdb44b 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.15', + version='0.16', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From 02dc5f7887fbf2171fbd69f095a0cd9c81417b96 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Wed, 9 Dec 2020 01:33:49 +0100 Subject: [PATCH 76/79] add 0.16 description --- CHANGELOG | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index ce71f4a..39af57a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +## [0.16] - 2020-12-09 +### Added +- scripts/register_feed2toot_app: --client-credentials-file option to change the filename in which the client credentials are stored +- scripts/register_feed2toot_app: --user-credentials-file option to change the filename in which the client credentials are stored +- scripts/register_feed2toot_app: --name to change the Mastodon app name + ## [0.15] - 2020-12-07 ### Changed - maintenance version @@ -5,6 +11,7 @@ - update copyrigth - update author email + ## [0.14] - 2019-12-31 ### Added - support for line breaks in toots From 475bdae9259d6fc04577cac4486258b9f4e0b4c5 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 26 Mar 2021 18:08:39 +0100 Subject: [PATCH 77/79] add __init__ for scripts dir. fixes #62 --- scripts/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scripts/__init__.py diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..979c3de --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# vim:ts=4:sw=4:ft=python:fileencoding=utf-8 +# Copyright © 2015-2021 Carl Chenet +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see From 9ae33644e563edd4c05cbe41bd056574f695d1b9 Mon Sep 17 00:00:00 2001 From: Carl Chenet Date: Fri, 26 Mar 2021 18:18:46 +0100 Subject: [PATCH 78/79] bump version and update version --- CHANGELOG | 5 +++++ README.md | 2 +- docs/source/conf.py | 6 +++--- docs/source/install.rst | 2 +- feed2toot.py | 2 +- feed2toot/__init__.py | 2 +- feed2toot/addtags.py | 2 +- feed2toot/cliparse.py | 4 ++-- feed2toot/confparse.py | 2 +- feed2toot/confparsers/__init__.py | 2 +- feed2toot/confparsers/cache.py | 2 +- feed2toot/confparsers/feedparser.py | 2 +- feed2toot/confparsers/hashtaglist.py | 2 +- feed2toot/confparsers/hashtags/__init__.py | 2 +- feed2toot/confparsers/hashtags/nohashtags.py | 2 +- feed2toot/confparsers/lock.py | 2 +- feed2toot/confparsers/media.py | 2 +- feed2toot/confparsers/plugins.py | 2 +- feed2toot/confparsers/rss/__init__.py | 2 +- feed2toot/confparsers/rss/addtags.py | 2 +- feed2toot/confparsers/rss/ignoressl.py | 2 +- feed2toot/confparsers/rss/pattern.py | 2 +- feed2toot/confparsers/rss/toot.py | 2 +- feed2toot/confparsers/rss/tootmaxlen.py | 2 +- feed2toot/confparsers/rss/uri.py | 2 +- feed2toot/confparsers/rss/urilist.py | 2 +- feed2toot/feedcache.py | 2 +- feed2toot/filterentry.py | 2 +- feed2toot/hashtags.py | 2 +- feed2toot/lock.py | 2 +- feed2toot/main.py | 2 +- feed2toot/message.py | 2 +- feed2toot/removeduplicates.py | 2 +- feed2toot/rss.py | 2 +- feed2toot/sortentries.py | 2 +- feed2toot/tootpost.py | 2 +- scripts/feed2toot | 2 +- scripts/register_feed2toot_app | 2 +- setup.py | 4 ++-- 39 files changed, 47 insertions(+), 42 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 39af57a..55a14f7 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +## [0.17] - 2021-03-26 +### Changed +- fix install bug while installing from sources +- bump changelog + ## [0.16] - 2020-12-09 ### Added - scripts/register_feed2toot_app: --client-credentials-file option to change the filename in which the client credentials are stored diff --git a/README.md b/README.md index e53e052..61b4fcd 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Alternatively you can donate cryptocurrencies: [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* - # tar zxvf feed2toot-0.16.tar.gz + # tar zxvf feed2toot-0.17.tar.gz # cd feed2toot # python3 setup.py install # # or diff --git a/docs/source/conf.py b/docs/source/conf.py index ddef383..54b10dc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,16 +48,16 @@ master_doc = 'index' # General information about the project. project = 'feed2toot' -copyright = '2015-2020, Carl Chenet ' +copyright = '2015-2021, Carl Chenet ' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.16' +version = '0.17' # The full version, including alpha/beta/rc tags. -release = '0.16' +release = '0.17' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/install.rst b/docs/source/install.rst index cd00e3a..3e885d7 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -25,7 +25,7 @@ Alternatively, Setuptools may be installed to a user-local path:: * Untar the tarball and go to the source directory with the following commands:: - $ tar zxvf feed2toot-0.16.tar.gz + $ tar zxvf feed2toot-0.17.tar.gz $ cd feed2toot * Next, to install Feed2toot on your computer, type the following command with the root user:: diff --git a/feed2toot.py b/feed2toot.py index 416e53b..a53bf2a 100755 --- a/feed2toot.py +++ b/feed2toot.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/__init__.py b/feed2toot/__init__.py index dad5d70..979c3de 100644 --- a/feed2toot/__init__.py +++ b/feed2toot/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/addtags.py b/feed2toot/addtags.py index 2cd8668..cca0954 100644 --- a/feed2toot/addtags.py +++ b/feed2toot/addtags.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/cliparse.py b/feed2toot/cliparse.py index 663a1ba..e8d8739 100644 --- a/feed2toot/cliparse.py +++ b/feed2toot/cliparse.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -24,7 +24,7 @@ import os import os.path import sys -__version__ = '0.16' +__version__ = '0.17' class CliParse: '''CliParse class''' diff --git a/feed2toot/confparse.py b/feed2toot/confparse.py index baa67c3..54010a2 100644 --- a/feed2toot/confparse.py +++ b/feed2toot/confparse.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/__init__.py b/feed2toot/confparsers/__init__.py index dad5d70..979c3de 100644 --- a/feed2toot/confparsers/__init__.py +++ b/feed2toot/confparsers/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/cache.py b/feed2toot/confparsers/cache.py index 2ca2515..79c013d 100644 --- a/feed2toot/confparsers/cache.py +++ b/feed2toot/confparsers/cache.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/feedparser.py b/feed2toot/confparsers/feedparser.py index c7467ba..cf72889 100644 --- a/feed2toot/confparsers/feedparser.py +++ b/feed2toot/confparsers/feedparser.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtaglist.py b/feed2toot/confparsers/hashtaglist.py index d965cfa..ffdf710 100644 --- a/feed2toot/confparsers/hashtaglist.py +++ b/feed2toot/confparsers/hashtaglist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtags/__init__.py b/feed2toot/confparsers/hashtags/__init__.py index dad5d70..979c3de 100644 --- a/feed2toot/confparsers/hashtags/__init__.py +++ b/feed2toot/confparsers/hashtags/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/hashtags/nohashtags.py b/feed2toot/confparsers/hashtags/nohashtags.py index 42a985b..0cab8b6 100644 --- a/feed2toot/confparsers/hashtags/nohashtags.py +++ b/feed2toot/confparsers/hashtags/nohashtags.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/lock.py b/feed2toot/confparsers/lock.py index c3ed485..45e7718 100644 --- a/feed2toot/confparsers/lock.py +++ b/feed2toot/confparsers/lock.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/media.py b/feed2toot/confparsers/media.py index 47e2708..2cd4f1b 100644 --- a/feed2toot/confparsers/media.py +++ b/feed2toot/confparsers/media.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/plugins.py b/feed2toot/confparsers/plugins.py index 05f3b80..0911a6b 100644 --- a/feed2toot/confparsers/plugins.py +++ b/feed2toot/confparsers/plugins.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/__init__.py b/feed2toot/confparsers/rss/__init__.py index dad5d70..979c3de 100644 --- a/feed2toot/confparsers/rss/__init__.py +++ b/feed2toot/confparsers/rss/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/addtags.py b/feed2toot/confparsers/rss/addtags.py index 4fc611d..6808254 100644 --- a/feed2toot/confparsers/rss/addtags.py +++ b/feed2toot/confparsers/rss/addtags.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/ignoressl.py b/feed2toot/confparsers/rss/ignoressl.py index da67dcb..f1f0340 100644 --- a/feed2toot/confparsers/rss/ignoressl.py +++ b/feed2toot/confparsers/rss/ignoressl.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/pattern.py b/feed2toot/confparsers/rss/pattern.py index cbe635f..e6b858d 100644 --- a/feed2toot/confparsers/rss/pattern.py +++ b/feed2toot/confparsers/rss/pattern.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/toot.py b/feed2toot/confparsers/rss/toot.py index 5f8b96c..58c8f56 100644 --- a/feed2toot/confparsers/rss/toot.py +++ b/feed2toot/confparsers/rss/toot.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/tootmaxlen.py b/feed2toot/confparsers/rss/tootmaxlen.py index 88af10c..eb2a448 100644 --- a/feed2toot/confparsers/rss/tootmaxlen.py +++ b/feed2toot/confparsers/rss/tootmaxlen.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/uri.py b/feed2toot/confparsers/rss/uri.py index 7e0a1b2..e7f88b2 100644 --- a/feed2toot/confparsers/rss/uri.py +++ b/feed2toot/confparsers/rss/uri.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/confparsers/rss/urilist.py b/feed2toot/confparsers/rss/urilist.py index 9027498..11265a2 100644 --- a/feed2toot/confparsers/rss/urilist.py +++ b/feed2toot/confparsers/rss/urilist.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/feedcache.py b/feed2toot/feedcache.py index 8b85edd..9bfad65 100644 --- a/feed2toot/feedcache.py +++ b/feed2toot/feedcache.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/filterentry.py b/feed2toot/filterentry.py index 44608d5..37a7a76 100644 --- a/feed2toot/filterentry.py +++ b/feed2toot/filterentry.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/hashtags.py b/feed2toot/hashtags.py index 893ba8c..6354614 100644 --- a/feed2toot/hashtags.py +++ b/feed2toot/hashtags.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/lock.py b/feed2toot/lock.py index e58ff3e..e03c56c 100644 --- a/feed2toot/lock.py +++ b/feed2toot/lock.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/main.py b/feed2toot/main.py index da76c73..1c70e03 100644 --- a/feed2toot/main.py +++ b/feed2toot/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/message.py b/feed2toot/message.py index a71eb20..ef431d6 100644 --- a/feed2toot/message.py +++ b/feed2toot/message.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/removeduplicates.py b/feed2toot/removeduplicates.py index f7f76c2..8141d4f 100644 --- a/feed2toot/removeduplicates.py +++ b/feed2toot/removeduplicates.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/rss.py b/feed2toot/rss.py index 27e63f4..da271d9 100644 --- a/feed2toot/rss.py +++ b/feed2toot/rss.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/sortentries.py b/feed2toot/sortentries.py index d88920b..f5ac5ef 100644 --- a/feed2toot/sortentries.py +++ b/feed2toot/sortentries.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/feed2toot/tootpost.py b/feed2toot/tootpost.py index 43eae3f..9cf7d0d 100644 --- a/feed2toot/tootpost.py +++ b/feed2toot/tootpost.py @@ -1,5 +1,5 @@ # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/scripts/feed2toot b/scripts/feed2toot index 0391cdb..09cf4dd 100755 --- a/scripts/feed2toot +++ b/scripts/feed2toot @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/scripts/register_feed2toot_app b/scripts/register_feed2toot_app index 5096c08..22975e8 100755 --- a/scripts/register_feed2toot_app +++ b/scripts/register_feed2toot_app @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # vim:ts=4:sw=4:ft=python:fileencoding=utf-8 -# Copyright © 2015-2020 Carl Chenet +# Copyright © 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/setup.py b/setup.py index 7fdb44b..6ce1662 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -# Copyright 2015-2020 Carl Chenet +# Copyright 2015-2021 Carl Chenet # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -31,7 +31,7 @@ CLASSIFIERS = [ setup( name='feed2toot', - version='0.16', + version='0.17', license='GNU GPL v3', description='Parse rss feeds and send new posts to Mastodon', long_description='Parse rss feeds and send new posts to the Mastodon social network', From 9782f0321f96b0d853b1b3bcc8333672d6961367 Mon Sep 17 00:00:00 2001 From: MassiveBox Date: Sun, 23 Oct 2022 22:09:20 +0200 Subject: [PATCH 79/79] Add docker support - Allow setup script to get credentials from flags - Add Dockerfile and docker_entrypoint - Edit README for forked version --- .gitignore | 3 ++ Dockerfile | 13 +++++ README.md | 87 +++++++++++++--------------------- docker_entrypoint.sh | 12 +++++ scripts/register_feed2toot_app | 21 ++++++-- 5 files changed, 78 insertions(+), 58 deletions(-) create mode 100644 Dockerfile create mode 100755 docker_entrypoint.sh diff --git a/.gitignore b/.gitignore index f5189cf..90063a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ docs/build/ __pycache__ feedlist +data *.swp *.pyc *.db @@ -8,3 +9,5 @@ feedlist *.txt *.bck *.png +*.yml +.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c6ca6e7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.7-bullseye + +ADD . /app +WORKDIR /app + +RUN python3 setup.py install; \ + chmod +x docker_entrypoint.sh; \ + mkdir -p /data /root/.config; \ + rm -rf docke.yml + +WORKDIR /data + +CMD ["/app/docker_entrypoint.sh"] diff --git a/README.md b/README.md index 61b4fcd..dc85036 100644 --- a/README.md +++ b/README.md @@ -1,64 +1,45 @@ +# Feed2Toot Docker + +Feed2Toot Docker is a fork of [chaica/feed2toot](https://gitlab.com/chaica/feed2toot) that allows for simple deployment over Docker. In the future I'll try to add other useful features. + +### Usage + +With Docker Compose: +```yaml +version: '3' + +services: + feed2toot: + image: gitea.massivebox.net/massivebox/feed2toot-docker:latest + environment: + - USERNAME=sampleusername@sampleinstance.url + - PASSWORD=samplepassword + - INSTANCE=https://sampleinstance.url + restart: always + volumes: + - ./data:/data +``` +Insert your full username (including the part with your homeserver's address), password and instance URL where required. +You also need to create a working `feed2toot.ini` in the `./data` folder. See the [upstream docs](`https://feed2toot.readthedocs.io/en/latest/configure.html#create-feed2toot-configuration`) for the guide. + +After the first successful run, you can remove the entire `environment` block if you want, however, if you delete and re-create the container, you will need to put it back. + +### Support + +Don't ask the upstream developers for help, if something is broken it's more likely that it is my fault. +You can get support by opening an [issue](https://gitea.massivebox.net/massivebox/feed2toot-docker/issues), or by [contacting me](https://massivebox.net/contact.html), or in the [Matrix support room](https://matrix.to/#/#support:massivebox.net). + ### Feed2toot -Feed2toot automatically parses rss feeds, identifies new posts and posts them on the [Mastodon](https://mastodon.social) social network. -For the full documentation, [read it online](https://feed2toot.readthedocs.io/en/latest/). - -If you would like, you can [support the development of this project on Liberapay](https://liberapay.com/carlchenet/). -Alternatively you can donate cryptocurrencies: +The following links and addresses are the upstream developers'. I don't want donations for this project, but I encourage you to donate to them. +- [Liberapay](https://liberapay.com/carlchenet/) - BTC: 1AW12Zw93rx4NzWn5evcG7RNNEM2RSLmAC - XMR: 43GGv8KzVhxehv832FWPTF7FSVuWjuBarFd17QP163uxMaFyoqwmDf1aiRtS5jWgCiRsi73yqedNJJ6V1La2joznKHGAhDi -### Quick Install - -* Install Feed2toot from PyPI - - # pip3 install feed2toot - -* Install Feed2toot from sources - *(see the installation guide for full details) - [Installation Guide](http://feed2toot.readthedocs.io/en/latest/install.html)* - - - # tar zxvf feed2toot-0.17.tar.gz - # cd feed2toot - # python3 setup.py install - # # or - # python3 setup.py install --install-scripts=/usr/bin - -### Create the authorization for the Feed2toot app - -* Just launch the following command:: - - $ register_feed2toot_app - -### Use Feed2toot - -* Create or modify feed2toot.ini file in order to configure feed2toot: - - [mastodon] - instance_url=https://mastodon.social - user_credentials=feed2toot_usercred.txt - client_credentials=feed2toot_clientcred.txt - ; Default visibility is public, but you can override it: - ; toot_visibility=unlisted - - [cache] - cachefile=cache.db - - [rss] - uri=https://www.journalduhacker.net/rss - toot={title} {link} - - [hashtaglist] - several_words_hashtags_list=hashtags.txt - -* Launch Feed2toot - - $ feed2toot -c /path/to/feed2toot.ini - ### Authors +* MassiveBox * Carl Chenet * Antoine Beaupré * First developed by Todd Eddy diff --git a/docker_entrypoint.sh b/docker_entrypoint.sh new file mode 100755 index 0000000..7360431 --- /dev/null +++ b/docker_entrypoint.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +cd /data + +if [[ ! -f "feed2toot_clientcred.txt" || ! -f "feed2toot_usercred.txt" ]]; then + register_feed2toot_app --instance $INSTANCE --username $USERNAME --password $PASSWORD +fi + +while :; do + feed2toot -c ./feed2toot.ini + sleep 5m +done diff --git a/scripts/register_feed2toot_app b/scripts/register_feed2toot_app index 22975e8..7453ecc 100755 --- a/scripts/register_feed2toot_app +++ b/scripts/register_feed2toot_app @@ -34,6 +34,9 @@ parser.add_argument('--version', action='version', version=__version__) parser.add_argument('--client-credentials-file', dest='clientcredfile', help='the name of the client credentials for the Mastodon app', default='feed2toot_clientcred.txt') parser.add_argument('--user-credentials-file', dest='usercredfile', help='the name of the user credentials for the Mastodon app', default='feed2toot_usercred.txt') parser.add_argument('--name', help='the name of the Mastodon app', default='feed2toot') +parser.add_argument('--instance', help='the URL of the Mastodon instance') +parser.add_argument('--username', help='the username of your Mastodon account') +parser.add_argument('--password', help='the password of your Mastodon account') opts = parser.parse_args() clientcredfile=opts.clientcredfile @@ -44,7 +47,9 @@ print(headline) # get the instance -instance = input('Mastodon instance URL (defaults to https://mastodon.social): ') +instance = opts.instance +if not instance: + instance = input('Mastodon instance URL (defaults to https://mastodon.social): ') if not instance: instance = 'https://mastodon.social' elif not instance.startswith('http'): @@ -52,19 +57,25 @@ elif not instance.startswith('http'): # get the username userok = False +quit_on_error = True while not userok: - user = input('Mastodon login: ') + user = opts.username + if not user: + user = input('Mastodon login: ') + quit_on_error = False if not user: print('Your Mastodon username can not be empty.') - userok = False elif '@' not in user or '.' not in user: print('Your Mastodon username should be an email.') - userok = False else: userok = True + if not userok and quit_on_error: + exit() # get the password -password = getpass(prompt='Mastodon password: ') +password = opts.password +if not password: + password = getpass(prompt='Mastodon password: ') Mastodon.create_app( opts.name, api_base_url=instance,