From 191eb97d6f90965b915f9165431b5165adf67796 Mon Sep 17 00:00:00 2001 From: Magnus Hagander Date: Sat, 11 Jul 2020 18:25:40 +0200 Subject: [PATCH] Misc fixes to make pep8 happy Mostly whitespace fixes, but also some bare-exception fixes. Add a setup.cfg that specifies which pep8 settings we normally run with. --- hamnadmin/hamnadmin/mailqueue/admin.py | 8 +- .../management/commands/send_queued_mail.py | 1 + hamnadmin/hamnadmin/mailqueue/models.py | 1 + hamnadmin/hamnadmin/mailqueue/util.py | 8 +- hamnadmin/hamnadmin/register/admin.py | 6 +- hamnadmin/hamnadmin/register/feeds.py | 5 +- hamnadmin/hamnadmin/register/forms.py | 8 +- .../management/commands/aggregate_feeds.py | 133 +++++++++--------- .../management/commands/delete_old_logs.py | 3 +- .../register/management/commands/send_logs.py | 41 +++--- hamnadmin/hamnadmin/register/models.py | 23 +-- .../hamnadmin/register/templatetags/hamn.py | 3 +- hamnadmin/hamnadmin/register/views.py | 123 +++++++++------- hamnadmin/hamnadmin/settings.py | 35 +++-- hamnadmin/hamnadmin/util/aggregate.py | 9 +- hamnadmin/hamnadmin/util/html.py | 29 ++-- hamnadmin/hamnadmin/util/varnish.py | 2 + listsync.py | 9 +- posttotwitter.py | 22 ++- redirector/redirector.py | 26 ++-- setup.cfg | 5 + setuptwitter.py | 3 - synctwitter.py | 9 +- twitterclient.py | 7 +- 24 files changed, 293 insertions(+), 226 deletions(-) create mode 100644 setup.cfg diff --git a/hamnadmin/hamnadmin/mailqueue/admin.py b/hamnadmin/hamnadmin/mailqueue/admin.py index 023776c..6b89205 100644 --- a/hamnadmin/hamnadmin/mailqueue/admin.py +++ b/hamnadmin/hamnadmin/mailqueue/admin.py @@ -4,6 +4,7 @@ from email.parser import Parser from .models import QueuedMail + class QueuedMailAdmin(admin.ModelAdmin): model = QueuedMail readonly_fields = ('parsed_content', ) @@ -15,16 +16,19 @@ class QueuedMailAdmin(admin.ModelAdmin): parser = Parser() msg = parser.parsestr(obj.fullmsg) b = msg.get_payload(decode=True) - if b: return b + if b: + return b pl = msg.get_payload() for p in pl: b = p.get_payload(decode=True) - if b: return b + if b: + return b return "Could not find body" except Exception as e: return "Failed to get body: %s" % e parsed_content.short_description = 'Parsed mail' + admin.site.register(QueuedMail, QueuedMailAdmin) diff --git a/hamnadmin/hamnadmin/mailqueue/management/commands/send_queued_mail.py b/hamnadmin/hamnadmin/mailqueue/management/commands/send_queued_mail.py index 52373f8..301c769 100755 --- a/hamnadmin/hamnadmin/mailqueue/management/commands/send_queued_mail.py +++ b/hamnadmin/hamnadmin/mailqueue/management/commands/send_queued_mail.py @@ -12,6 +12,7 @@ import smtplib from hamnadmin.mailqueue.models import QueuedMail + class Command(BaseCommand): help = 'Send queued mail' diff --git a/hamnadmin/hamnadmin/mailqueue/models.py b/hamnadmin/hamnadmin/mailqueue/models.py index f45fd72..6b002e7 100644 --- a/hamnadmin/hamnadmin/mailqueue/models.py +++ b/hamnadmin/hamnadmin/mailqueue/models.py @@ -1,5 +1,6 @@ from django.db import models + class QueuedMail(models.Model): sender = models.EmailField(max_length=100, null=False, blank=False) receiver = models.EmailField(max_length=100, null=False, blank=False) diff --git a/hamnadmin/hamnadmin/mailqueue/util.py b/hamnadmin/hamnadmin/mailqueue/util.py index 05394a3..1e39dd8 100644 --- a/hamnadmin/hamnadmin/mailqueue/util.py +++ b/hamnadmin/hamnadmin/mailqueue/util.py @@ -7,11 +7,13 @@ from email import encoders from .models import QueuedMail + def _encoded_email_header(name, email): if name: return formataddr((str(Header(name, 'utf-8')), email)) return email + def send_simple_mail(sender, receiver, subject, msgtxt, attachments=None, bcc=None, sendername=None, receivername=None): # attachment format, each is a tuple of (name, mimetype,contents) # content should be *binary* and not base64 encoded, since we need to @@ -27,20 +29,20 @@ def send_simple_mail(sender, receiver, subject, msgtxt, attachments=None, bcc=No if attachments: for filename, contenttype, content in attachments: - main,sub = contenttype.split('/') - part = MIMENonMultipart(main,sub) + main, sub = contenttype.split('/') + part = MIMENonMultipart(main, sub) part.set_payload(content) part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename) encoders.encode_base64(part) msg.attach(part) - # Just write it to the queue, so it will be transactionally rolled back QueuedMail(sender=sender, receiver=receiver, fullmsg=msg.as_string()).save() # Any bcc is just entered as a separate email if bcc: QueuedMail(sender=sender, receiver=bcc, fullmsg=msg.as_string()).save() + def send_mail(sender, receiver, fullmsg): # Send an email, prepared as the full MIME encoded mail already QueuedMail(sender=sender, receiver=receiver, fullmsg=fullmsg).save() diff --git a/hamnadmin/hamnadmin/register/admin.py b/hamnadmin/hamnadmin/register/admin.py index 5591a1f..7ca4a1f 100644 --- a/hamnadmin/hamnadmin/register/admin.py +++ b/hamnadmin/hamnadmin/register/admin.py @@ -6,9 +6,10 @@ from hamnadmin.register.models import Blog, Team, Post, AggregatorLog class TeamAdmin(admin.ModelAdmin): list_display = ['name', 'manager', 'teamurl'] + class BlogAdmin(admin.ModelAdmin): list_display = ['user', 'approved', 'name', 'feedurl', 'authorfilter', ] - ordering = ['approved', 'name', ] #meh, multiple ordering not supported + ordering = ['approved', 'name', ] search_fields = ['user__username', 'name', 'feedurl'] def change_view(self, request, object_id, extra_context=None): @@ -18,13 +19,16 @@ class BlogAdmin(admin.ModelAdmin): } return super(BlogAdmin, self).change_view(request, object_id, extra_context=my_context) + class PostAdmin(admin.ModelAdmin): list_display = ['dat', 'title', 'hidden', 'feed'] search_fields = ['title', 'feed__name', 'feed__feedurl'] + class AggregatorLogAdmin(admin.ModelAdmin): list_display = ['ts', 'success', 'feed', 'info'] + admin.site.register(Team, TeamAdmin) admin.site.register(Blog, BlogAdmin) admin.site.register(Post, PostAdmin) diff --git a/hamnadmin/hamnadmin/register/feeds.py b/hamnadmin/hamnadmin/register/feeds.py index c8fdfeb..d50c8c6 100644 --- a/hamnadmin/hamnadmin/register/feeds.py +++ b/hamnadmin/hamnadmin/register/feeds.py @@ -4,6 +4,7 @@ from hamnadmin.util.html import TruncateAndClean from .models import Post + class PostFeed(Feed): title = 'Planet PostgreSQL' link = 'https://planet.postgresql.org' @@ -17,7 +18,7 @@ class PostFeed(Feed): def items(self, type): qs = Post.objects.filter(feed__approved=True, hidden=False).order_by('-dat') if type == "_short": - qs = qs.extra(select = {'short': 1}) + qs = qs.extra(select={'short': 1}) return qs[:30] def item_title(self, item): @@ -36,7 +37,7 @@ class PostFeed(Feed): if hasattr(item, 'short'): try: return TruncateAndClean(item.txt) - except Exception as e: + except Exception: return "Unable to clean HTML" else: return item.txt diff --git a/hamnadmin/hamnadmin/register/forms.py b/hamnadmin/hamnadmin/register/forms.py index b8e323f..4050fe9 100644 --- a/hamnadmin/hamnadmin/register/forms.py +++ b/hamnadmin/hamnadmin/register/forms.py @@ -9,6 +9,7 @@ from hamnadmin.util.aggregate import FeedFetcher, ParserGotRedirect import requests import requests_oauthlib + class BlogEditForm(forms.ModelForm): class Meta: model = Blog @@ -21,12 +22,12 @@ class BlogEditForm(forms.ModelForm): f.widget.attrs['class'] = 'form-control' if kwargs['instance'].approved: - self.fields['feedurl'].help_text="Note that changing the feed URL will disable the blog pending new moderation" - self.fields['authorfilter'].help_text="Note that changing the author filter will disable the blog pending new moderation" - + self.fields['feedurl'].help_text = "Note that changing the feed URL will disable the blog pending new moderation" + self.fields['authorfilter'].help_text = "Note that changing the author filter will disable the blog pending new moderation" def clean(self): tracemessages = [] + def _trace(msg): tracemessages.append(msg) @@ -79,6 +80,7 @@ class BlogEditForm(forms.ModelForm): raise forms.ValidationError("Timeout trying to validate account with twitter") return u + class ModerateRejectForm(forms.Form): message = forms.CharField(min_length=30, required=True, widget=forms.Textarea) modsonly = forms.BooleanField(required=False, label="Moderators only", help_text="Should message be sent only to moderators, and not to the submitter (for spam submissions mainly)") diff --git a/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py b/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py index 075c4aa..4bac98f 100644 --- a/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py +++ b/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py @@ -16,7 +16,6 @@ class BreakoutException(Exception): pass - class Command(BaseCommand): help = 'Aggregate one or more feeds' @@ -34,7 +33,7 @@ class Command(BaseCommand): self.verbose = options['verbosity'] > 1 self.debug = options['debug'] if self.debug: - self.verbose=True + self.verbose = True self.full = options['full'] if options['id']: @@ -51,7 +50,7 @@ class Command(BaseCommand): pr = pool.map_async(self._fetch_one_feed, fetchers) while not pr.ready(): gevent.sleep(1) - self.trace("Fetching feeds (%s/%s done), please wait..." % (num-pool.task_queue.unfinished_tasks, num)) + self.trace("Fetching feeds (%s/%s done), please wait..." % (num - pool.task_queue.unfinished_tasks, num)) total_entries = 0 # Fetching was async, but results processing will be sync. Don't want to deal with @@ -72,19 +71,21 @@ class Command(BaseCommand): # OK, update it! AggregatorLog(feed=feed, success=True, info="Feed returned redirect to https, updating registration").save() - send_simple_mail(settings.EMAIL_SENDER, - feed.user.email, - "Your blog at Planet PostgreSQL redirected", - "The blog aggregator at Planet PostgreSQL has picked up a redirect for your blog.\nOld URL: {0}\nNew URL: {1}\n\nThe database has been updated, and new entries will be fetched from the secure URL in the future.\n".format(feed.feedurl, results.url), - sendername="Planet PostgreSQL", - receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), - ) - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "Blog redirect detected on Planet PostgreSQL", - "The blog at {0} by {1}\nis returning a redirect to a https version of itself.\n\nThe database has automatically been updated, and will start fetching using https in the future,\n\n".format(feed.feedurl, feed.user), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + feed.user.email, + "Your blog at Planet PostgreSQL redirected", + "The blog aggregator at Planet PostgreSQL has picked up a redirect for your blog.\nOld URL: {0}\nNew URL: {1}\n\nThe database has been updated, and new entries will be fetched from the secure URL in the future.\n".format(feed.feedurl, results.url), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), + ) + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "Blog redirect detected on Planet PostgreSQL", + "The blog at {0} by {1}\nis returning a redirect to a https version of itself.\n\nThe database has automatically been updated, and will start fetching using https in the future,\n\n".format(feed.feedurl, feed.user), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", ) feed.feedurl = results.url feed.save() @@ -140,67 +141,73 @@ class Command(BaseCommand): entries, feed.feedurl, settings.MAX_SAFE_ENTRIES_PER_FETCH)) Post.objects.filter(id__in=ids).update(hidden=True) # Email a notification that they were picked up - send_simple_mail(settings.EMAIL_SENDER, - feed.user.email, - "Many posts found at your blog at Planet PostgreSQL", - "The blog aggregator at Planet PostgreSQL has just picked up the following\nposts from your blog at {0}:\n\n{1}\n\nSince this is a large number of posts, they have been fetched\nand marked as hidden, to avoid possible duplicates.\n\nPlease go to https://planet.postgresql.org/register/edit/{2}\nand confirm (by unhiding) which of these should be posted.\n\nThank you!\n\n".format( - feed.blogurl, - "\n".join(["* " + t for t in titles]), - feed.id), - sendername="Planet PostgreSQL", - receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), + send_simple_mail( + settings.EMAIL_SENDER, + feed.user.email, + "Many posts found at your blog at Planet PostgreSQL", + "The blog aggregator at Planet PostgreSQL has just picked up the following\nposts from your blog at {0}:\n\n{1}\n\nSince this is a large number of posts, they have been fetched\nand marked as hidden, to avoid possible duplicates.\n\nPlease go to https://planet.postgresql.org/register/edit/{2}\nand confirm (by unhiding) which of these should be posted.\n\nThank you!\n\n".format( + feed.blogurl, + "\n".join(["* " + t for t in titles]), + feed.id), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), ) - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "Excessive posts from feed on Planet PostgreSQL", - "The blog at {0} by {1}\nreceived {2} new posts in a single fetch.\nAs this may be incorrect, the posts have been marked as hidden.\nThe author may individually mark them as visible depending on\nprevious posts, and has been sent a notification about this.".format(feed.feedurl, feed.user, len(ids)), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "Excessive posts from feed on Planet PostgreSQL", + "The blog at {0} by {1}\nreceived {2} new posts in a single fetch.\nAs this may be incorrect, the posts have been marked as hidden.\nThe author may individually mark them as visible depending on\nprevious posts, and has been sent a notification about this.".format(feed.feedurl, feed.user, len(ids)), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", ) else: # Email a notification that they were picked up - send_simple_mail(settings.EMAIL_SENDER, - feed.user.email, - "Posts found at your blog at Planet PostgreSQL", - "The blog aggregator at Planet PostgreSQL has just picked up the following\nposts from your blog at {0}:\n\n{1}\n\nIf these entries are correct, you don't have to do anything.\nIf any entry should not be there, head over to\n\nhttps://planet.postgresql.org/register/edit/{2}/\n\nand click the 'Hide' button for those entries as soon\nas possible.\n\nThank you!\n\n".format( - feed.blogurl, - "\n".join(["* " + t for t in titles]), - feed.id), - sendername="Planet PostgreSQL", - receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), + send_simple_mail( + settings.EMAIL_SENDER, + feed.user.email, + "Posts found at your blog at Planet PostgreSQL", + "The blog aggregator at Planet PostgreSQL has just picked up the following\nposts from your blog at {0}:\n\n{1}\n\nIf these entries are correct, you don't have to do anything.\nIf any entry should not be there, head over to\n\nhttps://planet.postgresql.org/register/edit/{2}/\n\nand click the 'Hide' button for those entries as soon\nas possible.\n\nThank you!\n\n".format( + feed.blogurl, + "\n".join(["* " + t for t in titles]), + feed.id), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), ) if entries > 0 and not had_entries: # Entries showed up on a blog that was previously empty - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog was added to Planet PostgreSQL", - "The blog at {0} by {1}\nwas added to Planet PostgreSQL, and has now received entries.\n\nTo moderate: https://planet.postgresql.org/register/moderate/\n\n".format(feed.feedurl, feed.user), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog was added to Planet PostgreSQL", + "The blog at {0} by {1}\nwas added to Planet PostgreSQL, and has now received entries.\n\nTo moderate: https://planet.postgresql.org/register/moderate/\n\n".format(feed.feedurl, feed.user), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", ) # If the blog URL changed, update it as requested if getattr(feed, 'new_blogurl', None): self.trace("URL changed for %s to %s" % (feed.feedurl, feed.new_blogurl)) - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog url changed on Planet PostgreSQL", - "When checking the blog at {0} by {1}\nthe blog URL was updated to:\n{2}\n(from previous value {3})\n\nTo moderate: https://planet.postgresql.org/register/moderate/\n\n".format(feed.feedurl, feed.user, feed.new_blogurl, feed.blogurl), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog url changed on Planet PostgreSQL", + "When checking the blog at {0} by {1}\nthe blog URL was updated to:\n{2}\n(from previous value {3})\n\nTo moderate: https://planet.postgresql.org/register/moderate/\n\n".format(feed.feedurl, feed.user, feed.new_blogurl, feed.blogurl), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", + ) + send_simple_mail( + settings.EMAIL_SENDER, + feed.user.email, + "URL of your blog at Planet PostgreSQL updated", + "The blog aggregator at Planet PostgreSQL has update the URL of your blog\nwith the feed at {0} to:\n{1} (from {2})\nIf this is correct, you don't have to do anything.\nIf not, please contact planet@postgresql.org\n".format( + feed.feedurl, + feed.new_blogurl, + feed.blogurl, + ), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), ) - send_simple_mail(settings.EMAIL_SENDER, - feed.user.email, - "URL of your blog at Planet PostgreSQL updated", - "The blog aggregator at Planet PostgreSQL has update the URL of your blog\nwith the feed at {0} to:\n{1} (from {2})\nIf this is correct, you don't have to do anything.\nIf not, please contact planet@postgresql.org\n".format( - feed.feedurl, - feed.new_blogurl, - feed.blogurl, - ), - sendername="Planet PostgreSQL", - receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), - ) feed.blogurl = feed.new_blogurl feed.save() if self.debug: diff --git a/hamnadmin/hamnadmin/register/management/commands/delete_old_logs.py b/hamnadmin/hamnadmin/register/management/commands/delete_old_logs.py index 09e9d87..50bde45 100644 --- a/hamnadmin/hamnadmin/register/management/commands/delete_old_logs.py +++ b/hamnadmin/hamnadmin/register/management/commands/delete_old_logs.py @@ -4,7 +4,8 @@ from django.db import transaction, connection from datetime import timedelta # How long should we keep logs? -LOG_KEEP_DAYS=300 +LOG_KEEP_DAYS = 300 + class Command(BaseCommand): help = "Delete old logs" diff --git a/hamnadmin/hamnadmin/register/management/commands/send_logs.py b/hamnadmin/hamnadmin/register/management/commands/send_logs.py index 6ba62b1..5e7dc32 100644 --- a/hamnadmin/hamnadmin/register/management/commands/send_logs.py +++ b/hamnadmin/hamnadmin/register/management/commands/send_logs.py @@ -9,24 +9,27 @@ from hamnadmin.mailqueue.util import send_simple_mail from hamnadmin.register.models import Blog # Number of errors in the past 24 hours to trigger email -THRESHOLD=20 +THRESHOLD = 20 + class Command(BaseCommand): help = "Send planet aggregation logs to blog owners" def handle(self, *args, **options): with transaction.atomic(): - for feed in Blog.objects.filter(archived=False, - aggregatorlog__success=False, - aggregatorlog__ts__gt=datetime.now()-timedelta(days=1), - ).annotate( - num=Count("aggregatorlog__id") - ).filter(num__gt=THRESHOLD).order_by(): + for feed in Blog.objects.filter( + archived=False, + aggregatorlog__success=False, + aggregatorlog__ts__gt=datetime.now() - timedelta(days=1), + ).annotate( + num=Count("aggregatorlog__id") + ).filter(num__gt=THRESHOLD).order_by(): # We assume this is only run once a day, so just generate one email - send_simple_mail(settings.EMAIL_SENDER, - feed.user.email, - "Errors retreiving your feed for Planet PostgreSQL", - """Your blog aggregated to Planet PostgreSQL with feed URL + send_simple_mail( + settings.EMAIL_SENDER, + feed.user.email, + "Errors retreiving your feed for Planet PostgreSQL", + """Your blog aggregated to Planet PostgreSQL with feed URL {0} @@ -47,10 +50,12 @@ and also stop these messages from being sent. A message like this will be sent once a day as long as your blog is generating more than {3} errors per day. -""".format(feed.feedurl, - feed.num, - feed.id, - THRESHOLD), - sendername="Planet PostgreSQL", - receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), - ) +""". + format( + feed.feedurl, + feed.num, + feed.id, + THRESHOLD), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(feed.user.first_name, feed.user.last_name), + ) diff --git a/hamnadmin/hamnadmin/register/models.py b/hamnadmin/hamnadmin/register/models.py index f598845..00b2857 100644 --- a/hamnadmin/hamnadmin/register/models.py +++ b/hamnadmin/hamnadmin/register/models.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta from hamnadmin.util.shortlink import urlvalmap + class Team(models.Model): teamurl = models.CharField(max_length=255, blank=False) name = models.CharField(max_length=255, blank=False) @@ -23,16 +24,17 @@ class Team(models.Model): def all_blogs(self): return self.blog_set.filter(approved=True, archived=False) + class Blog(models.Model): feedurl = models.CharField(max_length=255, blank=False) name = models.CharField(max_length=255, blank=False) blogurl = models.CharField(max_length=255, blank=False) - lastget = models.DateTimeField(default=datetime(2000,1,1)) + lastget = models.DateTimeField(default=datetime(2000, 1, 1)) user = models.ForeignKey(User, null=False, blank=False, on_delete=models.CASCADE) approved = models.BooleanField(default=False) archived = models.BooleanField(default=False) - authorfilter = models.CharField(max_length=255,default='',blank=True) - team = models.ForeignKey(Team,db_column='team', blank=True, null=True, on_delete=models.CASCADE) + authorfilter = models.CharField(max_length=255, default='', blank=True) + team = models.ForeignKey(Team, db_column='team', blank=True, null=True, on_delete=models.CASCADE) twitteruser = models.CharField(max_length=255, default='', blank=True) excludestats = models.BooleanField(null=False, blank=False, default=False) @@ -48,7 +50,7 @@ class Blog(models.Model): @property def recent_failures(self): - return self.aggregatorlog_set.filter(success=False, ts__gt=datetime.now()-timedelta(days=1)).count() + return self.aggregatorlog_set.filter(success=False, ts__gt=datetime.now() - timedelta(days=1)).count() @property def has_entries(self): @@ -58,7 +60,7 @@ class Blog(models.Model): def latestentry(self): try: return self.posts.filter(hidden=False)[0] - except: + except Exception: return None @property @@ -67,13 +69,14 @@ class Blog(models.Model): class Meta: db_table = 'feeds' - ordering = ['approved','name'] + ordering = ['approved', 'name'] class Admin: pass + class Post(models.Model): - feed = models.ForeignKey(Blog,db_column='feed',related_name='posts', on_delete=models.CASCADE) + feed = models.ForeignKey(Blog, db_column='feed', related_name='posts', on_delete=models.CASCADE) guid = models.CharField(max_length=255) link = models.CharField(max_length=255) txt = models.TextField() @@ -109,6 +112,7 @@ class Post(models.Model): i //= 64 return "https://postgr.es/p/%s" % s + class AuditEntry(models.Model): logtime = models.DateTimeField(default=datetime.now) user = models.CharField(max_length=32) @@ -125,13 +129,14 @@ class AuditEntry(models.Model): class Meta: db_table = 'auditlog' ordering = ['logtime'] - + + class AggregatorLog(models.Model): ts = models.DateTimeField(auto_now=True) feed = models.ForeignKey(Blog, db_column='feed', on_delete=models.CASCADE) success = models.BooleanField() info = models.TextField() - + class Meta: db_table = 'aggregatorlog' ordering = ['-ts'] diff --git a/hamnadmin/hamnadmin/register/templatetags/hamn.py b/hamnadmin/hamnadmin/register/templatetags/hamn.py index 0055650..8c59339 100644 --- a/hamnadmin/hamnadmin/register/templatetags/hamn.py +++ b/hamnadmin/hamnadmin/register/templatetags/hamn.py @@ -6,10 +6,11 @@ from hamnadmin.util.html import TruncateAndClean register = template.Library() + @register.filter(name='postcontents') @stringfilter def postcontents(value): try: return mark_safe(TruncateAndClean(value)) - except Exception as e: + except Exception: return "Unable to clean HTML" diff --git a/hamnadmin/hamnadmin/register/views.py b/hamnadmin/hamnadmin/register/views.py index cfc937f..d4012ab 100644 --- a/hamnadmin/hamnadmin/register/views.py +++ b/hamnadmin/hamnadmin/register/views.py @@ -14,6 +14,7 @@ import datetime from .forms import BlogEditForm, ModerateRejectForm + # Public planet def planet_home(request): statdate = datetime.datetime.now() - datetime.timedelta(days=61) @@ -33,28 +34,30 @@ def planet_feeds(request): 'teams': Team.objects.filter(blog__approved=True).distinct().order_by('name'), }) + def planet_add(request): return render(request, 'add.tmpl', { }) - # Registration interface (login and all) def issuperuser(user): return user.is_authenticated and user.is_superuser + @login_required def root(request): if request.user.is_superuser and 'admin' in request.GET and request.GET['admin'] == '1': blogs = Blog.objects.all().order_by('archived', 'approved', 'name') else: blogs = Blog.objects.filter(user=request.user).order_by('archived', 'approved', 'name') - return render(request, 'index.html',{ + return render(request, 'index.html', { 'blogs': blogs, 'teams': Team.objects.filter(manager=request.user).order_by('name'), 'title': 'Your blogs', }) + @login_required @transaction.atomic def edit(request, id=None): @@ -64,7 +67,7 @@ def edit(request, id=None): else: blog = get_object_or_404(Blog, id=id, user=request.user) else: - blog = Blog(user=request.user, name = "{0} {1}".format(request.user.first_name, request.user.last_name)) + blog = Blog(user=request.user, name="{0} {1}".format(request.user.first_name, request.user.last_name)) if request.method == 'POST': saved_url = blog.feedurl @@ -81,13 +84,14 @@ def edit(request, id=None): obj.approved = False obj.save() - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog was edited on Planet PostgreSQL", - "The blog at {0}\nwas edited by {1} in a way that needs new moderation.\n\nTo moderate: https://planet.postgresql.org/register/moderate/\n\n".format(blog.feedurl, blog.user), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", - ) + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog was edited on Planet PostgreSQL", + "The blog at {0}\nwas edited by {1} in a way that needs new moderation.\n\nTo moderate: https://planet.postgresql.org/register/moderate/\n\n".format(blog.feedurl, blog.user), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", + ) messages.warning(request, "Blog has been resubmitted for moderation, and is temporarily disabled.") @@ -114,7 +118,7 @@ def edit(request, id=None): return HttpResponseRedirect("/register/edit/{0}/".format(obj.id)) else: - form = BlogEditForm(request, instance=blog) + form = BlogEditForm(request, instance=blog) return render(request, 'edit.html', { 'new': id is None, @@ -125,6 +129,7 @@ def edit(request, id=None): 'title': 'Edit blog: %s' % blog.name, }) + @login_required @transaction.atomic def delete(request, id): @@ -133,12 +138,13 @@ def delete(request, id): else: blog = get_object_or_404(Blog, id=id, user=request.user) - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog was deleted on Planet PostgreSQL", - "The blog at {0} by {1}\nwas deleted by {2}\n\n".format(blog.feedurl, blog.name, request.user.username), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog was deleted on Planet PostgreSQL", + "The blog at {0} by {1}\nwas deleted by {2}\n\n".format(blog.feedurl, blog.name, request.user.username), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", ) blog.delete() messages.info(request, "Blog deleted.") @@ -146,6 +152,7 @@ def delete(request, id): purge_url('/feeds.html') return HttpResponseRedirect("/register/") + @login_required @transaction.atomic def archive(request, id): @@ -154,18 +161,20 @@ def archive(request, id): else: blog = get_object_or_404(Blog, id=id, user=request.user) - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog was archived on Planet PostgreSQL", - "The blog at {0} by {1}\nwas archived by {2}\n\n".format(blog.feedurl, blog.name, request.user.username), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog was archived on Planet PostgreSQL", + "The blog at {0} by {1}\nwas archived by {2}\n\n".format(blog.feedurl, blog.name, request.user.username), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", ) blog.archived = True blog.save() messages.info(request, "Blog archived.") return HttpResponseRedirect("/register/") + @login_required @transaction.atomic def remove_from_team(request, teamid, blogid): @@ -201,6 +210,7 @@ def remove_from_team(request, teamid, blogid): messages.info(request, "Blog {0} removed from team {1}".format(blog.feedurl, team.name)) return HttpResponseRedirect("/register/") + def __getvalidblogpost(request, blogid, postid): blog = get_object_or_404(Blog, id=blogid) post = get_object_or_404(Post, id=postid) @@ -210,6 +220,7 @@ def __getvalidblogpost(request, blogid, postid): raise Exception("Blog does not match post") return post + def __setposthide(request, blogid, postid, status): post = __getvalidblogpost(request, blogid, postid) post.hidden = status @@ -219,16 +230,19 @@ def __setposthide(request, blogid, postid, status): purge_root_and_feeds() return HttpResponseRedirect("/register/edit/{0}/".format(blogid)) + @login_required @transaction.atomic def blogpost_hide(request, blogid, postid): return __setposthide(request, blogid, postid, True) + @login_required @transaction.atomic def blogpost_unhide(request, blogid, postid): return __setposthide(request, blogid, postid, False) + @login_required @transaction.atomic def blogpost_delete(request, blogid, postid): @@ -237,7 +251,7 @@ def blogpost_delete(request, blogid, postid): # Update the feed last fetched date to be just before this entry, so that we end up # re-fetching it if necessary. - post.feed.lastget = post.dat - timedelta(minutes=1) + post.feed.lastget = post.dat - datetime.timedelta(minutes=1) post.feed.save() # Now actually delete it @@ -247,15 +261,17 @@ def blogpost_delete(request, blogid, postid): purge_root_and_feeds() return HttpResponseRedirect("/register/edit/{0}/".format(blogid)) + # Moderation @login_required @user_passes_test(issuperuser) def moderate(request): - return render(request, 'moderate.html',{ + return render(request, 'moderate.html', { 'blogs': Blog.objects.filter(approved=False).annotate(oldest=Max('posts__dat')).order_by('oldest'), 'title': 'Moderation', }) + @login_required @user_passes_test(issuperuser) @transaction.atomic @@ -267,22 +283,24 @@ def moderate_reject(request, blogid): if form.is_valid(): # Ok, actually reject this blog. # Always send moderator mail - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog was rejected on Planet PostgreSQL", - "The blog at {0} by {1} {2}\nwas marked as rejected by {3}. The message given was:\n\n{4}\n\n".format(blog.feedurl, blog.user.first_name, blog.user.last_name, request.user.username, form.cleaned_data['message']), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", - ) + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog was rejected on Planet PostgreSQL", + "The blog at {0} by {1} {2}\nwas marked as rejected by {3}. The message given was:\n\n{4}\n\n".format(blog.feedurl, blog.user.first_name, blog.user.last_name, request.user.username, form.cleaned_data['message']), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", + ) messages.info(request, "Blog {0} rejected, notification sent to moderators".format(blog.feedurl)) if not form.cleaned_data['modsonly']: - send_simple_mail(settings.EMAIL_SENDER, - blog.user.email, - "Your blog submission to Planet PostgreSQL", - "The blog at {0} that you submitted to Planet PostgreSQL has\nunfortunately been rejected. The reason given was:\n\n{1}\n\n".format(blog.feedurl, form.cleaned_data['message']), - sendername="Planet PostgreSQL", - receivername = "{0} {1}".format(blog.user.first_name, blog.user.last_name), - ) + send_simple_mail( + settings.EMAIL_SENDER, + blog.user.email, + "Your blog submission to Planet PostgreSQL", + "The blog at {0} that you submitted to Planet PostgreSQL has\nunfortunately been rejected. The reason given was:\n\n{1}\n\n".format(blog.feedurl, form.cleaned_data['message']), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(blog.user.first_name, blog.user.last_name), + ) messages.info(request, "Blog {0} rejected, notification sent to blog owner".format(blog.feedurl)) blog.delete() @@ -296,6 +314,7 @@ def moderate_reject(request, blogid): 'title': 'Reject blog', }) + @login_required @user_passes_test(issuperuser) @transaction.atomic @@ -306,20 +325,22 @@ def moderate_approve(request, blogid): messages.info(request, "Blog {0} was already approved.".format(blog.feedurl)) return HttpResponseRedirect("/register/moderate/") - send_simple_mail(settings.EMAIL_SENDER, - settings.NOTIFICATION_RECEIVER, - "A blog was approved on Planet PostgreSQL", - "The blog at {0} by {1} {2}\nwas marked as approved by {3}.\n\n".format(blog.feedurl, blog.user.first_name, blog.user.last_name, request.user.username), - sendername="Planet PostgreSQL", - receivername="Planet PostgreSQL Moderators", + send_simple_mail( + settings.EMAIL_SENDER, + settings.NOTIFICATION_RECEIVER, + "A blog was approved on Planet PostgreSQL", + "The blog at {0} by {1} {2}\nwas marked as approved by {3}.\n\n".format(blog.feedurl, blog.user.first_name, blog.user.last_name, request.user.username), + sendername="Planet PostgreSQL", + receivername="Planet PostgreSQL Moderators", ) - send_simple_mail(settings.EMAIL_SENDER, - blog.user.email, - "Your blog submission to Planet PostgreSQL", - "The blog at {0} that you submitted to Planet PostgreSQL has\nbeen approved.\n\n".format(blog.feedurl), - sendername="Planet PostgreSQL", - receivername = "{0} {1}".format(blog.user.first_name, blog.user.last_name), + send_simple_mail( + settings.EMAIL_SENDER, + blog.user.email, + "Your blog submission to Planet PostgreSQL", + "The blog at {0} that you submitted to Planet PostgreSQL has\nbeen approved.\n\n".format(blog.feedurl), + sendername="Planet PostgreSQL", + receivername="{0} {1}".format(blog.user.first_name, blog.user.last_name), ) blog.approved = True diff --git a/hamnadmin/hamnadmin/settings.py b/hamnadmin/hamnadmin/settings.py index e92d5e2..b4fff7b 100644 --- a/hamnadmin/hamnadmin/settings.py +++ b/hamnadmin/hamnadmin/settings.py @@ -4,12 +4,12 @@ import os DEBUG = False ADMINS = ( - ('PostgreSQL Webmaster', 'webmaster@postgresql.org'), + ('PostgreSQL Webmaster', 'webmaster@postgresql.org'), ) MANAGERS = ADMINS -DATABASES={ +DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'planetbeta', @@ -72,16 +72,16 @@ AUTHENTICATION_BACKENDS = ( LOGIN_URL = '/register/login' -ALLOWED_HOSTS=['*'] +ALLOWED_HOSTS = ['*'] -EMAIL_SENDER='planet@postgresql.org' -NOTIFICATION_RECEIVER='planet@postgresql.org' +EMAIL_SENDER = 'planet@postgresql.org' +NOTIFICATION_RECEIVER = 'planet@postgresql.org' # Set to None for testing -VARNISH_URL="http://localhost/varnish-purge" +VARNISH_URL = "http://localhost/varnish-purge" # Max number of entries in a fetch before we start marking them as hidden -MAX_SAFE_ENTRIES_PER_FETCH=4 +MAX_SAFE_ENTRIES_PER_FETCH = 4 # Dynamically load settings from the "outer" planet.ini that might # be needed. @@ -89,19 +89,18 @@ try: import configparser _configparser = configparser.ConfigParser() _configparser.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), '../../planet.ini')) - TWITTER_CLIENT=_configparser.get('twitter', 'consumer') - TWITTER_CLIENTSECRET=_configparser.get('twitter', 'consumersecret') - TWITTER_TOKEN=_configparser.get('twitter', 'token') - TWITTER_TOKENSECRET=_configparser.get('twitter', 'secret') -except: - TWITTER_CLIENT=None - TWITTER_CLIENTSECRET=None - TWITTER_TOKEN=None - TWITTER_TOKENSECRET=None + TWITTER_CLIENT = _configparser.get('twitter', 'consumer') + TWITTER_CLIENTSECRET = _configparser.get('twitter', 'consumersecret') + TWITTER_TOKEN = _configparser.get('twitter', 'token') + TWITTER_TOKENSECRET = _configparser.get('twitter', 'secret') +except Exception: + TWITTER_CLIENT = None + TWITTER_CLIENTSECRET = None + TWITTER_TOKEN = None + TWITTER_TOKENSECRET = None # If there is a local_settings.py, let it override our settings try: from .local_settings import * -except: +except Exception: pass - diff --git a/hamnadmin/hamnadmin/util/aggregate.py b/hamnadmin/hamnadmin/util/aggregate.py index e506e7a..bb499eb 100644 --- a/hamnadmin/hamnadmin/util/aggregate.py +++ b/hamnadmin/hamnadmin/util/aggregate.py @@ -7,11 +7,13 @@ from vendor.feedparser import feedparser from hamnadmin.register.models import Post + class ParserGotRedirect(Exception): def __init__(self, url): self.url = url super(Exception, self).__init__() + class FeedFetcher(object): def __init__(self, feed, tracefunc=None, update=True): self.feed = feed @@ -59,7 +61,7 @@ class FeedFetcher(object): self.feed.blogurl = parser.feed.link elif self.feed.blogurl != parser.feed.link: self.feed.new_blogurl = parser.feed.link - except: + except Exception: pass for entry in parser.entries: @@ -75,7 +77,7 @@ class FeedFetcher(object): txtalts = [] try: txtalts.append(entry.content[0].value) - except: + except Exception: pass if 'summary' in entry: txtalts.append(entry.summary) @@ -112,13 +114,12 @@ class FeedFetcher(object): title=entry.title, ) - # Check if we got back a Last-Modified time if hasattr(parser, 'modified_parsed') and parser['modified_parsed']: # Last-Modified header retreived. If we did receive it, we will # trust the content (assuming we can parse it) d = datetime.datetime(*parser['modified_parsed'][:6]) - if (d-datetime.datetime.now()).days > 5: + if (d - datetime.datetime.now()).days > 5: # Except if it's ridiculously long in the future, we'll set it # to right now instead, to deal with buggy blog software. We # currently define rediculously long as 5 days diff --git a/hamnadmin/hamnadmin/util/html.py b/hamnadmin/hamnadmin/util/html.py index 8401ebf..6dc9fa2 100644 --- a/hamnadmin/hamnadmin/util/html.py +++ b/hamnadmin/hamnadmin/util/html.py @@ -2,6 +2,7 @@ from html.parser import HTMLParser import tidylib import urllib.parse + def TruncateAndClean(txt): # First apply Tidy (txt, errors) = tidylib.tidy_document(txt, @@ -27,10 +28,11 @@ def TruncateAndClean(txt): # Remove initial
tags while out.startswith('')+1:] + out = out[out.find('>') + 1:] return out + class HtmlTruncator(HTMLParser): def __init__(self, maxlen): HTMLParser.__init__(self) @@ -40,18 +42,19 @@ class HtmlTruncator(HTMLParser): self.trunctxt = '' self.tagstack = [] self.skiprest = False - + def feed(self, txt): txt = txt.lstrip() self.fulltxt += txt HTMLParser.feed(self, txt) def handle_startendtag(self, tag, attrs): - if self.skiprest: return + if self.skiprest: + return self.trunctxt += self.get_starttag_text() - + def quoteurl(self, str): - p = str.split(":",2) + p = str.split(":", 2) if len(p) < 2: # Don't crash on invalid URLs return "" @@ -63,29 +66,33 @@ class HtmlTruncator(HTMLParser): return attrs def handle_starttag(self, tag, attrs): - if self.skiprest: return + if self.skiprest: + return self.trunctxt += "<" + tag - self.trunctxt += (' '.join([(' %s="%s"' % (k,v)) for k,v in map(self.cleanhref, attrs)])) + self.trunctxt += (' '.join([(' %s="%s"' % (k, v)) for k, v in map(self.cleanhref, attrs)])) self.trunctxt += ">" self.tagstack.append(tag) def handle_endtag(self, tag): - if self.skiprest: return + if self.skiprest: + return self.trunctxt += "" self.tagstack.pop() def handle_entityref(self, ref): self.len += 1 - if self.skiprest: return + if self.skiprest: + return self.trunctxt += "&" + ref + ";" def handle_data(self, data): self.len += len(data) - if self.skiprest: return + if self.skiprest: + return self.trunctxt += data if self.len > self.maxlen: # Passed max length, so truncate text as close to the limit as possible - self.trunctxt = self.trunctxt[0:len(self.trunctxt)-(self.len-self.maxlen)] + self.trunctxt = self.trunctxt[0:len(self.trunctxt) - (self.len - self.maxlen)] # Now append any tags that weren't properly closed self.tagstack.reverse() diff --git a/hamnadmin/hamnadmin/util/varnish.py b/hamnadmin/hamnadmin/util/varnish.py index 869a210..0c19860 100644 --- a/hamnadmin/hamnadmin/util/varnish.py +++ b/hamnadmin/hamnadmin/util/varnish.py @@ -2,6 +2,7 @@ from django.conf import settings import requests + def purge_url(url): if not settings.VARNISH_URL: print("Not purging {0}".format(url)) @@ -15,5 +16,6 @@ def purge_url(url): except Exception as e: raise Exception("Failed to purge '{0}': {1}'".format(url, e)) + def purge_root_and_feeds(): purge_url('/(|rss20.*)$') diff --git a/listsync.py b/listsync.py index 9341dc8..c245607 100755 --- a/listsync.py +++ b/listsync.py @@ -13,7 +13,7 @@ import psycopg2 import requests -if __name__=="__main__": +if __name__ == "__main__": c = configparser.ConfigParser() c.read('planet.ini') @@ -26,9 +26,10 @@ WHERE feeds.approved AND NOT feeds.archived """) syncstruct = [{'email': r[0]} for r in curs.fetchall()] - r = requests.put('{0}/api/subscribers/{1}/'.format(c.get('list', 'server'), c.get('list', 'listname')), - headers={'X-api-key': c.get('list', 'apikey')}, - json=syncstruct, + r = requests.put( + '{0}/api/subscribers/{1}/'.format(c.get('list', 'server'), c.get('list', 'listname')), + headers={'X-api-key': c.get('list', 'apikey')}, + json=syncstruct, ) if r.status_code != 200: print("Failed to talk to pglister api: %s" % r.status_code) diff --git a/posttotwitter.py b/posttotwitter.py index 4911adb..af37302 100755 --- a/posttotwitter.py +++ b/posttotwitter.py @@ -19,13 +19,13 @@ from twitterclient import TwitterClient # Simple map used to shorten id values to URLs _urlvalmap = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '-', '_'] + class PostToTwitter(TwitterClient): def __init__(self, cfg): TwitterClient.__init__(self, cfg) psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) - self.db = psycopg2.connect(c.get('planet','db')) - + self.db = psycopg2.connect(c.get('planet', 'db')) def do_post(self, msg): """ @@ -43,7 +43,7 @@ class PostToTwitter(TwitterClient): FROM posts INNER JOIN feeds ON posts.feed=feeds.id WHERE approved AND age(dat) < '7 days' AND NOT (twittered OR hidden) ORDER BY dat""") for post in c.fetchall(): - if post[3] and len(post[3])>1: + if post[3] and len(post[3]) > 1: short = post[3] else: # No short-link exists, so create one. We need the short-link @@ -62,18 +62,18 @@ class PostToTwitter(TwitterClient): self.db.commit() # Set up the string to twitter - if post[5] and len(post[5])>1: + if post[5] and len(post[5]) > 1: # Twitter username registered msg = "%s (@%s): %s %s" % ( post[4], post[5], - self.trimpost(post[1],len(post[4])+len(post[5])+len(short)+7), + self.trimpost(post[1], len(post[4]) + len(post[5]) + len(short) + 7), short, ) else: msg = "%s: %s %s" % ( post[4], - self.trimpost(post[1],len(post[4])+len(short)+3), + self.trimpost(post[1], len(post[4]) + len(short) + 3), short, ) @@ -86,20 +86,18 @@ class PostToTwitter(TwitterClient): continue # Flag this item as posted - c.execute("UPDATE posts SET twittered='t' WHERE id=%(id)s", { 'id': post[0] }) + c.execute("UPDATE posts SET twittered='t' WHERE id=%(id)s", {'id': post[0]}) self.db.commit() print("Twittered: %s" % msg) - # Trim a post to the length required by twitter, so we don't fail to post # if a title is really long. Assume other parts of the string to be # posted are characters. def trimpost(self, txt, otherlen): if len(txt) + otherlen < 140: return txt - return "%s..." % (txt[:(140-otherlen-3)]) - + return "%s..." % (txt[:(140 - otherlen - 3)]) # Trim an URL using https://postgr.es def shortid(self, id): @@ -110,9 +108,7 @@ class PostToTwitter(TwitterClient): return "https://postgr.es/p/%s" % s - -if __name__=="__main__": +if __name__ == "__main__": c = configparser.ConfigParser() c.read('planet.ini') PostToTwitter(c).Run() - diff --git a/redirector/redirector.py b/redirector/redirector.py index 4f07345..d6dcf87 100755 --- a/redirector/redirector.py +++ b/redirector/redirector.py @@ -17,6 +17,7 @@ _urlvalmap = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', ' connstr = "" + def iddecode(idstr): idval = 0 for c in idstr: @@ -24,6 +25,7 @@ def iddecode(idstr): idval += _urlvalmap.index(c) return idval + def application(environ, start_response): try: # If we have a querystring, get rid of it. This can (presumably) @@ -43,38 +45,38 @@ def application(environ, start_response): conn = psycopg2.connect(connstr) c = conn.cursor() c.execute("SELECT link FROM posts WHERE id=%(id)s", { - 'id': id - }) + 'id': id + }) r = c.fetchall() conn.close() if len(r) != 1: start_response('404 Not Found', [ - ('Content-type', 'text/plain'), - ]) + ('Content-type', 'text/plain'), + ]) return [b"Link not found\n"] # We have a link, return a redirect to it start_response('301 Moved Permanently', [ - ('Content-type', 'text/html'), - ('Location', r[0][0]), - ('X-Planet', str(id)) - ]) + ('Content-type', 'text/html'), + ('Location', r[0][0]), + ('X-Planet', str(id)) + ]) return [ b"\n\npostgr.es\n\n\n", b"moved here\n" % r[0][0].encode('utf8'), b"\n\n" - ] + ] except Exception as ex: start_response('500 Internal Server Error', [ - ('Content-type', 'text/plain') - ]) + ('Content-type', 'text/plain') + ]) return [ "An internal server error occured\n", str(ex) - ] + ] c = configparser.ConfigParser() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..c432b81 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[pycodestyle] +statistics=True +exclude=hamnadmin/vendor/* +ignore=E123,E124,E731,E501,W504 +max-line-length=120 diff --git a/setuptwitter.py b/setuptwitter.py index 562d1be..0e98dfc 100755 --- a/setuptwitter.py +++ b/setuptwitter.py @@ -41,6 +41,3 @@ print("Access token received.") print("Register the following two valuesi n planet.ini under [twitter]:") print("token={0}".format(tokens.get('oauth_token'))) print("secret={0}".format(tokens.get('oauth_token_secret'))) - - - diff --git a/synctwitter.py b/synctwitter.py index f5472ec..0907027 100755 --- a/synctwitter.py +++ b/synctwitter.py @@ -13,18 +13,19 @@ import psycopg2.extensions import configparser from twitterclient import TwitterClient + class SyncTwitter(TwitterClient): def __init__(self, cfg): TwitterClient.__init__(self, cfg) psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) - self.db = psycopg2.connect(cfg.get('planet','db')) + self.db = psycopg2.connect(cfg.get('planet', 'db')) def Run(self): # Get list of handles that should be on the list curs = self.db.cursor() - curs.execute("SELECT DISTINCT lower(twitteruser) FROM feeds WHERE approved AND NOT (twitteruser IS NULL OR twitteruser='') ORDER BY lower(twitteruser)"); - expected = set([r[0].replace('@','') for r in curs.fetchall()]) + curs.execute("SELECT DISTINCT lower(twitteruser) FROM feeds WHERE approved AND NOT (twitteruser IS NULL OR twitteruser='') ORDER BY lower(twitteruser)") + expected = set([r[0].replace('@', '') for r in curs.fetchall()]) # Get list of current screen names the list is following current = set(self.list_subscribers()) @@ -49,7 +50,7 @@ class SyncTwitter(TwitterClient): self.db.commit() -if __name__=="__main__": +if __name__ == "__main__": c = configparser.ConfigParser() c.read('planet.ini') SyncTwitter(c).Run() diff --git a/twitterclient.py b/twitterclient.py index 0b367c0..79b7083 100644 --- a/twitterclient.py +++ b/twitterclient.py @@ -9,6 +9,7 @@ Copyright (C) 2009-2019 PostgreSQL Global Development Group """ import requests_oauthlib + class TwitterClient(object): """ Base class representing a twitter client, implementing all those twitter @@ -35,7 +36,7 @@ class TwitterClient(object): def list_subscribers(self): # Eek. It seems subscribers are paged even if we don't ask for it # Thus, we need to loop with multiple requests - cursor=-1 + cursor = -1 handles = [] while cursor != 0: response = self.tw.get('{0}lists/members.json'.format(self.twitter_api), params={ @@ -62,7 +63,7 @@ class TwitterClient(object): if r.status_code != 200: try: err = r.json()['errors'][0]['message'] - except: + except Exception: err = 'Response does not contain error messages with json' print("Failed to remove subscriber {0}: {1}".format(name, err)) return False @@ -78,7 +79,7 @@ class TwitterClient(object): if r.status_code != 200: try: err = r.json()['errors'][0]['message'] - except: + except Exception: err = 'Response does not contain error messages with json' print("Failed to add subscriber {0}: {1}".format(name, err)) return False -- 2.39.5