From: Magnus Hagander Date: Thu, 4 Sep 2025 21:33:50 +0000 (+0200) Subject: Add support for bulk downloading benefit images and files X-Git-Url: http://git.postgresql.org/gitweb/static/gitweb.js?a=commitdiff_plain;h=bee49688281de43999856753d432d18ef5b59776;p=pgeu-system.git Add support for bulk downloading benefit images and files This will return a .tar.gz of all the benefit uploaded images and files. This can be useful for example to archive the contents of a conference without having to use a token to loop over all files etc. --- diff --git a/postgresqleu/confsponsor/urls.py b/postgresqleu/confsponsor/urls.py index 2059f849..09f184cd 100644 --- a/postgresqleu/confsponsor/urls.py +++ b/postgresqleu/confsponsor/urls.py @@ -43,6 +43,7 @@ urlpatterns = [ re_path(r'^admin/(\w+)/viewmail/(\d+)/$', views.sponsor_admin_view_mail), re_path(r'^admin/(\w+)/testvat/$', views.sponsor_admin_test_vat), re_path(r'^admin/(\w+)/benefitreports/$', views.sponsor_admin_benefit_reports), + re_path(r'^admin/(\w+)/benefitdownload/(image|file)/$', views.sponsor_admin_benefit_downloads), re_path(r'^admin/(\w+)/levels/(.*/)?$', backendviews.edit_sponsorship_levels), re_path(r'^admin/(\w+)/contracts/(\d+)/editfields/$', backendviews.edit_sponsorship_contract_fields), re_path(r'^admin/(\w+)/contracts/(\d+)/previewfields/$', backendviews.preview_sponsorship_contract_fields), diff --git a/postgresqleu/confsponsor/views.py b/postgresqleu/confsponsor/views.py index 15621908..a9091d02 100644 --- a/postgresqleu/confsponsor/views.py +++ b/postgresqleu/confsponsor/views.py @@ -1,5 +1,5 @@ from django.shortcuts import render, get_object_or_404 -from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404 +from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404, StreamingHttpResponse from django.http import HttpResponseNotModified from django.core.exceptions import PermissionDenied from django.contrib.auth.decorators import login_required @@ -24,11 +24,12 @@ from postgresqleu.confreg.util import get_authenticated_conference, get_conferen from postgresqleu.confreg.util import send_conference_mail from postgresqleu.confreg.twitter import post_conference_social, render_multiprovider_tweet from postgresqleu.confreg.twitter import get_all_conference_social_media -from postgresqleu.util.db import exec_no_result +from postgresqleu.util.db import exec_no_result, exec_to_list, exec_to_keyed_scalar, ensure_conference_timezone from postgresqleu.util.storage import InlineEncodedStorage from postgresqleu.util.decorators import superuser_required from postgresqleu.util.request import get_int_or_error from postgresqleu.util.time import today_global +from postgresqleu.util.tar import generate_streaming_tar from postgresqleu.invoices.util import InvoiceWrapper, InvoiceManager from postgresqleu.digisign.pdfutil import fill_pdf_fields, pdf_watermark_preview from postgresqleu.digisign.models import DigisignDocument, DigisignLog @@ -1035,6 +1036,16 @@ ORDER BY l.levelcost DESC, l.levelname, s.name, b.sortkey, b.benefitname""", {'c else: shipments = None + has_downloads = exec_to_keyed_scalar("""SELECT DISTINCT +CASE WHEN benefit_class=1 THEN 'image' ELSE 'file' END, true +FROM confsponsor_sponsorclaimedbenefit cb +INNER JOIN confsponsor_sponsorshipbenefit b ON b.id=cb.benefit_id +INNER JOIN confsponsor_sponsorshiplevel l ON l.id=b.level_id +WHERE l.conference_id=%(confid)s AND benefit_class IN (1,8) AND NOT declined""", + { + 'confid': conference.id, + }) + return render(request, 'confsponsor/admin_dashboard.html', { 'conference': conference, 'confirmed_sponsors': confirmed_sponsors, @@ -1045,6 +1056,7 @@ ORDER BY l.levelcost DESC, l.levelname, s.name, b.sortkey, b.benefitname""", {'c 'has_shipment_tracking': has_shipment_tracking, 'shipments': shipments, 'additionalcontracts': SponsorAdditionalContract.objects.select_related('sponsor').filter(sponsor__conference=conference).order_by('id'), + 'downloads': has_downloads, 'helplink': 'sponsors', }) @@ -1559,6 +1571,55 @@ def sponsor_admin_benefit_reports(request, confurlname): }) +@login_required +def sponsor_admin_benefit_downloads(request, confurlname, downloadtype): + conference = get_authenticated_conference(request, confurlname) + + benefitclass = 1 if downloadtype == 'image' else 8 + + with ensure_conference_timezone(conference): + downloads = exec_to_list("""SELECT s.name, b.benefitname, cb.id, EXTRACT(epoch FROM cb.claimedat), count(*) OVER w, row_number() OVER w +FROM confsponsor_sponsorclaimedbenefit cb +INNER JOIN confsponsor_sponsorshipbenefit b ON b.id=cb.benefit_id +INNER JOIN confsponsor_sponsorshiplevel l ON l.id=b.level_id +INNER JOIN confsponsor_sponsor s ON s.id=cb.sponsor_id +WHERE l.conference_id=%(confid)s AND benefit_class = %(class)s AND NOT declined +WINDOW w AS (PARTITION BY b.benefitname, s.name ORDER BY b.benefitname, s.name) +ORDER BY b.benefitname, s.name""", + { + 'confid': conference.id, + 'class': benefitclass, + }) + + def _generate_tar(): + for benefitname, sponsorname, storageid, claimtime, benefitcount, benefitnum in downloads: + data, datalen, metadata = exec_to_list("SELECT data, length(data), metadata FROM util_storage WHERE key=%(key)s AND storageid=%(id)s", { + 'key': 'benefit_{}'.format(downloadtype), + 'id': storageid, + })[0] + if downloadtype == 'image': + if benefitcount == 1: + filename = '{}/{}.png'.format(slugify(benefitname), slugify(sponsorname)) + else: + filename = '{}/{}_{}.png'.format(slugify(benefitname), slugify(sponsorname), benefitnum) + else: + if benefitcount == 1: + filename = '{}/{}/{}'.format(slugify(benefitname), slugify(sponsorname), metadata['filename']) + else: + filename = '{}/{}/{}/{}'.format(slugify(benefitname), slugify(sponsorname), benefitnum, metadata['filename']) + yield ( + filename, + claimtime, + data, + datalen, + ) + + resp = StreamingHttpResponse(generate_streaming_tar(_generate_tar)) + resp['Content-Type'] = 'application/tar+gzip' + resp['Content-Disposition'] = 'attachment; filename={}.tar.gz'.format(downloadtype) + return resp + + @superuser_required def sponsor_admin_test_vat(request, confurlname): # Just verify the conference exists and we have permissions diff --git a/postgresqleu/util/tar.py b/postgresqleu/util/tar.py new file mode 100644 index 00000000..b5f81e05 --- /dev/null +++ b/postgresqleu/util/tar.py @@ -0,0 +1,58 @@ +import io +import tarfile + + +class TarStreamer: + def __init__(self): + self.buf = io.BytesIO() + self.ofs = 0 + + def write(self, s): + self.buf.write(s) + self.ofs += len(s) + + def tell(self): + return self.ofs + + def close(self): + self.buf.close() + + def pop(self): + print("Returning {} bytes at offset {}".format(len(self.buf.getvalue()), self.ofs)) + s = self.buf.getvalue() + self.buf.close() + self.buf = io.BytesIO() + return s + + +class BytesReader: + def __init__(self, buf): + self.buf = buf + self.len = len(buf) + self.pos = 0 + + def read(self, size=-1): + if size == -1: + readsize = self.len - self.pos + else: + if self.pos + size > self.len: + readsize = self.len - self.pos + else: + readsize = size + + oldpos = self.pos + self.pos += readsize + return self.buf[oldpos:oldpos + readsize] + + +def generate_streaming_tar(tar_generator): + streamer = TarStreamer() + tar = tarfile.TarFile.open(mode='w|gz', fileobj=streamer, bufsize=tarfile.BLOCKSIZE) + for name, mtime, data, datalen in tar_generator(): + info = tarfile.TarInfo(name) + info.size = int(datalen) + info.mtime = mtime + tar.addfile(info, BytesReader(data)) + yield streamer.pop() + tar.close() + yield streamer.pop() diff --git a/template/confsponsor/admin_dashboard.html b/template/confsponsor/admin_dashboard.html index e7d0ae89..52cac96b 100644 --- a/template/confsponsor/admin_dashboard.html +++ b/template/confsponsor/admin_dashboard.html @@ -289,6 +289,13 @@ This matrix gives a quick overview of the status of the different benefits for e
Benefit reports
+

Downloads

+
+{%if downloads.image %}
Download images
{% endif %} +{%if downloads.file %}
Download files
{% endif %} +
+ +

Metadata

Sponsorship levels