Add support for bulk downloading benefit images and files
authorMagnus Hagander <magnus@hagander.net>
Thu, 4 Sep 2025 21:33:50 +0000 (23:33 +0200)
committerMagnus Hagander <magnus@hagander.net>
Thu, 4 Sep 2025 21:33:50 +0000 (23:33 +0200)
This will return a .tar.gz of all the benefit uploaded images and files.
This can be useful for example to archive the contents of a conference
without having to use a token to loop over all files etc.

postgresqleu/confsponsor/urls.py
postgresqleu/confsponsor/views.py
postgresqleu/util/tar.py [new file with mode: 0644]
template/confsponsor/admin_dashboard.html

index 2059f8490b2dc92d1f4aac585c752b1f89ad6c97..09f184cd48cd275e68effa509b582b48afc8327c 100644 (file)
@@ -43,6 +43,7 @@ urlpatterns = [
     re_path(r'^admin/(\w+)/viewmail/(\d+)/$', views.sponsor_admin_view_mail),
     re_path(r'^admin/(\w+)/testvat/$', views.sponsor_admin_test_vat),
     re_path(r'^admin/(\w+)/benefitreports/$', views.sponsor_admin_benefit_reports),
+    re_path(r'^admin/(\w+)/benefitdownload/(image|file)/$', views.sponsor_admin_benefit_downloads),
     re_path(r'^admin/(\w+)/levels/(.*/)?$', backendviews.edit_sponsorship_levels),
     re_path(r'^admin/(\w+)/contracts/(\d+)/editfields/$', backendviews.edit_sponsorship_contract_fields),
     re_path(r'^admin/(\w+)/contracts/(\d+)/previewfields/$', backendviews.preview_sponsorship_contract_fields),
index 15621908ed53ac17a581bc70547cb247badbccf6..a9091d02c42a04c880d668c9d4c29404e689496a 100644 (file)
@@ -1,5 +1,5 @@
 from django.shortcuts import render, get_object_or_404
-from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404
+from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden, Http404, StreamingHttpResponse
 from django.http import HttpResponseNotModified
 from django.core.exceptions import PermissionDenied
 from django.contrib.auth.decorators import login_required
@@ -24,11 +24,12 @@ from postgresqleu.confreg.util import get_authenticated_conference, get_conferen
 from postgresqleu.confreg.util import send_conference_mail
 from postgresqleu.confreg.twitter import post_conference_social, render_multiprovider_tweet
 from postgresqleu.confreg.twitter import get_all_conference_social_media
-from postgresqleu.util.db import exec_no_result
+from postgresqleu.util.db import exec_no_result, exec_to_list, exec_to_keyed_scalar, ensure_conference_timezone
 from postgresqleu.util.storage import InlineEncodedStorage
 from postgresqleu.util.decorators import superuser_required
 from postgresqleu.util.request import get_int_or_error
 from postgresqleu.util.time import today_global
+from postgresqleu.util.tar import generate_streaming_tar
 from postgresqleu.invoices.util import InvoiceWrapper, InvoiceManager
 from postgresqleu.digisign.pdfutil import fill_pdf_fields, pdf_watermark_preview
 from postgresqleu.digisign.models import DigisignDocument, DigisignLog
@@ -1035,6 +1036,16 @@ ORDER BY l.levelcost DESC, l.levelname, s.name, b.sortkey, b.benefitname""", {'c
     else:
         shipments = None
 
+    has_downloads = exec_to_keyed_scalar("""SELECT DISTINCT
+CASE WHEN benefit_class=1 THEN 'image' ELSE 'file' END, true
+FROM confsponsor_sponsorclaimedbenefit cb
+INNER JOIN confsponsor_sponsorshipbenefit b ON b.id=cb.benefit_id
+INNER JOIN confsponsor_sponsorshiplevel l ON l.id=b.level_id
+WHERE l.conference_id=%(confid)s AND benefit_class IN (1,8) AND NOT declined""",
+                                         {
+                                             'confid': conference.id,
+                                         })
+
     return render(request, 'confsponsor/admin_dashboard.html', {
         'conference': conference,
         'confirmed_sponsors': confirmed_sponsors,
@@ -1045,6 +1056,7 @@ ORDER BY l.levelcost DESC, l.levelname, s.name, b.sortkey, b.benefitname""", {'c
         'has_shipment_tracking': has_shipment_tracking,
         'shipments': shipments,
         'additionalcontracts': SponsorAdditionalContract.objects.select_related('sponsor').filter(sponsor__conference=conference).order_by('id'),
+        'downloads': has_downloads,
         'helplink': 'sponsors',
         })
 
@@ -1559,6 +1571,55 @@ def sponsor_admin_benefit_reports(request, confurlname):
         })
 
 
+@login_required
+def sponsor_admin_benefit_downloads(request, confurlname, downloadtype):
+    conference = get_authenticated_conference(request, confurlname)
+
+    benefitclass = 1 if downloadtype == 'image' else 8
+
+    with ensure_conference_timezone(conference):
+        downloads = exec_to_list("""SELECT s.name, b.benefitname, cb.id, EXTRACT(epoch FROM cb.claimedat), count(*) OVER w, row_number() OVER w
+FROM confsponsor_sponsorclaimedbenefit cb
+INNER JOIN confsponsor_sponsorshipbenefit b ON b.id=cb.benefit_id
+INNER JOIN confsponsor_sponsorshiplevel l ON l.id=b.level_id
+INNER JOIN confsponsor_sponsor s ON s.id=cb.sponsor_id
+WHERE l.conference_id=%(confid)s AND benefit_class = %(class)s AND NOT declined
+WINDOW w AS (PARTITION BY b.benefitname, s.name ORDER BY b.benefitname, s.name)
+ORDER BY b.benefitname, s.name""",
+                                 {
+                                     'confid': conference.id,
+                                     'class': benefitclass,
+                                 })
+
+    def _generate_tar():
+        for benefitname, sponsorname, storageid, claimtime, benefitcount, benefitnum in downloads:
+            data, datalen, metadata = exec_to_list("SELECT data, length(data), metadata FROM util_storage WHERE key=%(key)s AND storageid=%(id)s", {
+                'key': 'benefit_{}'.format(downloadtype),
+                'id': storageid,
+            })[0]
+            if downloadtype == 'image':
+                if benefitcount == 1:
+                    filename = '{}/{}.png'.format(slugify(benefitname), slugify(sponsorname))
+                else:
+                    filename = '{}/{}_{}.png'.format(slugify(benefitname), slugify(sponsorname), benefitnum)
+            else:
+                if benefitcount == 1:
+                    filename = '{}/{}/{}'.format(slugify(benefitname), slugify(sponsorname), metadata['filename'])
+                else:
+                    filename = '{}/{}/{}/{}'.format(slugify(benefitname), slugify(sponsorname), benefitnum, metadata['filename'])
+            yield (
+                filename,
+                claimtime,
+                data,
+                datalen,
+            )
+
+    resp = StreamingHttpResponse(generate_streaming_tar(_generate_tar))
+    resp['Content-Type'] = 'application/tar+gzip'
+    resp['Content-Disposition'] = 'attachment; filename={}.tar.gz'.format(downloadtype)
+    return resp
+
+
 @superuser_required
 def sponsor_admin_test_vat(request, confurlname):
     # Just verify the conference exists and we have permissions
diff --git a/postgresqleu/util/tar.py b/postgresqleu/util/tar.py
new file mode 100644 (file)
index 0000000..b5f81e0
--- /dev/null
@@ -0,0 +1,58 @@
+import io
+import tarfile
+
+
+class TarStreamer:
+    def __init__(self):
+        self.buf = io.BytesIO()
+        self.ofs = 0
+
+    def write(self, s):
+        self.buf.write(s)
+        self.ofs += len(s)
+
+    def tell(self):
+        return self.ofs
+
+    def close(self):
+        self.buf.close()
+
+    def pop(self):
+        print("Returning {} bytes at offset {}".format(len(self.buf.getvalue()), self.ofs))
+        s = self.buf.getvalue()
+        self.buf.close()
+        self.buf = io.BytesIO()
+        return s
+
+
+class BytesReader:
+    def __init__(self, buf):
+        self.buf = buf
+        self.len = len(buf)
+        self.pos = 0
+
+    def read(self, size=-1):
+        if size == -1:
+            readsize = self.len - self.pos
+        else:
+            if self.pos + size > self.len:
+                readsize = self.len - self.pos
+            else:
+                readsize = size
+
+        oldpos = self.pos
+        self.pos += readsize
+        return self.buf[oldpos:oldpos + readsize]
+
+
+def generate_streaming_tar(tar_generator):
+    streamer = TarStreamer()
+    tar = tarfile.TarFile.open(mode='w|gz', fileobj=streamer, bufsize=tarfile.BLOCKSIZE)
+    for name, mtime, data, datalen in tar_generator():
+        info = tarfile.TarInfo(name)
+        info.size = int(datalen)
+        info.mtime = mtime
+        tar.addfile(info, BytesReader(data))
+        yield streamer.pop()
+    tar.close()
+    yield streamer.pop()
index e7d0ae8982665f5b847bc0e1110b3e16bd3b7245..52cac96b17a6eb67e711f73445af049d8f30ac38 100644 (file)
@@ -289,6 +289,13 @@ This matrix gives a quick overview of the status of the different benefits for e
   <div class="col-md-3 col-sm-6 col-xs-12 buttonrow"><a class="btn btn-default btn-block" href="benefitreports/">Benefit reports</a></div>
 </div>
 
+<h2>Downloads</h2>
+<div class="row">
+{%if downloads.image %}  <div class="col-md-3 col-sm-6 col-xs-12 buttonrow"><a class="btn btn-default btn-block" href="benefitdownload/image/">Download images</a></div>{% endif %}
+{%if downloads.file %}  <div class="col-md-3 col-sm-6 col-xs-12 buttonrow"><a class="btn btn-default btn-block" href="benefitdownload/file/">Download files</a></div>{% endif %}
+</div>
+
+
 <h2>Metadata</h2>
 <div class="row">
   <div class="col-md-3 col-sm-6 col-xs-12 buttonrow"><a class="btn btn-default btn-block" href="levels/">Sponsorship levels</a></div>