Implement checking command and daemon

qlinks/admin.py

@@ -5,7 +5,6 @@ from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.translation import gettext, gettext_lazy as _
 
-from qlinks.health import check_url
 from qlinks.models import Link
 
 
@@ -32,8 +31,7 @@ class LinkAdmin(admin.ModelAdmin):
     def save_model(self, request, obj: Link, form, change):
         obj.created_by = request.user
         obj.updated_on = timezone.now()
-        obj.is_working = check_url(obj.long)
+        obj.check_url(save=False)
-        obj.last_check = timezone.now()
         super().save_model(request, obj, form, change)
         obj.purge_cdn()
 
@@ -52,4 +50,3 @@ if settings.QLINKS_SITE_TITLE:
 
 if settings.QLINKS_INDEX_TITLE:
     admin.site.index_title = settings.QLINKS_INDEX_TITLE
-

qlinks/management/commands/qlinks_check.py

@@ -0,0 +1,54 @@
+import logging
+import time
+
+from django.conf import settings
+from django.core.management import BaseCommand
+from django.db.models import Min
+from django.utils import timezone
+
+from qlinks.models import Link
+
+logger = logging.getLogger('qlinks.checker')
+
+
+class Command(BaseCommand):
+    verbosity = 0
+    help = 'Check QLinks for broken redirect destinations.'
+
+    def add_arguments(self, parser):
+        parser.add_argument('-d', '--daemon', action='store_true',
+                            help='run as a daemon, constantly checking links')
+
+    def handle(self, *args, **options):
+        self.verbosity = int(options['verbosity'])
+
+        try:
+            if options['daemon']:
+                self.daemon()
+            else:
+                self.check_links()
+        except KeyboardInterrupt:
+            self.stdout.write('Interrupted.')
+
+    def daemon(self):
+        while True:
+            self.check_links()
+
+            next_check = Link.objects.aggregate(min=Min('next_check'))['min']
+            wait = (next_check - timezone.now()).total_seconds()
+            if wait > 0:
+                if self.verbosity > 2:
+                    self.stdout.write(f'Sleeping for: {wait:.0f} seconds')
+                time.sleep(wait)
+
+    def check_links(self):
+        for link in Link.objects.filter(next_check__lte=timezone.now()).order_by('next_check'):
+            if self.verbosity > 0:
+                self.stdout.write(f'Checking URL for {link.short}: {link.long}')
+
+            was_working = link.is_working
+            link.check_url()
+            if was_working and not link.is_working:
+                self.stdout.write(f'URL for {link.short} just broke: {link.long}')
+
+            time.sleep(settings.QLINKS_CHECK_THROTTLE)

qlinks/migrations/0002_link_next_check.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.0.1 on 2022-01-25 00:13
+
+from django.db import migrations, models
+import qlinks.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('qlinks', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='link',
+            name='next_check',
+            field=models.DateTimeField(db_index=True, default=qlinks.models.compute_next_check, verbose_name='the next time the URL will be checked'),
+        ),
+    ]

qlinks/models.py

@@ -1,3 +1,5 @@
+import random
+from datetime import timedelta
 from functools import cached_property
 
 from django.conf import settings
@@ -6,6 +8,13 @@ from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 from qlinks.cdn_cache import cdn_cache
+from qlinks.health import check_url
+
+
+def compute_next_check():
+    min_check = settings.QLINKS_CHECK_MIN.total_seconds()
+    max_check = settings.QLINKS_CHECK_MAX.total_seconds()
+    return timezone.now() + timedelta(seconds=random.randint(min_check, max_check))
 
 
 class Link(models.Model):
@@ -19,6 +28,8 @@ class Link(models.Model):
                                    verbose_name=_('created by'))
     is_working = models.BooleanField(verbose_name=_('URL is working'))
     last_check = models.DateTimeField(verbose_name=_('last time URL was checked'))
+    next_check = models.DateTimeField(verbose_name=_('the next time the URL will be checked'),
+                                      default=compute_next_check, db_index=True)
 
     def __str__(self):
         return self.short or '/'
@@ -27,6 +38,16 @@ class Link(models.Model):
     def short_url(self):
         return settings.QLINKS_CANONICAL + self.short
 
+    def check_url(self, save=True):
+        self.is_working = check_url(self.long)
+        self.last_check = timezone.now()
+        self.next_check = compute_next_check()
+
+        if save:
+            self.save()
+    check_url.alters_data = True
+
     def purge_cdn(self):
         if cdn_cache:
             cdn_cache.purge(self.short_url)
+    purge_cdn.alters_data = True

qlinks/settings/base.py

@@ -1,3 +1,4 @@
+from datetime import timedelta
 from pathlib import Path
 
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -87,3 +88,6 @@ QLINKS_SITE_HEADER = 'QLinks Admin'
 QLINKS_SITE_TITLE = 'QLinks Admin'
 QLINKS_INDEX_TITLE = 'Welcome to the QLinks admin interface!'
 QLINKS_CDN_CACHE = None
+QLINKS_CHECK_MIN = timedelta(days=6)
+QLINKS_CHECK_MAX = timedelta(days=8)
+QLINKS_CHECK_THROTTLE = 1

qlinks/settings/template.py

@@ -33,3 +33,12 @@ QLINKS_CANONICAL = None
 # QLINKS_CDN_CACHE = 'qlinks.cdn_cache.cloudflare_cache.CloudflareCDNCache'
 # QLINKS_CDN_CLOUDFLARE_API_TOKEN = ...
 # QLINKS_CDN_CLOUDFLARE_API_ZONE_ID = ...
+
+# Automatic link checking settings.
+# Minimum and maximum time before next check.
+# A time is randomly chosen to spread out the load.
+# QLINKS_CHECK_MIN = timedelta(days=6)
+# QLINKS_CHECK_MAX = timedelta(days=8)
+
+# Minimum time in seconds between two consecutive checks.
+# QLINKS_CHECK_THROTTLE = 1