From f065351dda0801980df32aa7a8222f68bad7f619 Mon Sep 17 00:00:00 2001 From: Quantum Date: Mon, 24 Jan 2022 19:28:00 -0500 Subject: [PATCH] Implement checking command and daemon --- qlinks/admin.py | 5 +- qlinks/management/__init__.py | 0 qlinks/management/commands/__init__.py | 0 qlinks/management/commands/qlinks_check.py | 54 ++++++++++++++++++++++ qlinks/migrations/0002_link_next_check.py | 19 ++++++++ qlinks/models.py | 21 +++++++++ qlinks/settings/base.py | 4 ++ qlinks/settings/template.py | 9 ++++ 8 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 qlinks/management/__init__.py create mode 100644 qlinks/management/commands/__init__.py create mode 100644 qlinks/management/commands/qlinks_check.py create mode 100644 qlinks/migrations/0002_link_next_check.py diff --git a/qlinks/admin.py b/qlinks/admin.py index 4b8825e..098a280 100644 --- a/qlinks/admin.py +++ b/qlinks/admin.py @@ -5,7 +5,6 @@ from django.utils import timezone from django.utils.html import format_html from django.utils.translation import gettext, gettext_lazy as _ -from qlinks.health import check_url from qlinks.models import Link @@ -32,8 +31,7 @@ class LinkAdmin(admin.ModelAdmin): def save_model(self, request, obj: Link, form, change): obj.created_by = request.user obj.updated_on = timezone.now() - obj.is_working = check_url(obj.long) - obj.last_check = timezone.now() + obj.check_url(save=False) super().save_model(request, obj, form, change) obj.purge_cdn() @@ -52,4 +50,3 @@ if settings.QLINKS_SITE_TITLE: if settings.QLINKS_INDEX_TITLE: admin.site.index_title = settings.QLINKS_INDEX_TITLE - diff --git a/qlinks/management/__init__.py b/qlinks/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qlinks/management/commands/__init__.py b/qlinks/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/qlinks/management/commands/qlinks_check.py b/qlinks/management/commands/qlinks_check.py new file mode 100644 index 0000000..b659fad --- /dev/null +++ b/qlinks/management/commands/qlinks_check.py @@ -0,0 +1,54 @@ +import logging +import time + +from django.conf import settings +from django.core.management import BaseCommand +from django.db.models import Min +from django.utils import timezone + +from qlinks.models import Link + +logger = logging.getLogger('qlinks.checker') + + +class Command(BaseCommand): + verbosity = 0 + help = 'Check QLinks for broken redirect destinations.' + + def add_arguments(self, parser): + parser.add_argument('-d', '--daemon', action='store_true', + help='run as a daemon, constantly checking links') + + def handle(self, *args, **options): + self.verbosity = int(options['verbosity']) + + try: + if options['daemon']: + self.daemon() + else: + self.check_links() + except KeyboardInterrupt: + self.stdout.write('Interrupted.') + + def daemon(self): + while True: + self.check_links() + + next_check = Link.objects.aggregate(min=Min('next_check'))['min'] + wait = (next_check - timezone.now()).total_seconds() + if wait > 0: + if self.verbosity > 2: + self.stdout.write(f'Sleeping for: {wait:.0f} seconds') + time.sleep(wait) + + def check_links(self): + for link in Link.objects.filter(next_check__lte=timezone.now()).order_by('next_check'): + if self.verbosity > 0: + self.stdout.write(f'Checking URL for {link.short}: {link.long}') + + was_working = link.is_working + link.check_url() + if was_working and not link.is_working: + self.stdout.write(f'URL for {link.short} just broke: {link.long}') + + time.sleep(settings.QLINKS_CHECK_THROTTLE) diff --git a/qlinks/migrations/0002_link_next_check.py b/qlinks/migrations/0002_link_next_check.py new file mode 100644 index 0000000..9f44b58 --- /dev/null +++ b/qlinks/migrations/0002_link_next_check.py @@ -0,0 +1,19 @@ +# Generated by Django 4.0.1 on 2022-01-25 00:13 + +from django.db import migrations, models +import qlinks.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('qlinks', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='link', + name='next_check', + field=models.DateTimeField(db_index=True, default=qlinks.models.compute_next_check, verbose_name='the next time the URL will be checked'), + ), + ] diff --git a/qlinks/models.py b/qlinks/models.py index f06a3f6..7f3c8f5 100644 --- a/qlinks/models.py +++ b/qlinks/models.py @@ -1,3 +1,5 @@ +import random +from datetime import timedelta from functools import cached_property from django.conf import settings @@ -6,6 +8,13 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from qlinks.cdn_cache import cdn_cache +from qlinks.health import check_url + + +def compute_next_check(): + min_check = settings.QLINKS_CHECK_MIN.total_seconds() + max_check = settings.QLINKS_CHECK_MAX.total_seconds() + return timezone.now() + timedelta(seconds=random.randint(min_check, max_check)) class Link(models.Model): @@ -19,6 +28,8 @@ class Link(models.Model): verbose_name=_('created by')) is_working = models.BooleanField(verbose_name=_('URL is working')) last_check = models.DateTimeField(verbose_name=_('last time URL was checked')) + next_check = models.DateTimeField(verbose_name=_('the next time the URL will be checked'), + default=compute_next_check, db_index=True) def __str__(self): return self.short or '/' @@ -27,6 +38,16 @@ class Link(models.Model): def short_url(self): return settings.QLINKS_CANONICAL + self.short + def check_url(self, save=True): + self.is_working = check_url(self.long) + self.last_check = timezone.now() + self.next_check = compute_next_check() + + if save: + self.save() + check_url.alters_data = True + def purge_cdn(self): if cdn_cache: cdn_cache.purge(self.short_url) + purge_cdn.alters_data = True diff --git a/qlinks/settings/base.py b/qlinks/settings/base.py index 09dd77c..5758b68 100644 --- a/qlinks/settings/base.py +++ b/qlinks/settings/base.py @@ -1,3 +1,4 @@ +from datetime import timedelta from pathlib import Path BASE_DIR = Path(__file__).resolve().parent.parent.parent @@ -87,3 +88,6 @@ QLINKS_SITE_HEADER = 'QLinks Admin' QLINKS_SITE_TITLE = 'QLinks Admin' QLINKS_INDEX_TITLE = 'Welcome to the QLinks admin interface!' QLINKS_CDN_CACHE = None +QLINKS_CHECK_MIN = timedelta(days=6) +QLINKS_CHECK_MAX = timedelta(days=8) +QLINKS_CHECK_THROTTLE = 1 diff --git a/qlinks/settings/template.py b/qlinks/settings/template.py index dd02744..2a230b4 100644 --- a/qlinks/settings/template.py +++ b/qlinks/settings/template.py @@ -33,3 +33,12 @@ QLINKS_CANONICAL = None # QLINKS_CDN_CACHE = 'qlinks.cdn_cache.cloudflare_cache.CloudflareCDNCache' # QLINKS_CDN_CLOUDFLARE_API_TOKEN = ... # QLINKS_CDN_CLOUDFLARE_API_ZONE_ID = ... + +# Automatic link checking settings. +# Minimum and maximum time before next check. +# A time is randomly chosen to spread out the load. +# QLINKS_CHECK_MIN = timedelta(days=6) +# QLINKS_CHECK_MAX = timedelta(days=8) + +# Minimum time in seconds between two consecutive checks. +# QLINKS_CHECK_THROTTLE = 1