From a6e057cf6011770a6b455a4da7a46f6ac7f38074 Mon Sep 17 00:00:00 2001 From: Quantum Date: Fri, 12 Apr 2024 21:06:08 -0400 Subject: [PATCH] Build basic foundations for syncing --- .gitignore | 135 ++++++++++++++++++ manage.py | 22 +++ peeringdb_api/__init__.py | 0 peeringdb_api/apps.py | 6 + peeringdb_api/client_adapter.py | 5 + peeringdb_api/management/__init__.py | 0 peeringdb_api/management/commands/__init__.py | 0 .../management/commands/sync_peeringdb.py | 13 ++ peeringdb_api/pdb_client.py | 16 +++ peeringdb_api/views.py | 3 + peeringdb_mirror/__init__.py | 0 peeringdb_mirror/asgi.py | 16 +++ peeringdb_mirror/settings/__init__.py | 0 peeringdb_mirror/settings/base.py | 103 +++++++++++++ peeringdb_mirror/settings/template.py | 22 +++ peeringdb_mirror/urls.py | 22 +++ peeringdb_mirror/wsgi.py | 16 +++ requirements.txt | 3 + 18 files changed, 382 insertions(+) create mode 100644 .gitignore create mode 100755 manage.py create mode 100644 peeringdb_api/__init__.py create mode 100644 peeringdb_api/apps.py create mode 100644 peeringdb_api/client_adapter.py create mode 100644 peeringdb_api/management/__init__.py create mode 100644 peeringdb_api/management/commands/__init__.py create mode 100644 peeringdb_api/management/commands/sync_peeringdb.py create mode 100644 peeringdb_api/pdb_client.py create mode 100644 peeringdb_api/views.py create mode 100644 peeringdb_mirror/__init__.py create mode 100644 peeringdb_mirror/asgi.py create mode 100644 peeringdb_mirror/settings/__init__.py create mode 100644 peeringdb_mirror/settings/base.py create mode 100644 peeringdb_mirror/settings/template.py create mode 100644 peeringdb_mirror/urls.py create mode 100644 peeringdb_mirror/wsgi.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6f77d8c --- /dev/null +++ b/.gitignore @@ -0,0 +1,135 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# PyCharm +.idea + +# Local configuration +/peeringdb_mirror/settings/local.py diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..417e5d5 --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'peeringdb_mirror.settings.local') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/peeringdb_api/__init__.py b/peeringdb_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peeringdb_api/apps.py b/peeringdb_api/apps.py new file mode 100644 index 0000000..9ad17f9 --- /dev/null +++ b/peeringdb_api/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class PeeringdbApiConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'peeringdb_api' diff --git a/peeringdb_api/client_adapter.py b/peeringdb_api/client_adapter.py new file mode 100644 index 0000000..f22842e --- /dev/null +++ b/peeringdb_api/client_adapter.py @@ -0,0 +1,5 @@ +from django_peeringdb.client_adaptor import backend + + +def load_backend(**kwargs): + return backend diff --git a/peeringdb_api/management/__init__.py b/peeringdb_api/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peeringdb_api/management/commands/__init__.py b/peeringdb_api/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peeringdb_api/management/commands/sync_peeringdb.py b/peeringdb_api/management/commands/sync_peeringdb.py new file mode 100644 index 0000000..4ee84f9 --- /dev/null +++ b/peeringdb_api/management/commands/sync_peeringdb.py @@ -0,0 +1,13 @@ +from django.conf import settings +from django.core.management.base import BaseCommand +from peeringdb import resource + +from peeringdb_api.pdb_client import client + + +class Command(BaseCommand): + help = 'Syncs the PeeringDB database.' + + def handle(self, *args, **options): + rs = resource.all_resources() + client.updater.update_all(rs, fetch_private=bool(settings.PEERINGDB_API_KEY)) diff --git a/peeringdb_api/pdb_client.py b/peeringdb_api/pdb_client.py new file mode 100644 index 0000000..5831089 --- /dev/null +++ b/peeringdb_api/pdb_client.py @@ -0,0 +1,16 @@ +from django.conf import settings +from peeringdb import SUPPORTED_BACKENDS +from peeringdb.client import Client + +SUPPORTED_BACKENDS['peeringdb_mirror_api'] = 'peeringdb_api.client_adapter' + +client = Client({ + 'orm': { + 'backend': 'peeringdb_mirror_api', + }, + 'sync': { + 'url': settings.PEERINGDB_API, + 'api_key': settings.PEERINGDB_API_KEY or '', + 'timeout': 0, + } +}) diff --git a/peeringdb_api/views.py b/peeringdb_api/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/peeringdb_api/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/peeringdb_mirror/__init__.py b/peeringdb_mirror/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peeringdb_mirror/asgi.py b/peeringdb_mirror/asgi.py new file mode 100644 index 0000000..525fd37 --- /dev/null +++ b/peeringdb_mirror/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for peeringdb_mirror project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'peeringdb_mirror.settings') + +application = get_asgi_application() diff --git a/peeringdb_mirror/settings/__init__.py b/peeringdb_mirror/settings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/peeringdb_mirror/settings/base.py b/peeringdb_mirror/settings/base.py new file mode 100644 index 0000000..1a1bbc8 --- /dev/null +++ b/peeringdb_mirror/settings/base.py @@ -0,0 +1,103 @@ +""" +Django settings for peeringdb_mirror project. + +Generated by 'django-admin startproject' using Django 5.0.4. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.0/ref/settings/ +""" + +from pathlib import Path + +from django.utils.translation import gettext_lazy as _ + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django_peeringdb', + 'peeringdb_api', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'peeringdb_mirror.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'peeringdb_mirror.wsgi.application' + +# Password validation +# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + {'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'}, + {'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'}, + {'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'}, + {'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'}, +] + +# Internationalization +# https://docs.djangoproject.com/en/5.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.0/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + +# django-peeringdb expected config +TABLE_PREFIX = "peeringdb_" +ABSTRACT_ONLY = False +COUNTRIES_OVERRIDE = { + "XK": _("Kosovo"), +} + +# PeeringDB configuration +PEERINGDB_API = 'https://www.peeringdb.com/api' +PEERINGDB_API_KEY = None diff --git a/peeringdb_mirror/settings/template.py b/peeringdb_mirror/settings/template.py new file mode 100644 index 0000000..ab21be2 --- /dev/null +++ b/peeringdb_mirror/settings/template.py @@ -0,0 +1,22 @@ +from .base import * + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-6)a5p^78%y9fvd)1r5troa=chey4tusp5!8t)pnda^&52)k3r(' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +ALLOWED_HOSTS = [] + +# Database +# https://docs.djangoproject.com/en/5.0/ref/settings/#databases +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + +# PeeringDB configuration +PEERINGDB_API = 'https://www.peeringdb.com/api' +PEERINGDB_API_KEY = None diff --git a/peeringdb_mirror/urls.py b/peeringdb_mirror/urls.py new file mode 100644 index 0000000..3f8d646 --- /dev/null +++ b/peeringdb_mirror/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for peeringdb_mirror project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/5.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/peeringdb_mirror/wsgi.py b/peeringdb_mirror/wsgi.py new file mode 100644 index 0000000..02e604a --- /dev/null +++ b/peeringdb_mirror/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for peeringdb_mirror project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'peeringdb_mirror.settings') + +application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e6e51e8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +Django>=5,<5.1 +peeringdb +django-peeringdb