From b713c39b6aeca0517580638080de3e59d44d0805 Mon Sep 17 00:00:00 2001 From: xenua Date: Sun, 19 Jun 2022 16:42:22 +0200 Subject: [PATCH] remove django_scopes --- leftists/middleware.py | 19 ++++++++++ leftists/migrations/0001_initial.py | 14 +++++-- leftists/models.py | 57 ++++++++++++++++++++++++++--- leftists/views.py | 33 +++++++---------- lonk/settings.py | 5 +-- requirements.txt | 2 - tests/performance.py | 39 ++++++++++++++++++++ 7 files changed, 134 insertions(+), 35 deletions(-) create mode 100644 leftists/middleware.py create mode 100644 tests/performance.py diff --git a/leftists/middleware.py b/leftists/middleware.py new file mode 100644 index 0000000..3b556f3 --- /dev/null +++ b/leftists/middleware.py @@ -0,0 +1,19 @@ +from django.http import Http404 +from django.utils.deprecation import MiddlewareMixin + +from leftists.models import Domain + + +class DomainAutoCreateMiddleware(MiddlewareMixin): + def __init__(self, get_response): + super().__init__(get_response) + self.cache = set() + + def process_request(self, r): + if (host := r.get_host()) in self.cache: + return + try: + Domain.get_from_request(r) + self.cache.add(host) + except Domain.DoesNotExist: + Domain.objects.create(fqdn=host.lower()) diff --git a/leftists/migrations/0001_initial.py b/leftists/migrations/0001_initial.py index efda774..30ae6d5 100644 --- a/leftists/migrations/0001_initial.py +++ b/leftists/migrations/0001_initial.py @@ -1,7 +1,7 @@ -# Generated by Django 4.0.5 on 2022-06-18 23:16 +# Generated by Django 4.0.5 on 2022-06-19 14:05 -import django.db.models.deletion from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): @@ -9,10 +9,16 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('scopedsites', '0001_initial'), ] operations = [ + migrations.CreateModel( + name='Domain', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('fqdn', models.CharField(max_length=512, unique=True, verbose_name='FQDN')), + ], + ), migrations.CreateModel( name='ShortLink', fields=[ @@ -20,7 +26,7 @@ class Migration(migrations.Migration): ('location', models.CharField(max_length=100, verbose_name='short link')), ('to', models.URLField(max_length=2500, verbose_name='redirect to')), ('click_count', models.IntegerField(default=0)), - ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='scopedsites.domain')), + ('domain', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='links', to='leftists.domain')), ], options={ 'unique_together': {('domain', 'location')}, diff --git a/leftists/models.py b/leftists/models.py index 7a8a7a5..1037a00 100644 --- a/leftists/models.py +++ b/leftists/models.py @@ -1,11 +1,23 @@ -from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ -from django_scopes import ScopedManager -from scopedsites.models import Domain from xenua.django.models import RandomSlugPKMixin +class Domain(models.Model): + fqdn = models.CharField( + _('FQDN'), + max_length=512, + unique=True, + ) + + def __str__(self): + return self.fqdn + + @classmethod + def get_from_request(cls, r): + return cls.objects.get(fqdn__iexact=r.get_host()) + + class ShortLink(RandomSlugPKMixin, models.Model): class Meta: unique_together = ('domain', 'location') @@ -15,11 +27,44 @@ class ShortLink(RandomSlugPKMixin, models.Model): to = models.URLField(_("redirect to"), max_length=2500) click_count = models.IntegerField(default=0) - objects = ScopedManager(domain='domain') + cache = {} - def click(self): # todo: replace with cache + regular cleanup impl + def click(self): # todo: further assess performance. initial testing suggests minimal impact self.click_count += 1 self.save() def link(self): - return f"https://{self.domain.fqdn}/{self.location}" + return f"{self.domain.fqdn}/{self.location}" + + @classmethod + def get_from_request(cls, req): + d = Domain.get_from_request(req) + return cls.objects.filter(domain=d).get(location=req.path[1:]) + + @classmethod + def hit(cls, req, loc): + if r := cls.try_cache(req, loc): + return r + + return cls.miss(req, loc) + + @classmethod + def miss(cls, req, loc): + lnk = cls.get_from_request(req) + cls.cache[req.get_host()][loc] = lnk.to + return lnk.to + + @classmethod + def try_cache(cls, req, loc): + host = req.get_host() + try: + domaincache = cls.cache[host] + try: + return domaincache[loc] + except KeyError: + cls.miss(req, loc) + except KeyError: + cls.cache[host] = {} + + return '' + diff --git a/leftists/views.py b/leftists/views.py index 366f459..08ddd68 100644 --- a/leftists/views.py +++ b/leftists/views.py @@ -4,18 +4,14 @@ from django.shortcuts import get_object_or_404 from django.urls import reverse_lazy from django.views.generic import (CreateView, DeleteView, ListView, RedirectView, TemplateView, UpdateView) -from django_scopes import scope, scopes_disabled -from scopedsites.models import Domain from leftists.forms import LinkForm -from leftists.models import ShortLink +from leftists.models import ShortLink, Domain class ShortLinkRedirectView(RedirectView): def get_redirect_url(self, *args, **kwargs): - sl = get_object_or_404(ShortLink, location=kwargs.get('link')) - sl.click() - return sl.to + return ShortLink.hit(self.request, kwargs.get("link")) class CoolerLoginView(LoginView): @@ -27,26 +23,23 @@ class OverView(LoginRequiredMixin, TemplateView): template_name = 'interface/overview.html' def get_context_data(self, **kwargs): - with scopes_disabled(): - ctx = super().get_context_data(**kwargs) - ds = Domain.objects.all() - ctx.setdefault('domains', ds) - links = [] - for d in ds: - [links.append(l) for l in d.links.all()] - ctx.setdefault('links', links) - return ctx + ctx = super().get_context_data(**kwargs) + ds = Domain.objects.all() + ctx.setdefault('domains', ds) + links = [] + for d in ds: + [links.append(l) for l in d.links.all()] + ctx.setdefault('links', links) + return ctx class LinkListView(LoginRequiredMixin, ListView): template_name = 'interface/linkedlist.html' model = ShortLink - def get(self, request, *args, **kwargs): - with scopes_disabled(): - d = Domain.objects.get(fqdn__contains=kwargs.get('domain')) - with scope(domain=d): - return super().get(request, *args, **kwargs) + def get_queryset(self): + d = Domain.get_from_request(self.request) + return ShortLink.objects.filter(domain=d) class LinkCreateView(LoginRequiredMixin, CreateView): diff --git a/lonk/settings.py b/lonk/settings.py index 894ecd8..5b5e27a 100644 --- a/lonk/settings.py +++ b/lonk/settings.py @@ -36,7 +36,6 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'scopedsites', 'leftists', ] @@ -48,8 +47,8 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'scopedsites.middleware.DomainAutoCreateMiddleware', - 'scopedsites.middleware.DomainScopeMiddleware', + 'leftists.middleware.DomainAutoCreateMiddleware', + # 'django_cprofile_middleware.middleware.ProfilerMiddleware', ] ROOT_URLCONF = 'lonk.urls' diff --git a/requirements.txt b/requirements.txt index f0233ac..168ba7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,2 @@ django -django_scopes -git+https://git.xenua.me/xenua/django_scopedsites.git@main#egg=django_scopedsites git+https://git.xenua.me/xenua/xenua_tools.git@main#egg=xenua_tools \ No newline at end of file diff --git a/tests/performance.py b/tests/performance.py new file mode 100644 index 0000000..b1eccc8 --- /dev/null +++ b/tests/performance.py @@ -0,0 +1,39 @@ +import asyncio +import sys +import time + +import aiohttp +import uvloop + + +async def do_req(sesh, url): + async with sesh.get(url, allow_redirects=False) as resp: + assert resp.status == 302 + + +async def main(): + if len(sys.argv) < 2: + exit("usage: python performance.py ") + url = sys.argv[1] + async with aiohttp.ClientSession() as sesh: + await do_req(sesh, url) + + await asyncio.sleep(1) + + tasks = [] + for i in range(10000): + tasks.append(do_req(sesh, url)) + + # aaand liftoff! + before = time.time() + await asyncio.gather(*tasks) + after = time.time() + + return after - before + +uvloop.install() +t = asyncio.run(main()) + +print(f"completed 10k requests in {t:.4f} seconds") +if t > 10: + print('if that was a redirect endpoint something seems off here. go optimize performance')