Update to Django 4

Signed-off-by: Alfredos-Panagiotis Damkalis <fredy@fredy.gr>

.gitlab-ci.yml

@@ -2,7 +2,7 @@ variables:
   GITLAB_CI_IMAGE_ALPINE: 'alpine:3.9'
   GITLAB_CI_IMAGE_DOCKER: 'docker:20.10.6'
   GITLAB_CI_IMAGE_NODE: 'node:13.12'
-  GITLAB_CI_IMAGE_PYTHON: 'python:3.8.6'
+  GITLAB_CI_IMAGE_PYTHON: 'python:3.9.12'
   GITLAB_CI_IMAGE_OPENAPI_GENERATOR_CLI: 'openapitools/openapi-generator-cli:v5.3.0'
   GITLAB_CI_IMAGE_SENTRY_CLI: 'getsentry/sentry-cli'
   GITLAB_CI_PYPI_DOCKER_COMPOSE: 'docker-compose~=1.23.0'

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8.7
+FROM python:3.9.12
 LABEL maintainer="SatNOGS project <dev@satnogs.org>"
 
 WORKDIR /workdir/

db/api/urls.py

@@ -1,6 +1,5 @@
 """SatNOGS DB django rest framework API url routings"""
-from django.conf.urls import include
+from django.urls import include, path
-from django.urls import path
 from rest_framework import routers
 
 from db.api import views

db/base/__init__.py

@@ -1,2 +0,0 @@
-"""SatNOGS DB Base app initialization"""
-default_app_config = 'db.base.apps.BaseConfig'  # pylint: disable=C0103

db/base/forms.py

@@ -57,7 +57,7 @@ class TransmitterUpdateForm(BSModalModelForm):  # pylint: disable=too-many-ances
         }
 
 
-class SatelliteCreateForm(BSModalModelForm):
+class SatelliteCreateForm(BSModalModelForm):  # pylint: disable=too-many-ancestors
     """Form that uses django-bootstrap-modal-forms for satellite editing"""
     class Meta:
         model = SatelliteEntry
@@ -81,7 +81,7 @@ class SatelliteCreateForm(BSModalModelForm):
         widgets = {'names': TextInput()}
 
 
-class SatelliteUpdateForm(BSModalModelForm):
+class SatelliteUpdateForm(BSModalModelForm):  # pylint: disable=too-many-ancestors
     """Form that uses django-bootstrap-modal-forms for satellite editing"""
     class Meta:
         model = SatelliteEntry

db/base/views.py

@@ -421,7 +421,8 @@ class TransmitterCreateView(LoginRequiredMixin, BSModalCreateView):
         transmitter.created = now()
         transmitter.created_by = self.user
         # Prevents sending notification twice as form_valid is triggered for validation and saving
-        if not self.request.is_ajax():
+        # Check if request is an AJAX one
+        if not self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
             notify_suggestion.delay(
                 transmitter.satellite.satellite_entry.id, self.user.id, 'transmitter'
             )
@@ -455,7 +456,8 @@ class TransmitterUpdateView(LoginRequiredMixin, BSModalUpdateView):
         transmitter.created = now()
         transmitter.created_by = self.user
         # Prevents sending notification twice as form_valid is triggered for validation and saving
-        if not self.request.is_ajax():
+        # Check if request is an AJAX one
+        if not self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
             notify_suggestion.delay(
                 transmitter.satellite.satellite_entry.id, self.user.id, 'transmitter'
             )
@@ -481,7 +483,8 @@ class MergeSatellitesView(LoginRequiredMixin, BSModalFormView):
         response = super().form_valid(form)
 
         if self.user.has_perm('base.merge_satellites'):
-            if not self.request.is_ajax():
+            # Check if request is an AJAX one
+            if not self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
                 primary_satellite = form.cleaned_data['primary_satellite']
                 associated_satellite = form.cleaned_data['associated_satellite']
                 associated_satellite.associated_satellite = primary_satellite
@@ -516,7 +519,8 @@ class SatelliteCreateView(LoginRequiredMixin, BSModalCreateView):
         satellite_obj = None
         # Create Satellite Identifier only when POST request is for saving and
         # NORAD ID is not used by other Satellite.
-        if not self.request.is_ajax():
+        # Check if request is an AJAX one
+        if not self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
             try:
                 # If the form doesn't contain NORAD ID, create a new satellite
                 if satellite_entry.norad_cat_id:
@@ -539,7 +543,8 @@ class SatelliteCreateView(LoginRequiredMixin, BSModalCreateView):
         # Prevents sending notification twice as form_valid is triggered for
         # validation and saving. Also create and Satellite object only when POST
         # request is for saving and NORAD ID is not used by other Satellite.
-        if not self.request.is_ajax():
+        # Check if request is an AJAX one
+        if not self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
             if not satellite_obj:
                 satellite_obj = Satellite.objects.create(
                     satellite_identifier=satellite_entry.satellite_identifier,
@@ -578,7 +583,8 @@ class SatelliteUpdateView(LoginRequiredMixin, BSModalUpdateView):
         satellite_entry.created = now()
         satellite_entry.created_by = self.user
         # Prevents sending notification twice as form_valid is triggered for validation and saving
-        if not self.request.is_ajax():
+        # Check if request is an AJAX one
+        if not self.request.headers.get('x-requested-with') == 'XMLHttpRequest':
             notify_suggestion.delay(initial_satellite_entry_pk, self.user.id, 'satellite')
         return super().form_valid(form)
 

db/settings.py

@@ -4,13 +4,14 @@ For local installation settings please copy .env-dist to .env and edit
 the appropriate settings in that file. You should not need to edit this
 file for local settings!
 """
+from pathlib import Path
+
 import sentry_sdk
 from decouple import Csv, config
 from dj_database_url import parse as db_url
 from sentry_sdk.integrations.celery import CeleryIntegration
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
-from unipath import Path
 
 from db import __version__
 
@@ -38,7 +39,7 @@ THIRD_PARTY_APPS = (
     'drf_spectacular',
     'django_countries',
     'django_filters',
-    'fontawesome_5',
+    'fontawesomefree',
     'widget_tweaks',
     'allauth',
     'allauth.account',
@@ -100,16 +101,17 @@ SERVER_EMAIL = DEFAULT_FROM_EMAIL
 CACHES = {
     'default': {
         'BACKEND': config(
-            'CACHE_BACKEND', default='django.core.cache.backends.locmem.LocMemCache'
+            'CACHE_BACKEND',
+            default='django.core.cache.backends.locmem.LocMemCache',
         ),
         'LOCATION': config('CACHE_LOCATION', default='unique-location'),
-        'OPTIONS': {
-            'MAX_ENTRIES': 5000,
-            'CLIENT_CLASS': config('CACHE_CLIENT_CLASS', default=''),
-        },
         'KEY_PREFIX': 'db-{0}'.format(ENVIRONMENT),
     }
 }
+
+if CACHES['default']['BACKEND'] == 'django.core.cache.backends.locmem.LocMemCache':
+    CACHES['default']['OPTIONS'] = {'MAX_ENTRIES': 5000}
+
 CACHE_TTL = config('CACHE_TTL', default=300, cast=int)
 
 # Internationalization
@@ -125,7 +127,7 @@ TEMPLATES = [
     {
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [
-            Path(ROOT).child('templates').resolve(),
+            Path(ROOT).joinpath('templates').resolve(),
         ],
         'OPTIONS': {
             'context_processors': [
@@ -158,7 +160,7 @@ TEMPLATES = [
 STATIC_ROOT = config('STATIC_ROOT', default=Path('staticfiles').resolve())
 STATIC_URL = config('STATIC_URL', default='/static/')
 STATICFILES_DIRS = [
-    Path(ROOT).child('static').resolve(),
+    Path(ROOT).joinpath('static').resolve(),
 ]
 STATICFILES_FINDERS = (
     'django.contrib.staticfiles.finders.FileSystemFinder',
@@ -178,12 +180,14 @@ COMPRESS_FILTERS = {
     'css': [
         'compressor.filters.css_default.CssAbsoluteFilter',
         'compressor.filters.cssmin.rCSSMinFilter'
-    ]
+    ],
+    'js': ['compressor.filters.jsmin.JSMinFilter']
 }
 
 # App conf
 ROOT_URLCONF = 'db.urls'
 WSGI_APPLICATION = 'db.wsgi.application'
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
 # Auth
 AUTHENTICATION_BACKENDS = ('django.contrib.auth.backends.ModelBackend', )

db/templates/base.html

@@ -2,7 +2,6 @@
 {% load avatar_tags %}
 {% load tags %}
 {% load compress %}
-{% load fontawesome_5 %}
 
 <!DOCTYPE html>
 <html lang="en" ng-app>
@@ -15,8 +14,8 @@
   {% block css %}{% endblock %}
   <link rel="stylesheet" href="{% static 'lib/admin-lte/dist/css/adminlte.min.css' %}">
   <link rel="stylesheet" href="{% static 'css/app.css' %}">
+  <link href="{% static 'fontawesomefree/css/all.min.css' %}" rel="stylesheet" type="text/css">
   {% endcompress %}
-  {% fontawesome_5_static %}
 
   <link rel="shortcut icon" href="{% static 'favicon.ico' %}">
 
@@ -233,6 +232,7 @@
   <!-- /.wrapper -->
 
   {% compress js %}
+  <script src="{% static 'fontawesomefree/js/all.min.js' %}"></script>
   <script src="{% static 'lib/admin-lte/plugins/jquery/jquery.min.js' %}"></script>
   <script src="{% static 'lib/admin-lte/plugins/popper/umd/popper.min.js' %}"></script>
   <script src="{% static 'lib/admin-lte/plugins/bootstrap/js/bootstrap.min.js' %}"></script>

db/urls.py

@@ -1,9 +1,8 @@
 """ Base Django URL mapping for SatNOGS DB"""
 from allauth import urls as allauth_urls
 from django.conf import settings
-from django.conf.urls import include
 from django.contrib import admin
-from django.urls import path
+from django.urls import include, path
 from django.views.static import serve
 from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerSplitView
 

docker-compose.yml

@@ -30,8 +30,7 @@ services:
       DATABASE_URL: 'mysql://satnogsdb:satnogsdb@db/satnogsdb'
       CELERY_BROKER_URL: 'redis://redis:6379/0'
       CELERY_RESULT_BACKEND: 'redis://redis:6379/0'
-      CACHE_BACKEND: 'redis_cache.RedisCache'
+      CACHE_BACKEND: 'django.core.cache.backends.redis.RedisCache'
-      CACHE_CLIENT_CLASS: 'django_redis.client.DefaultClient'
       CACHE_LOCATION: 'redis://redis:6379/1'
       MEDIA_ROOT: '/var/lib/satnogs-db/media'
     command: ["djangoctl.sh", "develop_celery", "/usr/local/src/satnogs-db"]
@@ -56,8 +55,7 @@ services:
       DATABASE_URL: 'mysql://satnogsdb:satnogsdb@db/satnogsdb'
       CELERY_BROKER_URL: 'redis://redis:6379/0'
       CELERY_RESULT_BACKEND: 'redis://redis:6379/0'
-      CACHE_BACKEND: 'redis_cache.RedisCache'
+      CACHE_BACKEND: 'django.core.cache.backends.redis.RedisCache'
-      CACHE_CLIENT_CLASS: 'django_redis.client.DefaultClient'
       CACHE_LOCATION: 'redis://redis:6379/1'
       STATIC_ROOT: '/var/lib/satnogs-db/staticfiles'
       MEDIA_ROOT: '/var/lib/satnogs-db/media'

requirements-dev.txt

@@ -10,24 +10,20 @@ docopt==0.6.1
 docopts==0.6.1
 execnet==1.9.0
 factory-boy==3.2.1
-Faker==8.1.4
+Faker==13.3.4
 filelock==3.6.0
 iniconfig==1.1.1
 mock==4.0.3
-packaging==21.3
 platformdirs==2.5.1
 pluggy==1.0.0
-pur==5.4.2
 py==1.11.0
-pyparsing==3.0.7
 pytest==7.1.1
 pytest-celery==0.0.0
-pytest-cov==2.12.1
+pytest-cov==3.0.0
-pytest-django==4.2.0
+pytest-django==4.5.2
-pytest-forked==1.3.0
+pytest-forked==1.4.0
-pytest-xdist==2.2.1
+pytest-xdist==2.5.0
-text-unidecode==1.3
 toml==0.10.2
 tomli==2.0.1
-tox==3.23.1
+tox==3.24.5
-virtualenv==20.14.0
+virtualenv==20.14.1

requirements.txt

@@ -5,48 +5,48 @@
 
 amqp==5.1.0
 asgiref==3.5.0
+async-timeout==4.0.2
 attrs==21.4.0
 billiard==3.6.4.0
 cachetools==5.0.0
-celery==5.0.5
+celery==5.2.6
 certifi==2021.10.8
 cffi==1.15.0
-chardet==4.0.0
+charset-normalizer==2.0.12
-click==7.1.2
+click==8.1.2
 click-didyoumean==0.3.0
 click-plugins==1.1.1
 click-repl==0.2.0
 cryptography==36.0.2
 defusedxml==0.7.1
+Deprecated==1.2.13
 dj-database-url==0.5.0
-Django==3.2.12
+Django==4.0.4
 django-allauth==0.50.0
 django-appconf==1.0.5
-django-avatar==5.0.0
+django-avatar @ git+https://github.com/ntoll/django-avatar@68340e8ac603401cfbf05b71bd3f6ad6232fe66c
 django-bootstrap-modal-forms==2.2.0
-django-compressor==2.4.1
+django-compressor @ git+https://github.com/django-compressor/django-compressor.git@00910e0bd5295044befacdad7b88249ec7bbf142
-django-cors-headers==3.10.1
+django-cors-headers==3.11.0
-django-countries==7.2.1
+django-countries==7.3.2
-django-crispy-forms==1.11.2
+django-crispy-forms==1.14.0
 django-csp==3.7
 django-debug-toolbar==3.2.4
 django-filter==21.1
-django-fontawesome-5==1.0.18
-django-jsonfield==1.4.1
-django-redis-cache==3.0.1
 django-shortuuidfield==0.1.3
 django-widget-tweaks==1.4.12
 djangorestframework==3.13.1
 dnspython==1.16.0
 drf-spectacular==0.22.0
-ecdsa==0.14.1
 enum34==1.1.10
-eventlet==0.29.1
+eventlet==0.30.2
-frozendict==2.3.0
+fontawesomefree==6.1.1
+frozendict==2.3.1
 greenlet==1.1.2
 gunicorn==19.9.0
-h5py==3.2.1
+h5py==3.6.0
-idna==2.10
+hiredis==2.0.0
+idna==3.3
 importlib-metadata==4.11.3
 inflection==0.5.1
 influxdb==5.3.1
@@ -57,37 +57,36 @@ Logbook==1.5.3
 lxml==4.8.0
 Markdown==3.3.6
 msgpack==1.0.3
-mysqlclient==2.0.3
+mysqlclient==2.1.0
 nanoid==2.0.0
 numpy==1.22.3
 oauthlib==3.2.0
-Pillow==9.0.1
+packaging==21.3
-prompt-toolkit==3.0.28
+Pillow==9.1.0
-pyasn1==0.4.8
+prompt-toolkit==3.0.29
 pycparser==2.21
 PyJWT==2.3.0
 PyLD==2.0.3
+pyparsing==3.0.8
 pyrsistent==0.18.1
 python-dateutil==2.8.2
-python-decouple==3.4
+python-decouple==3.6
-python-dotenv==0.17.1
+python-dotenv==0.20.0
-python-jose==3.2.0
 python3-openid==3.2.0
 pytz==2022.1
-PyYAML==5.4.1
+PyYAML==6.0
-pyzmq==22.0.3
+pyzmq==22.3.0
-rcssmin==1.0.6
+rcssmin==1.1.0
-redis==3.5.3
+redis==4.2.2
 Represent==1.6.0.post0
-requests==2.25.1
+requests==2.27.1
 requests-oauthlib==1.3.1
-rjsmin==1.1.0
+rjsmin==1.2.0
-rsa==4.8
 rush==2021.4.0
-satellitetle==0.11.1
+satellitetle==0.12.0
 satnogs-decoders~=1.0
-sentry-sdk==1.1.0
+sentry-sdk==1.5.10
-sgp4==2.19
+sgp4==2.21
 shortuuid==1.0.8
 simplejson==3.17.6
 six==1.16.0
@@ -95,9 +94,10 @@ social-auth-app-django==4.0.0
 social-auth-core==4.2.0
 spacetrack==0.16.0
 sqlparse==0.4.2
-Unipath==1.1
+typing_extensions==4.1.1
-uritemplate==3.0.1
+uritemplate==4.1.1
 urllib3==1.26.9
 vine==5.0.0
 wcwidth==0.2.5
-zipp==3.7.0
+wrapt==1.14.0
+zipp==3.8.0

setup.cfg

@@ -24,78 +24,73 @@ packages = find:
 include_package_data = True
 install_requires =
     # Basic
-    Django~=3.2.0
+    Django~=4.0.0
     django-shortuuidfield~=0.1.0
-    django-jsonfield~=1.4.0
+    celery~=5.2.0
-    celery~=5.0.0
     # Deployment
-    mysqlclient~=2.0.0
+    mysqlclient~=2.1.0
-    # Cache
+    # pinning for https://github.com/benoitc/gunicorn/pull/2581
-    django-redis-cache~=3.0.0
+    eventlet==0.30.2
+    gunicorn[eventlet]~=19.9.0
+    # Cache https://docs.djangoproject.com/en/4.0/topics/cache/#redis
+    redis~=4.2.0
+    hiredis~=2.0.0
     # Logging
-    sentry-sdk~=1.1.0
+    sentry-sdk~=1.5.0
     # Configuration
-    python-decouple~=3.4.0
+    python-decouple~=3.6.0
     dj-database-url~=0.5.0
-    pytz
+    python-dotenv~=0.20.0
-    Unipath~=1.1
-    python-dotenv~=0.17.0
     # Security
-    django_csp~=3.7.0
+    django-csp~=3.7.0
-    django-cors-headers~=3.10.0
+    django-cors-headers~=3.11.0
     # Users
     django-allauth~=0.50.0
-    django-avatar~=5.0.0
+    # pinning until https://github.com/grantmcconnaughey/django-avatar/pull/204 is merged
-    django-crispy-forms~=1.11.0
+    django-avatar @ git+https://github.com/ntoll/django-avatar@68340e8ac603401cfbf05b71bd3f6ad6232fe66c
-    python-jose[cryptography]~=3.2.0
+    django-crispy-forms~=1.14.0
     social-auth-app-django~=4.0.0
     # Static
-    django_compressor~=2.4.0
+    # pinning until https://github.com/django-compressor/django-compressor/pull/1107 is released
+    django_compressor @ git+https://github.com/django-compressor/django-compressor.git@00910e0bd5295044befacdad7b88249ec7bbf142
     # API
     djangorestframework~=3.13.0
     drf-spectacular~=0.22.0
     Markdown~=3.3.0
     django-filter~=21.1
     # Astronomy
-    sgp4~=2.19.0
+    sgp4~=2.21.0
-    satellitetle~=0.11.0
+    satellitetle~=0.12.0
     # Unsorted
     influxdb~=5.3.0
     django-widget-tweaks~=1.4.8
     django-bootstrap-modal-forms~=2.2.0
-    django-fontawesome-5
+    fontawesomefree~=6.1.1
     satnogs-decoders~=1.0
     simplejson~=3.17.0
-    uritemplate~=3.0.0
+    h5py~=3.6.0
-    PyYAML~=5.4.0
+    pyzmq~=22.3.0
-    h5py~=3.2.0
-    PyLD~=2.0.3
-    pyzmq~=22.0.0
     nanoid~=2.0.0
-    urllib3~=1.26
+    urllib3~=1.26.0
-    requests~=2.25.0
+    requests~=2.27.0
     # Metasat
-    django-countries~=7.2.0
+    django-countries~=7.3.0
+    PyLD~=2.0.3
     # Debugging
     django-debug-toolbar~=3.2.0
-    # pinning for https://github.com/benoitc/gunicorn/pull/2581
-    eventlet==0.29.1
-    gunicorn[eventlet]==19.9.0
 
 
 [options.extras_require]
 dev =
-    pytest-celery
+    pytest-celery # https://docs.celeryq.dev/en/stable/userguide/testing.html?highlight=pytest-celery#enabling
-    pytest-cov~=2.12.0
+    pytest-cov~=3.0.0
-    pytest-django~=4.2.0
+    pytest-django~=4.5.0
-    pytest-forked~=1.3.0
+    pytest-forked~=1.4.0
-    pytest-xdist~=2.2.0
+    pytest-xdist~=2.5.0
     mock~=4.0.0
-    Faker~=8.1.0
     factory-boy~=3.2.0
-    pur~=5.4.0
     docopts~=0.6.0
-    tox~=3.23.0
+    tox~=3.24.0
 
 [flake8]
 max-complexity = 23