Clean up old observations on stage

* Remove unnecessary steps from test docker container

.gitlab-ci.yml

@@ -7,7 +7,7 @@ stages:
 test:
   stage: test
   script:
-    - pip install --no-cache-dir --require-hashes --no-deps -r /app/requirements/dev.txt
+    - pip install --no-cache-dir --require-hashes --no-deps -r requirements/dev.txt
     - cp env-dist .env
     - flake8 .
     - pytest

docker/test

@@ -4,6 +4,3 @@ RUN dnf -y install python python-pip python-devel git gcc libjpeg-turbo-devel fi
 RUN dnf -y clean all
 
 RUN npm install -g eslint stylelint
-
-WORKDIR /app
-COPY . /app

network/base/tasks.py

@@ -1,11 +1,13 @@
+from datetime import timedelta
 import json
 import urllib2
 
 from orbit import satellite
 
 from django.conf import settings
+from django.utils.timezone import now
 
-from network.base.models import Satellite, Tle, Mode, Transmitter
+from network.base.models import Satellite, Tle, Mode, Transmitter, Observation
 from network.celery import app
 
 
@@ -93,3 +95,14 @@ def fetch_data():
             new_transmitter.satellite = sat
             new_transmitter.mode = mode
             new_transmitter.save()
+
+
+@app.task
+def clean_observations():
+    """Task to clean up old observations that lack actual data."""
+    if settings.ENVIRONMENT == 'stage':
+        threshold = now() - timedelta(days=int(settings.OBSERVATION_OLD_RANGE))
+        observations = Observation.objects.filter(end__lt=threshold)
+        for obs in observations:
+            if not obs.is_verified():
+                obs.delete()

network/celery.py

@@ -23,7 +23,7 @@ app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
 
 @app.on_after_finalize.connect
 def setup_periodic_tasks(sender, **kwargs):
-    from network.base.tasks import update_all_tle, fetch_data
+    from network.base.tasks import update_all_tle, fetch_data, clean_observations
 
     sender.add_periodic_task(RUN_HOURLY, update_all_tle.s(),
                              name='update-all-tle')
@@ -31,8 +31,11 @@ def setup_periodic_tasks(sender, **kwargs):
     sender.add_periodic_task(RUN_HOURLY, fetch_data.s(),
                              name='fetch-data')
 
+    sender.add_periodic_task(RUN_DAILY, clean_observations.s(),
+                             name='clean-observations')
 
-if settings.ENVIRONMENT == 'production':
+
+if settings.ENVIRONMENT == 'production' or settings.ENVIRONMENT == 'stage':
     from opbeat.contrib.django.models import client, logger, register_handlers
     from opbeat.contrib.celery import register_signal
 

network/settings.py

@@ -301,11 +301,13 @@ OPBEAT = {
 # Observations settings
 # Datetimes in minutes for scheduling OPTIONS
 # Deletion range in minutes
+# Clean up threshold in days
 DATE_MIN_START = '15'
 DATE_MIN_END = '75'
 DATE_MAX_RANGE = '480'
 OBSERVATION_MAX_DELETION_RANGE = getenv('OBSERVATION_MAX_DELETION_RANGE', 10)
 OBSERVATION_MIN_DELETION_RANGE = getenv('OBSERVATION_MIN_DELETION_RANGE', 60)
+OBSERVATION_OLD_RANGE = getenv('OBSERVATION_OLD_RANGE', 30)
 
 # Station settings
 # Heartbeat for keeping a station online in minutes