Clean up old observations on stage
* Remove unnecessary steps from test docker containermerge-requests/408/head
parent
a39792c2ef
commit
f5cbad6fbd
|
@ -7,7 +7,7 @@ stages:
|
|||
test:
|
||||
stage: test
|
||||
script:
|
||||
- pip install --no-cache-dir --require-hashes --no-deps -r /app/requirements/dev.txt
|
||||
- pip install --no-cache-dir --require-hashes --no-deps -r requirements/dev.txt
|
||||
- cp env-dist .env
|
||||
- flake8 .
|
||||
- pytest
|
||||
|
|
|
@ -4,6 +4,3 @@ RUN dnf -y install python python-pip python-devel git gcc libjpeg-turbo-devel fi
|
|||
RUN dnf -y clean all
|
||||
|
||||
RUN npm install -g eslint stylelint
|
||||
|
||||
WORKDIR /app
|
||||
COPY . /app
|
||||
|
|
|
@ -1,11 +1,13 @@
|
|||
from datetime import timedelta
|
||||
import json
|
||||
import urllib2
|
||||
|
||||
from orbit import satellite
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils.timezone import now
|
||||
|
||||
from network.base.models import Satellite, Tle, Mode, Transmitter
|
||||
from network.base.models import Satellite, Tle, Mode, Transmitter, Observation
|
||||
from network.celery import app
|
||||
|
||||
|
||||
|
@ -93,3 +95,14 @@ def fetch_data():
|
|||
new_transmitter.satellite = sat
|
||||
new_transmitter.mode = mode
|
||||
new_transmitter.save()
|
||||
|
||||
|
||||
@app.task
|
||||
def clean_observations():
|
||||
"""Task to clean up old observations that lack actual data."""
|
||||
if settings.ENVIRONMENT == 'stage':
|
||||
threshold = now() - timedelta(days=int(settings.OBSERVATION_OLD_RANGE))
|
||||
observations = Observation.objects.filter(end__lt=threshold)
|
||||
for obs in observations:
|
||||
if not obs.is_verified():
|
||||
obs.delete()
|
||||
|
|
|
@ -23,7 +23,7 @@ app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
|
|||
|
||||
@app.on_after_finalize.connect
|
||||
def setup_periodic_tasks(sender, **kwargs):
|
||||
from network.base.tasks import update_all_tle, fetch_data
|
||||
from network.base.tasks import update_all_tle, fetch_data, clean_observations
|
||||
|
||||
sender.add_periodic_task(RUN_HOURLY, update_all_tle.s(),
|
||||
name='update-all-tle')
|
||||
|
@ -31,8 +31,11 @@ def setup_periodic_tasks(sender, **kwargs):
|
|||
sender.add_periodic_task(RUN_HOURLY, fetch_data.s(),
|
||||
name='fetch-data')
|
||||
|
||||
sender.add_periodic_task(RUN_DAILY, clean_observations.s(),
|
||||
name='clean-observations')
|
||||
|
||||
if settings.ENVIRONMENT == 'production':
|
||||
|
||||
if settings.ENVIRONMENT == 'production' or settings.ENVIRONMENT == 'stage':
|
||||
from opbeat.contrib.django.models import client, logger, register_handlers
|
||||
from opbeat.contrib.celery import register_signal
|
||||
|
||||
|
|
|
@ -301,11 +301,13 @@ OPBEAT = {
|
|||
# Observations settings
|
||||
# Datetimes in minutes for scheduling OPTIONS
|
||||
# Deletion range in minutes
|
||||
# Clean up threshold in days
|
||||
DATE_MIN_START = '15'
|
||||
DATE_MIN_END = '75'
|
||||
DATE_MAX_RANGE = '480'
|
||||
OBSERVATION_MAX_DELETION_RANGE = getenv('OBSERVATION_MAX_DELETION_RANGE', 10)
|
||||
OBSERVATION_MIN_DELETION_RANGE = getenv('OBSERVATION_MIN_DELETION_RANGE', 60)
|
||||
OBSERVATION_OLD_RANGE = getenv('OBSERVATION_OLD_RANGE', 30)
|
||||
|
||||
# Station settings
|
||||
# Heartbeat for keeping a station online in minutes
|
||||
|
|
Loading…
Reference in New Issue