Cache statistics through a celery task
parent
a674098e72
commit
fc8e8531e6
|
@ -3,12 +3,15 @@ from datetime import datetime, timedelta
|
|||
|
||||
from orbit import satellite
|
||||
|
||||
from django.db.models import Count, Max
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.core.mail import send_mail
|
||||
from django.template.loader import render_to_string
|
||||
from django.utils.timezone import make_aware
|
||||
|
||||
from db.base.models import Satellite, DemodData
|
||||
from db.base.utils import calculate_statistics
|
||||
from db.celery import app
|
||||
|
||||
|
||||
|
@ -70,3 +73,23 @@ def export_frames(norad, email, uid, period=None):
|
|||
}
|
||||
message = render_to_string(template, {'data': data})
|
||||
send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [email], False)
|
||||
|
||||
|
||||
@app.task
|
||||
def cache_statistics():
|
||||
statistics = calculate_statistics()
|
||||
cache.set('stats_transmitters', statistics, 60 * 60 * 2)
|
||||
|
||||
satellites = Satellite.objects \
|
||||
.values('name', 'norad_cat_id') \
|
||||
.annotate(count=Count('telemetry_data'),
|
||||
latest_payload=Max('telemetry_data__timestamp')) \
|
||||
.order_by('-count')
|
||||
cache.set('stats_satellites', satellites, 60 * 60 * 2)
|
||||
|
||||
observers = DemodData.objects \
|
||||
.values('observer') \
|
||||
.annotate(count=Count('observer'),
|
||||
latest_payload=Max('timestamp')) \
|
||||
.order_by('-count')
|
||||
cache.set('stats_observers', observers, 60 * 60 * 2)
|
||||
|
|
|
@ -0,0 +1,99 @@
|
|||
from db.base.models import Satellite, Transmitter, Mode, DemodData
|
||||
|
||||
|
||||
def calculate_statistics():
|
||||
"""View to create statistics endpoint."""
|
||||
satellites = Satellite.objects.all()
|
||||
transmitters = Transmitter.objects.all()
|
||||
modes = Mode.objects.all()
|
||||
|
||||
total_satellites = satellites.count()
|
||||
total_transmitters = transmitters.count()
|
||||
total_data = DemodData.objects.all().count()
|
||||
alive_transmitters = transmitters.filter(alive=True).count()
|
||||
alive_transmitters_percentage = '{0}%'.format(round((float(alive_transmitters) /
|
||||
float(total_transmitters)) * 100, 2))
|
||||
|
||||
mode_label = []
|
||||
mode_data = []
|
||||
for mode in modes:
|
||||
tr = transmitters.filter(mode=mode).count()
|
||||
mode_label.append(mode.name)
|
||||
mode_data.append(tr)
|
||||
|
||||
band_label = []
|
||||
band_data = []
|
||||
|
||||
# <30.000.000 - HF
|
||||
filtered = transmitters.filter(downlink_low__lt=30000000).count()
|
||||
band_label.append('HF')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 30.000.000 ~ 300.000.000 - VHF
|
||||
filtered = transmitters.filter(downlink_low__gte=30000000,
|
||||
downlink_low__lt=300000000).count()
|
||||
band_label.append('VHF')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 300.000.000 ~ 1.000.000.000 - UHF
|
||||
filtered = transmitters.filter(downlink_low__gte=300000000,
|
||||
downlink_low__lt=1000000000).count()
|
||||
band_label.append('UHF')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 1G ~ 2G - L
|
||||
filtered = transmitters.filter(downlink_low__gte=1000000000,
|
||||
downlink_low__lt=2000000000).count()
|
||||
band_label.append('L')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 2G ~ 4G - S
|
||||
filtered = transmitters.filter(downlink_low__gte=2000000000,
|
||||
downlink_low__lt=4000000000).count()
|
||||
band_label.append('S')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 4G ~ 8G - C
|
||||
filtered = transmitters.filter(downlink_low__gte=4000000000,
|
||||
downlink_low__lt=8000000000).count()
|
||||
band_label.append('C')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 8G ~ 12G - X
|
||||
filtered = transmitters.filter(downlink_low__gte=8000000000,
|
||||
downlink_low__lt=12000000000).count()
|
||||
band_label.append('X')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 12G ~ 18G - Ku
|
||||
filtered = transmitters.filter(downlink_low__gte=12000000000,
|
||||
downlink_low__lt=18000000000).count()
|
||||
band_label.append('Ku')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 18G ~ 27G - K
|
||||
filtered = transmitters.filter(downlink_low__gte=18000000000,
|
||||
downlink_low__lt=27000000000).count()
|
||||
band_label.append('K')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 27G ~ 40G - Ka
|
||||
filtered = transmitters.filter(downlink_low__gte=27000000000,
|
||||
downlink_low__lt=40000000000).count()
|
||||
band_label.append('Ka')
|
||||
band_data.append(filtered)
|
||||
|
||||
mode_data_sorted, mode_label_sorted = zip(*sorted(zip(mode_data, mode_label), reverse=True))
|
||||
band_data_sorted, band_label_sorted = zip(*sorted(zip(band_data, band_label), reverse=True))
|
||||
|
||||
statistics = {
|
||||
'total_satellites': total_satellites,
|
||||
'total_data': total_data,
|
||||
'transmitters': total_transmitters,
|
||||
'transmitters_alive': alive_transmitters_percentage,
|
||||
'mode_label': mode_label_sorted,
|
||||
'mode_data': mode_data_sorted,
|
||||
'band_label': band_label_sorted,
|
||||
'band_data': band_data_sorted
|
||||
}
|
||||
return statistics
|
125
db/base/views.py
125
db/base/views.py
|
@ -3,7 +3,6 @@ import logging
|
|||
import requests
|
||||
from datetime import datetime
|
||||
|
||||
from django.db.models import Count, Max
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.contrib.auth.models import User
|
||||
|
@ -18,8 +17,8 @@ from django.views.decorators.http import require_POST
|
|||
|
||||
from db.base.models import Mode, Transmitter, Satellite, Suggestion, DemodData
|
||||
from db.base.forms import SuggestionForm
|
||||
from db.base.helpers import get_apikey, cache_for
|
||||
from db.base.tasks import export_frames
|
||||
from db.base.helpers import get_apikey
|
||||
from db.base.tasks import export_frames, cache_statistics
|
||||
|
||||
|
||||
logger = logging.getLogger('db')
|
||||
|
@ -184,126 +183,18 @@ def faq(request):
|
|||
def stats(request):
|
||||
"""View to render stats page."""
|
||||
satellites = cache.get('stats_satellites')
|
||||
if not satellites:
|
||||
satellites = Satellite.objects \
|
||||
.values('name', 'norad_cat_id') \
|
||||
.annotate(count=Count('telemetry_data'),
|
||||
latest_payload=Max('telemetry_data__timestamp')) \
|
||||
.order_by('-count')
|
||||
cache.set('stats_satellites', satellites, settings.CACHE_TTL)
|
||||
observers = cache.get('stats_observers')
|
||||
if not observers:
|
||||
observers = DemodData.objects \
|
||||
.values('observer') \
|
||||
.annotate(count=Count('observer'),
|
||||
latest_payload=Max('timestamp')) \
|
||||
.order_by('-count')
|
||||
cache.set('stats_observers', observers, settings.CACHE_TTL)
|
||||
if not satellites or not observers:
|
||||
cache_statistics.delay()
|
||||
return render(request, 'base/stats.html', {'satellites': satellites,
|
||||
'observers': observers})
|
||||
|
||||
|
||||
@cache_for(settings.CACHE_TTL)
|
||||
def _calculate_statistics():
|
||||
"""View to create statistics endpoint."""
|
||||
satellites = Satellite.objects.all()
|
||||
transmitters = Transmitter.objects.all()
|
||||
modes = Mode.objects.all()
|
||||
|
||||
total_satellites = satellites.count()
|
||||
total_transmitters = transmitters.count()
|
||||
total_data = DemodData.objects.all().count()
|
||||
alive_transmitters = transmitters.filter(alive=True).count()
|
||||
alive_transmitters_percentage = '{0}%'.format(round((float(alive_transmitters) /
|
||||
float(total_transmitters)) * 100, 2))
|
||||
|
||||
mode_label = []
|
||||
mode_data = []
|
||||
for mode in modes:
|
||||
tr = transmitters.filter(mode=mode).count()
|
||||
mode_label.append(mode.name)
|
||||
mode_data.append(tr)
|
||||
|
||||
band_label = []
|
||||
band_data = []
|
||||
|
||||
# <30.000.000 - HF
|
||||
filtered = transmitters.filter(downlink_low__lt=30000000).count()
|
||||
band_label.append('HF')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 30.000.000 ~ 300.000.000 - VHF
|
||||
filtered = transmitters.filter(downlink_low__gte=30000000,
|
||||
downlink_low__lt=300000000).count()
|
||||
band_label.append('VHF')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 300.000.000 ~ 1.000.000.000 - UHF
|
||||
filtered = transmitters.filter(downlink_low__gte=300000000,
|
||||
downlink_low__lt=1000000000).count()
|
||||
band_label.append('UHF')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 1G ~ 2G - L
|
||||
filtered = transmitters.filter(downlink_low__gte=1000000000,
|
||||
downlink_low__lt=2000000000).count()
|
||||
band_label.append('L')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 2G ~ 4G - S
|
||||
filtered = transmitters.filter(downlink_low__gte=2000000000,
|
||||
downlink_low__lt=4000000000).count()
|
||||
band_label.append('S')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 4G ~ 8G - C
|
||||
filtered = transmitters.filter(downlink_low__gte=4000000000,
|
||||
downlink_low__lt=8000000000).count()
|
||||
band_label.append('C')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 8G ~ 12G - X
|
||||
filtered = transmitters.filter(downlink_low__gte=8000000000,
|
||||
downlink_low__lt=12000000000).count()
|
||||
band_label.append('X')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 12G ~ 18G - Ku
|
||||
filtered = transmitters.filter(downlink_low__gte=12000000000,
|
||||
downlink_low__lt=18000000000).count()
|
||||
band_label.append('Ku')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 18G ~ 27G - K
|
||||
filtered = transmitters.filter(downlink_low__gte=18000000000,
|
||||
downlink_low__lt=27000000000).count()
|
||||
band_label.append('K')
|
||||
band_data.append(filtered)
|
||||
|
||||
# 27G ~ 40G - Ka
|
||||
filtered = transmitters.filter(downlink_low__gte=27000000000,
|
||||
downlink_low__lt=40000000000).count()
|
||||
band_label.append('Ka')
|
||||
band_data.append(filtered)
|
||||
|
||||
mode_data_sorted, mode_label_sorted = zip(*sorted(zip(mode_data, mode_label), reverse=True))
|
||||
band_data_sorted, band_label_sorted = zip(*sorted(zip(band_data, band_label), reverse=True))
|
||||
|
||||
statistics = {
|
||||
'total_satellites': total_satellites,
|
||||
'total_data': total_data,
|
||||
'transmitters': total_transmitters,
|
||||
'transmitters_alive': alive_transmitters_percentage,
|
||||
'mode_label': mode_label_sorted,
|
||||
'mode_data': mode_data_sorted,
|
||||
'band_label': band_label_sorted,
|
||||
'band_data': band_data_sorted
|
||||
}
|
||||
return statistics
|
||||
|
||||
|
||||
def statistics(request):
|
||||
statistics = _calculate_statistics()
|
||||
statistics = cache.get('stats_transmitters')
|
||||
if not statistics:
|
||||
cache_statistics.delay()
|
||||
statistics = []
|
||||
return JsonResponse(statistics, safe=False)
|
||||
|
||||
|
||||
|
|
|
@ -12,6 +12,7 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'db.settings')
|
|||
|
||||
from django.conf import settings # noqa
|
||||
|
||||
RUN_HOURLY = 60 * 60
|
||||
RUN_DAILY = 60 * 60 * 24
|
||||
|
||||
app = Celery('db')
|
||||
|
@ -22,11 +23,14 @@ app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
|
|||
|
||||
@app.on_after_finalize.connect
|
||||
def setup_periodic_tasks(sender, **kwargs):
|
||||
from db.base.tasks import update_all_tle
|
||||
from db.base.tasks import update_all_tle, cache_statistics
|
||||
|
||||
sender.add_periodic_task(RUN_DAILY, update_all_tle.s(),
|
||||
name='update-all-tle')
|
||||
|
||||
sender.add_periodic_task(RUN_HOURLY, cache_statistics.s(),
|
||||
name='cache-statistics')
|
||||
|
||||
|
||||
from opbeat.contrib.django.models import client, logger, register_handlers # noqa
|
||||
from opbeat.contrib.celery import register_signal # noqa
|
||||
|
|
|
@ -68,8 +68,12 @@ $(document).ready(function() {
|
|||
},
|
||||
});
|
||||
|
||||
$.getJSON('/statistics/', function( data ) {
|
||||
|
||||
$.getJSON('/statistics/', function(data) {
|
||||
if (data.length == 0) {
|
||||
$('#transmitters-charts h2').text('still calculating...');
|
||||
$('#transmitters-charts div').append('<p>please come back later</p>');
|
||||
$('#transmitters-numbers').hide();
|
||||
} else {
|
||||
var i;
|
||||
var r;
|
||||
var g;
|
||||
|
@ -165,5 +169,8 @@ $(document).ready(function() {
|
|||
$('#stats-transmitters').html(data.transmitters);
|
||||
$('#stats-satellites').html(data.total_satellites);
|
||||
$('#stats-data').html(data.total_data);
|
||||
}
|
||||
}).fail(function() {
|
||||
$('.transmitters-charts').hide();
|
||||
});
|
||||
});
|
||||
|
|
|
@ -7,12 +7,12 @@
|
|||
|
||||
{% block content %}
|
||||
<!-- Transmitter stats -->
|
||||
<div class="row text">
|
||||
<div class="row text" id="transmitters-charts">
|
||||
<div class="col-md-12">
|
||||
<h2>Transmitters</h2>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row stats">
|
||||
<div class="row stats" id="transmitters-numbers">
|
||||
<div class="col-md-4">
|
||||
<canvas id="modes" width="300" height="300"></canvas>
|
||||
</div>
|
||||
|
@ -48,6 +48,8 @@
|
|||
</div>
|
||||
|
||||
<hr>
|
||||
|
||||
{% if satellites and observers %}
|
||||
<!-- Data paylod stats -->
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
|
@ -98,6 +100,8 @@
|
|||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endblock %}
|
||||
|
||||
{% block javascript %}
|
||||
|
|
|
@ -22,6 +22,9 @@ services:
|
|||
- ENVIRONMENT=stage
|
||||
- DEBUG=True
|
||||
- DATABASE_URL=mysql://satnogsdb:satnogsdb@db/satnogsdb
|
||||
- CACHE_BACKEND=redis_cache.RedisCache
|
||||
- CACHE_LOCATION=redis://redis:6379/1
|
||||
- CACHE_CLIENT_CLASS=django_redis.client.DefaultClient
|
||||
command:
|
||||
./bin/run-celery.sh
|
||||
web:
|
||||
|
|
Loading…
Reference in New Issue