spacecruft
/
satnogs-db
1
0
Fork 0
Code Releases Activity

Cache statistics through a celery task

merge-requests/208/head
Nikos Roussos 2017-11-13 12:27:11 +02:00
parent a674098e72
commit fc8e8531e6
No known key found for this signature in database
GPG Key ID: BADFF1767BA7C8E1
7 changed files with 286 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
db/base/tasks.py
View File

@ -3,12 +3,15 @@ from datetime import datetime, timedelta
from orbit import satellite
from django.db.models import Count, Max
from django.conf import settings
from django.core.cache import cache
from django.core.mail import send_mail
from django.template.loader import render_to_string
from django.utils.timezone import make_aware
from db.base.models import Satellite, DemodData
from db.base.utils import calculate_statistics
from db.celery import app
@ -70,3 +73,23 @@ def export_frames(norad, email, uid, period=None):
}
message = render_to_string(template, {'data': data})
send_mail(subject, message, settings.DEFAULT_FROM_EMAIL, [email], False)
@app.task
def cache_statistics():
statistics = calculate_statistics()
cache.set('stats_transmitters', statistics, 60 * 60 * 2)
satellites = Satellite.objects \
.values('name', 'norad_cat_id') \
.annotate(count=Count('telemetry_data'),
latest_payload=Max('telemetry_data__timestamp')) \
.order_by('-count')
cache.set('stats_satellites', satellites, 60 * 60 * 2)
observers = DemodData.objects \
.values('observer') \
.annotate(count=Count('observer'),
latest_payload=Max('timestamp')) \
.order_by('-count')
cache.set('stats_observers', observers, 60 * 60 * 2)

99
db/base/utils.py 100644
View File

@ -0,0 +1,99 @@
from db.base.models import Satellite, Transmitter, Mode, DemodData
def calculate_statistics():
"""View to create statistics endpoint."""
satellites = Satellite.objects.all()
transmitters = Transmitter.objects.all()
modes = Mode.objects.all()
total_satellites = satellites.count()
total_transmitters = transmitters.count()
total_data = DemodData.objects.all().count()
alive_transmitters = transmitters.filter(alive=True).count()
alive_transmitters_percentage = '{0}%'.format(round((float(alive_transmitters) /
float(total_transmitters)) * 100, 2))
mode_label = []
mode_data = []
for mode in modes:
tr = transmitters.filter(mode=mode).count()
mode_label.append(mode.name)
mode_data.append(tr)
band_label = []
band_data = []
# <30.000.000 - HF
filtered = transmitters.filter(downlink_low__lt=30000000).count()
band_label.append('HF')
band_data.append(filtered)
# 30.000.000 ~ 300.000.000 - VHF
filtered = transmitters.filter(downlink_low__gte=30000000,
downlink_low__lt=300000000).count()
band_label.append('VHF')
band_data.append(filtered)
# 300.000.000 ~ 1.000.000.000 - UHF
filtered = transmitters.filter(downlink_low__gte=300000000,
downlink_low__lt=1000000000).count()
band_label.append('UHF')
band_data.append(filtered)
# 1G ~ 2G - L
filtered = transmitters.filter(downlink_low__gte=1000000000,
downlink_low__lt=2000000000).count()
band_label.append('L')
band_data.append(filtered)
# 2G ~ 4G - S
filtered = transmitters.filter(downlink_low__gte=2000000000,
downlink_low__lt=4000000000).count()
band_label.append('S')
band_data.append(filtered)
# 4G ~ 8G - C
filtered = transmitters.filter(downlink_low__gte=4000000000,
downlink_low__lt=8000000000).count()
band_label.append('C')
band_data.append(filtered)
# 8G ~ 12G - X
filtered = transmitters.filter(downlink_low__gte=8000000000,
downlink_low__lt=12000000000).count()
band_label.append('X')
band_data.append(filtered)
# 12G ~ 18G - Ku
filtered = transmitters.filter(downlink_low__gte=12000000000,
downlink_low__lt=18000000000).count()
band_label.append('Ku')
band_data.append(filtered)
# 18G ~ 27G - K
filtered = transmitters.filter(downlink_low__gte=18000000000,
downlink_low__lt=27000000000).count()
band_label.append('K')
band_data.append(filtered)
# 27G ~ 40G - Ka
filtered = transmitters.filter(downlink_low__gte=27000000000,
downlink_low__lt=40000000000).count()
band_label.append('Ka')
band_data.append(filtered)
mode_data_sorted, mode_label_sorted = zip(*sorted(zip(mode_data, mode_label), reverse=True))
band_data_sorted, band_label_sorted = zip(*sorted(zip(band_data, band_label), reverse=True))
statistics = {
'total_satellites': total_satellites,
'total_data': total_data,
'transmitters': total_transmitters,
'transmitters_alive': alive_transmitters_percentage,
'mode_label': mode_label_sorted,
'mode_data': mode_data_sorted,
'band_label': band_label_sorted,
'band_data': band_data_sorted
}
return statistics

125
db/base/views.py
View File

@ -3,7 +3,6 @@ import logging
import requests
from datetime import datetime
from django.db.models import Count, Max
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
@ -18,8 +17,8 @@ from django.views.decorators.http import require_POST
from db.base.models import Mode, Transmitter, Satellite, Suggestion, DemodData
from db.base.forms import SuggestionForm
from db.base.helpers import get_apikey, cache_for
from db.base.tasks import export_frames
from db.base.helpers import get_apikey
from db.base.tasks import export_frames, cache_statistics
logger = logging.getLogger('db')
@ -184,126 +183,18 @@ def faq(request):
def stats(request):
"""View to render stats page."""
satellites = cache.get('stats_satellites')
if not satellites:
satellites = Satellite.objects \
.values('name', 'norad_cat_id') \
.annotate(count=Count('telemetry_data'),
latest_payload=Max('telemetry_data__timestamp')) \
.order_by('-count')
cache.set('stats_satellites', satellites, settings.CACHE_TTL)
observers = cache.get('stats_observers')
if not observers:
observers = DemodData.objects \
.values('observer') \
.annotate(count=Count('observer'),
latest_payload=Max('timestamp')) \
.order_by('-count')
cache.set('stats_observers', observers, settings.CACHE_TTL)
if not satellites or not observers:
cache_statistics.delay()
return render(request, 'base/stats.html', {'satellites': satellites,
'observers': observers})
@cache_for(settings.CACHE_TTL)
def _calculate_statistics():
"""View to create statistics endpoint."""
satellites = Satellite.objects.all()
transmitters = Transmitter.objects.all()
modes = Mode.objects.all()
total_satellites = satellites.count()
total_transmitters = transmitters.count()
total_data = DemodData.objects.all().count()
alive_transmitters = transmitters.filter(alive=True).count()
alive_transmitters_percentage = '{0}%'.format(round((float(alive_transmitters) /
float(total_transmitters)) * 100, 2))
mode_label = []
mode_data = []
for mode in modes:
tr = transmitters.filter(mode=mode).count()
mode_label.append(mode.name)
mode_data.append(tr)
band_label = []
band_data = []
# <30.000.000 - HF
filtered = transmitters.filter(downlink_low__lt=30000000).count()
band_label.append('HF')
band_data.append(filtered)
# 30.000.000 ~ 300.000.000 - VHF
filtered = transmitters.filter(downlink_low__gte=30000000,
downlink_low__lt=300000000).count()
band_label.append('VHF')
band_data.append(filtered)
# 300.000.000 ~ 1.000.000.000 - UHF
filtered = transmitters.filter(downlink_low__gte=300000000,
downlink_low__lt=1000000000).count()
band_label.append('UHF')
band_data.append(filtered)
# 1G ~ 2G - L
filtered = transmitters.filter(downlink_low__gte=1000000000,
downlink_low__lt=2000000000).count()
band_label.append('L')
band_data.append(filtered)
# 2G ~ 4G - S
filtered = transmitters.filter(downlink_low__gte=2000000000,
downlink_low__lt=4000000000).count()
band_label.append('S')
band_data.append(filtered)
# 4G ~ 8G - C
filtered = transmitters.filter(downlink_low__gte=4000000000,
downlink_low__lt=8000000000).count()
band_label.append('C')
band_data.append(filtered)
# 8G ~ 12G - X
filtered = transmitters.filter(downlink_low__gte=8000000000,
downlink_low__lt=12000000000).count()
band_label.append('X')
band_data.append(filtered)
# 12G ~ 18G - Ku
filtered = transmitters.filter(downlink_low__gte=12000000000,
downlink_low__lt=18000000000).count()
band_label.append('Ku')
band_data.append(filtered)
# 18G ~ 27G - K
filtered = transmitters.filter(downlink_low__gte=18000000000,
downlink_low__lt=27000000000).count()
band_label.append('K')
band_data.append(filtered)
# 27G ~ 40G - Ka
filtered = transmitters.filter(downlink_low__gte=27000000000,
downlink_low__lt=40000000000).count()
band_label.append('Ka')
band_data.append(filtered)
mode_data_sorted, mode_label_sorted = zip(*sorted(zip(mode_data, mode_label), reverse=True))
band_data_sorted, band_label_sorted = zip(*sorted(zip(band_data, band_label), reverse=True))
statistics = {
'total_satellites': total_satellites,
'total_data': total_data,
'transmitters': total_transmitters,
'transmitters_alive': alive_transmitters_percentage,
'mode_label': mode_label_sorted,
'mode_data': mode_data_sorted,
'band_label': band_label_sorted,
'band_data': band_data_sorted
}
return statistics
def statistics(request):
statistics = _calculate_statistics()
statistics = cache.get('stats_transmitters')
if not statistics:
cache_statistics.delay()
statistics = []
return JsonResponse(statistics, safe=False)

6
db/celery.py
View File

@ -12,6 +12,7 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'db.settings')
from django.conf import settings # noqa
RUN_HOURLY = 60 * 60
RUN_DAILY = 60 * 60 * 24
app = Celery('db')
@ -22,11 +23,14 @@ app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
@app.on_after_finalize.connect
def setup_periodic_tasks(sender, **kwargs):
from db.base.tasks import update_all_tle
from db.base.tasks import update_all_tle, cache_statistics
sender.add_periodic_task(RUN_DAILY, update_all_tle.s(),
name='update-all-tle')
sender.add_periodic_task(RUN_HOURLY, cache_statistics.s(),
name='cache-statistics')
from opbeat.contrib.django.models import client, logger, register_handlers # noqa
from opbeat.contrib.celery import register_signal # noqa

183
db/static/js/stats.js
View File

@ -68,102 +68,109 @@ $(document).ready(function() {
},
});
$.getJSON('/statistics/', function( data ) {
$.getJSON('/statistics/', function(data) {
if (data.length == 0) {
$('#transmitters-charts h2').text('still calculating...');
$('#transmitters-charts div').append('<p>please come back later</p>');
$('#transmitters-numbers').hide();
} else {
var i;
var r;
var g;
var b;
var a;
var color;
// Create colors for Mode Chart
var mode_colors = [];
for (i = 0; i < data.mode_label.length; i++) {
r = Math.floor(data.mode_data[i]* 10);
b = Math.floor(0.3 * 255);
g = Math.floor(data.mode_data[i]* 10);
a = 0.5;
color = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';
mode_colors.push(color);
}
var i;
var r;
var g;
var b;
var a;
var color;
// Create colors for Mode Chart
var mode_colors = [];
for (i = 0; i < data.mode_label.length; i++) {
r = Math.floor(data.mode_data[i]* 10);
b = Math.floor(0.3 * 255);
g = Math.floor(data.mode_data[i]* 10);
a = 0.5;
color = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';
mode_colors.push(color);
}
// Create colors for Band Chart
var band_colors = [];
for (i = 0; i < data.band_label.length; i++) {
b = Math.floor(0.1 * 255);
g = Math.floor(data.band_data[i]);
r = Math.floor(data.band_data[i]);
a = 0.5;
color = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';
band_colors.push(color);
}
// Create colors for Band Chart
var band_colors = [];
for (i = 0; i < data.band_label.length; i++) {
b = Math.floor(0.1 * 255);
g = Math.floor(data.band_data[i]);
r = Math.floor(data.band_data[i]);
a = 0.5;
color = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';
band_colors.push(color);
}
// Global chart configuration
Chart.defaults.global.legend.display = false;
Chart.defaults.global.title.display = true;
Chart.defaults.global.title.fontSize = 16;
Chart.defaults.global.title.fontColor= '#444';
// Global chart configuration
Chart.defaults.global.legend.display = false;
Chart.defaults.global.title.display = true;
Chart.defaults.global.title.fontSize = 16;
Chart.defaults.global.title.fontColor= '#444';
//Mode Chart
var mode_c = document.getElementById('modes');
new Chart(mode_c, {
type: 'doughnut',
data: {
labels: data.mode_label,
datasets: [{
backgroundColor: mode_colors,
data: data.mode_data,
borderWidth: 1
}]
},
options: {
elements: {
center: {
// the longest text that could appear in the center
maxText: '100%',
text: data.mode_data.length + ' Modes',
fontColor: '#000',
fontFamily: 'Helvetica Neue',
fontStyle: 'normal',
minFontSize: 1,
maxFontSize: 20,
//Mode Chart
var mode_c = document.getElementById('modes');
new Chart(mode_c, {
type: 'doughnut',
data: {
labels: data.mode_label,
datasets: [{
backgroundColor: mode_colors,
data: data.mode_data,
borderWidth: 1
}]
},
options: {
elements: {
center: {
// the longest text that could appear in the center
maxText: '100%',
text: data.mode_data.length + ' Modes',
fontColor: '#000',
fontFamily: 'Helvetica Neue',
fontStyle: 'normal',
minFontSize: 1,
maxFontSize: 20,
}
}
}
}
});
});
//Band Chart
var band_c = document.getElementById('bands');
new Chart(band_c, {
type: 'doughnut',
data: {
labels: data.band_label,
datasets: [{
backgroundColor: band_colors,
data: data.band_data,
borderWidth: 1
}]
},
options: {
elements: {
center: {
// the longest text that could appear in the center
maxText: '100%',
text: data.band_data.length + ' Bands',
fontColor: '#000',
fontFamily: 'Helvetica Neue',
fontStyle: 'normal',
minFontSize: 1,
maxFontSize: 20,
//Band Chart
var band_c = document.getElementById('bands');
new Chart(band_c, {
type: 'doughnut',
data: {
labels: data.band_label,
datasets: [{
backgroundColor: band_colors,
data: data.band_data,
borderWidth: 1
}]
},
options: {
elements: {
center: {
// the longest text that could appear in the center
maxText: '100%',
text: data.band_data.length + ' Bands',
fontColor: '#000',
fontFamily: 'Helvetica Neue',
fontStyle: 'normal',
minFontSize: 1,
maxFontSize: 20,
}
}
}
}
});
});
//HUD Stats
$('#stats-alive').html(data.transmitters_alive);
$('#stats-transmitters').html(data.transmitters);
$('#stats-satellites').html(data.total_satellites);
$('#stats-data').html(data.total_data);
//HUD Stats
$('#stats-alive').html(data.transmitters_alive);
$('#stats-transmitters').html(data.transmitters);
$('#stats-satellites').html(data.total_satellites);
$('#stats-data').html(data.total_data);
}
}).fail(function() {
$('.transmitters-charts').hide();
});
});

102
db/templates/base/stats.html
View File

@ -7,12 +7,12 @@
{% block content %}
<!-- Transmitter stats -->
<div class="row text">
<div class="row text" id="transmitters-charts">
<div class="col-md-12">
<h2>Transmitters</h2>
</div>
</div>
<div class="row stats">
<div class="row stats" id="transmitters-numbers">
<div class="col-md-4">
<canvas id="modes" width="300" height="300"></canvas>
</div>
@ -48,56 +48,60 @@
</div>
<hr>
<!-- Data paylod stats -->
<div class="row">
<div class="col-md-6">
<h2>Data per satellite</h2>
<table class="table table-hover">
<thead>
<tr>
<th>Norad ID</th>
<th>Name</th>
<th>Data</th>
<th>Latest</th>
</tr>
</thead>
<tbody>
{% for sat in satellites %}
{% if sat.count != 0 %}
<tr>
<td>{{sat.norad_cat_id}}</td>
<td><a href="{% url 'satellite' norad=sat.norad_cat_id %}">{{sat.name}}</a></td>
<td>{{sat.count}}</td>
<td>{{sat.latest_payload|date:"Y-m-d H:i:s"}}</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</div>
<div class="col-md-6">
<h2>Data per station</h2>
<table class="table table-hover">
<thead>
<tr>
<th>#</th>
<th>Name</th>
<th>Data</th>
<th>Latest</th>
</tr>
</thead>
<tbody>
{% for obs in observers %}
{% if satellites and observers %}
<!-- Data paylod stats -->
<div class="row">
<div class="col-md-6">
<h2>Data per satellite</h2>
<table class="table table-hover">
<thead>
<tr>
<td>{{forloop.counter}}</td>
<td>{{obs.observer}}</td>
<td>{{obs.count}}</td>
<td>{{obs.latest_payload|date:"Y-m-d H:i:s"}}</td>
<th>Norad ID</th>
<th>Name</th>
<th>Data</th>
<th>Latest</th>
</tr>
{% endfor %}
</tbody>
</table>
</thead>
<tbody>
{% for sat in satellites %}
{% if sat.count != 0 %}
<tr>
<td>{{sat.norad_cat_id}}</td>
<td><a href="{% url 'satellite' norad=sat.norad_cat_id %}">{{sat.name}}</a></td>
<td>{{sat.count}}</td>
<td>{{sat.latest_payload|date:"Y-m-d H:i:s"}}</td>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
</div>
<div class="col-md-6">
<h2>Data per station</h2>
<table class="table table-hover">
<thead>
<tr>
<th>#</th>
<th>Name</th>
<th>Data</th>
<th>Latest</th>
</tr>
</thead>
<tbody>
{% for obs in observers %}
<tr>
<td>{{forloop.counter}}</td>
<td>{{obs.observer}}</td>
<td>{{obs.count}}</td>
<td>{{obs.latest_payload|date:"Y-m-d H:i:s"}}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
{% endif %}
{% endblock %}
{% block javascript %}

3
docker-compose.yml
View File

@ -22,6 +22,9 @@ services:
- ENVIRONMENT=stage
- DEBUG=True
- DATABASE_URL=mysql://satnogsdb:satnogsdb@db/satnogsdb
- CACHE_BACKEND=redis_cache.RedisCache
- CACHE_LOCATION=redis://redis:6379/1
- CACHE_CLIENT_CLASS=django_redis.client.DefaultClient
command:
./bin/run-celery.sh
web: