spacecruft
/
satnogs-db
1
0
Fork 0
Code Releases Activity
satnogs-db/db/base/utils.py

556 lines
20 KiB
Python
Raw Blame History

"""Miscellaneous functions for SatNOGS DB"""
import binascii
import json
import logging
import math
from datetime import datetime, timedelta
from decimal import Decimal
from django.conf import settings
from django.core.cache import cache
from django.db import transaction
from django.db.models import Count, Max, OuterRef, Q, Subquery
from django.db.models.functions import Substr
from django.utils.timezone import make_aware, now
from influxdb import InfluxDBClient
from satnogsdecoders import __version__ as decoders_version
from satnogsdecoders import decoder
from db.base.models import DemodData, LatestTleSet, Mode, Satellite, Tle, Transmitter
LOGGER = logging.getLogger('db')
def remove_latest_tle_set(satellite_pk):
"""Remove LatestTleSet entry for specific Satellite"""
LatestTleSet.objects.filter(satellite__pk=satellite_pk).delete()
def update_latest_tle_sets(satellite_pks=None):
"""Update LatestTleSet entries for all or specific Satellites"""
# Select satellite models
if satellite_pks:
satellites = Satellite.objects.filter(
pk__in=satellite_pks,
associated_satellite__isnull=True,
satellite_entry__approved=True
).exclude(satellite_entry__status__in=['re-entered', 'future'])
else:
satellites = Satellite.objects.filter(
associated_satellite__isnull=True, satellite_entry__approved=True
).exclude(satellite_entry__status__in=['re-entered', 'future'])
# Create dictionary with Satellite ids as keys and Tle ids as values
latest_triplets = LatestTleSet.objects.filter(
satellite__in=satellites
).values_list('satellite', 'latest', 'latest_distributable')
latest_dictionary = {
latest_triplet[0]: (latest_triplet[1], latest_triplet[2])
for latest_triplet in latest_triplets
}
# For each satellite update LatestTleSet
for satellite in satellites:
# Keep the Tle ids of the LatestTleSet to check Tle entries inserted after them
# If there isn't one (new satellite or remove Tle entry) then check all of them
tle_id, tle_id_dist = (0, 0)
if satellite.id in latest_dictionary:
tle_ids = latest_dictionary[satellite.id]
tle_id = tle_ids[0] if tle_ids[0] else 0
tle_id_dist = tle_ids[1] if tle_ids[1] else 0
if tle_id_dist:
tle_dist = Tle.objects.get(id=tle_id_dist)
if tle_dist.tle_source not in settings.TLE_SOURCES_REDISTRIBUTABLE:
tle_id_dist = 0
# Query for the latest Tle set for this satellite
sub_subquery = Tle.objects.filter(
pk__gte=tle_id, satellite=satellite
).exclude(tle_source__in=settings.TLE_SOURCES_IGNORE_FROM_LATEST).filter(
satellite=OuterRef('satellite'), tle_source=OuterRef('tle_source')
).order_by('-updated')
subquery = Tle.objects.filter(
pk__gte=tle_id, satellite=satellite
).exclude(tle_source__in=settings.TLE_SOURCES_IGNORE_FROM_LATEST
).filter(pk=Subquery(sub_subquery.values('pk')[:1])
).filter(satellite=OuterRef('satellite')
).annotate(epoch=Max(Substr('tle1', 19, 14))
).order_by('-epoch')
new_latest = Tle.objects.filter(
pk__gte=tle_id, satellite=satellite
).exclude(tle_source__in=settings.TLE_SOURCES_IGNORE_FROM_LATEST
).filter(pk=Subquery(subquery.values('pk')[:1]))
# Query for the latest Tle set that is distributable for this satellite
sub_subquery = Tle.objects.filter(
pk__gte=tle_id_dist,
satellite=satellite,
tle_source__in=settings.TLE_SOURCES_REDISTRIBUTABLE
).exclude(tle_source__in=settings.TLE_SOURCES_IGNORE_FROM_LATEST).filter(
satellite=OuterRef('satellite'), tle_source=OuterRef('tle_source')
).order_by('-updated')
subquery = Tle.objects.filter(
pk__gte=tle_id_dist,
satellite=satellite,
tle_source__in=settings.TLE_SOURCES_REDISTRIBUTABLE
).exclude(tle_source__in=settings.TLE_SOURCES_IGNORE_FROM_LATEST
).filter(pk=Subquery(sub_subquery.values('pk')[:1])
).filter(satellite=OuterRef('satellite')
).annotate(epoch=Max(Substr('tle1', 19, 14))
).order_by('-epoch')
new_latest_dist = Tle.objects.filter(
pk__gte=tle_id_dist,
satellite=satellite,
tle_source__in=settings.TLE_SOURCES_REDISTRIBUTABLE
).exclude(tle_source__in=settings.TLE_SOURCES_IGNORE_FROM_LATEST
).filter(pk=Subquery(subquery.values('pk')[:1]))
# Add the latest Tle set if there is a LatestTleSet entry, if not create one
if new_latest:
LatestTleSet.objects.update_or_create(
satellite=satellite, defaults={
'latest': new_latest[0],
'last_modified': now()
}
)
# Add the latest distributable Tle set if there is a LatestTleSet entry, if not create one
if new_latest_dist:
LatestTleSet.objects.update_or_create(
satellite=satellite,
defaults={
'latest_distributable': new_latest_dist[0],
'last_modified': now()
}
)
# Remove any LatestTleSet that hasn't any Tle entry (satellite without Tle entries)
LatestTleSet.objects.filter(latest__isnull=True, latest_distributable__isnull=True).delete()
def get_tle_sources():
"""Check for and return TLE custom sources"""
sources = {}
if settings.TLE_SOURCES_JSON:
try:
sources_json = json.loads(settings.TLE_SOURCES_JSON)
sources['sources'] = list(sources_json.items())
except json.JSONDecodeError as error:
print('TLE Sources JSON ignored as it is invalid: {}'.format(error))
if settings.SPACE_TRACK_USERNAME and settings.SPACE_TRACK_PASSWORD:
sources['spacetrack_config'] = {
'identity': settings.SPACE_TRACK_USERNAME,
'password': settings.SPACE_TRACK_PASSWORD
}
return sources
def calculate_statistics():
"""Calculates statistics about the data we have in DB
:returns: a dictionary of statistics
"""
# satellite statistics
satellites = Satellite.objects.filter(
associated_satellite__isnull=True, satellite_entry__approved=True
)
total_satellites = satellites.count()
# data statistics
total_data = DemodData.objects.all().count()
# transmitter statistics
transmitters, total_transmitters, alive_transmitters_percentage = \
calculate_transmitters_stats()
# mode statistics
mode_data_sorted, mode_label_sorted = \
calculate_mode_stats(transmitters)
# band statistics
band_label_sorted, band_data_sorted = \
calculate_band_stats(transmitters)
statistics = {
'total_satellites': total_satellites,
'total_data': total_data,
'transmitters': total_transmitters,
'transmitters_alive': alive_transmitters_percentage,
'mode_label': mode_label_sorted,
'mode_data': mode_data_sorted,
'band_label': band_label_sorted,
'band_data': band_data_sorted
}
return statistics
def calculate_transmitters_stats():
"""Helper function to provite transmitters and statistics about
transmitters in db (such as total and percentage of alive)"""
transmitters = Transmitter.objects.filter(
satellite__associated_satellite__isnull=True, satellite__satellite_entry__approved=True
)
total_transmitters = transmitters.count()
alive_transmitters = transmitters.filter(status='active').count()
if alive_transmitters > 0 and total_transmitters > 0:
try:
alive_transmitters_percentage = '{0}%'.format(
round((float(alive_transmitters) / float(total_transmitters)) * 100, 2)
)
except ZeroDivisionError as error:
LOGGER.error(error, exc_info=True)
alive_transmitters_percentage = '0%'
else:
alive_transmitters_percentage = '0%'
return transmitters, total_transmitters, alive_transmitters_percentage
def calculate_mode_stats(transmitters):
"""Helper function to provide data and labels for modes associated with
transmitters provided"""
modes = Mode.objects.all()
mode_label = []
mode_data = []
for mode in modes:
filtered_transmitters = transmitters.filter(
downlink_mode=mode
).count() + transmitters.filter(uplink_mode=mode).count()
mode_label.append(mode.name)
mode_data.append(filtered_transmitters)
# needed to pass testing in a fresh environment with no modes in db
if not mode_label:
mode_label = ['FM']
if not mode_data:
mode_data = ['FM']
mode_data_sorted, mode_label_sorted = \
list(zip(*sorted(zip(mode_data, mode_label), reverse=True)))
return mode_data_sorted, mode_label_sorted
def calculate_band_stats(transmitters):
"""Helper function to provide data and labels for bands associated with
transmitters provided"""
band_label = []
band_data = []
bands = [
# <30.000.000 - HF
{
'lower_limit': 0,
'upper_limit': 30000000,
'label': 'HF'
},
# 30.000.000 ~ 300.000.000 - VHF
{
'lower_limit': 30000000,
'upper_limit': 300000000,
'label': 'VHF'
},
# 300.000.000 ~ 1.000.000.000 - UHF
{
'lower_limit': 300000000,
'upper_limit': 1000000000,
'label': 'UHF',
},
# 1G ~ 2G - L
{
'lower_limit': 1000000000,
'upper_limit': 2000000000,
'label': 'L',
},
# 2G ~ 4G - S
{
'lower_limit': 2000000000,
'upper_limit': 4000000000,
'label': 'S',
},
# 4G ~ 8G - C
{
'lower_limit': 4000000000,
'upper_limit': 8000000000,
'label': 'C',
},
# 8G ~ 12G - X
{
'lower_limit': 8000000000,
'upper_limit': 12000000000,
'label': 'X',
},
# 12G ~ 18G - Ku
{
'lower_limit': 12000000000,
'upper_limit': 18000000000,
'label': 'Ku',
},
# 18G ~ 27G - K
{
'lower_limit': 18000000000,
'upper_limit': 27000000000,
'label': 'K',
},
# 27G ~ 40G - Ka
{
'lower_limit': 27000000000,
'upper_limit': 40000000000,
'label': 'Ka',
},
]
for band in bands:
filtered = transmitters.filter(
downlink_low__gte=band['lower_limit'], downlink_low__lt=band['upper_limit']
).count()
band_label.append(band['label'])
band_data.append(filtered)
band_data_sorted, band_label_sorted = \
list(zip(*sorted(zip(band_data, band_label), reverse=True)))
return band_label_sorted, band_data_sorted
def create_point(fields, satellite, telemetry, demoddata, version):
"""Create a decoded data point in JSON format that is influxdb compatible
:returns: a JSON formatted time series data point
"""
point = [
{
'time': demoddata.timestamp.strftime('%Y-%m-%dT%H:%M:%SZ'),
'measurement': satellite.satellite_entry.norad_cat_id,
'tags': {
'satellite': satellite.satellite_entry.name,
'decoder': telemetry.decoder,
'station': demoddata.station,
'observer': demoddata.observer,
'source': demoddata.app_source,
'version': version
},
'fields': fields
}
]
return point
def write_influx(json_obj):
"""Take a json object and send to influxdb."""
client = InfluxDBClient(
settings.INFLUX_HOST,
settings.INFLUX_PORT,
settings.INFLUX_USER,
settings.INFLUX_PASS,
settings.INFLUX_DB,
ssl=settings.INFLUX_SSL,
verify_ssl=settings.INFLUX_VERIFY_SSL
)
client.write_points(json_obj)
def decode_demoddata(demoddata, satellite, tlmdecoder):
"""Decode a DemodData object and push it either in InfluxDB instance or in local DB"""
try:
decoder_class = getattr(decoder, tlmdecoder.decoder.capitalize())
except AttributeError:
return
try:
with open(demoddata.payload_frame.path) as frame_file:
# we get data frames in hex but kaitai requires binary
hexdata = frame_file.read()
bindata = binascii.unhexlify(hexdata)
try:
frame = decoder_class.from_bytes(bindata)
json_obj = create_point(
decoder.get_fields(frame), satellite, tlmdecoder, demoddata, decoders_version
)
# if we are set to use InfluxDB, send the decoded data
# there, otherwise we store it in the local DB.
if settings.USE_INFLUX:
write_influx(json_obj)
DemodData.objects.filter(pk=demoddata.id).update(
is_decoded=True, payload_decoded='influxdb'
)
else:
DemodData.objects.filter(pk=demoddata.id).update(
is_decoded=True, payload_decoded=json_obj
)
except BaseException: # pylint: disable=W0703
DemodData.objects.filter(pk=demoddata.id).update(is_decoded=False, payload_decoded='')
except (IOError, binascii.Error) as error:
LOGGER.error(error, exc_info=True)
def decode_data(sat_id, demoddata_id=None, redecode=False):
"""Decode data for a satellite, with an option to limit the scope.
:param sat_id: the Satellite Identifier of the satellite to decode data for
:param demoddata_id: if demoddata_id exists, try to decode this demoddata object
:param redecode: redecode demoddata or only recent
"""
satellite = Satellite.objects.get(satellite_identifier__sat_id=sat_id)
# If satellite is merged use telemetries from the associated satellite
if satellite.associated_satellite:
satellite = satellite.associated_satellite
telemetry_decoders = satellite.telemetries.all()
if telemetry_decoders:
data = DemodData.objects.select_for_update()
if demoddata_id:
if redecode:
data = data.filter(pk=demoddata_id)
else:
data = data.filter(pk=demoddata_id, is_decoded=False)
else:
# Get a list of all associated satellites for decoding all related data
satellites_list = list(satellite.associated_with.all().values_list('pk', flat=True))
satellites_list.append(satellite.pk)
if redecode:
data = data.filter(satellite__in=satellites_list)
else:
time_period = make_aware(datetime.utcnow() - timedelta(hours=48))
data = data.filter(
satellite__in=satellites_list, timestamp__gte=time_period, is_decoded=False
)
with transaction.atomic():
# iterate over DemodData objects
for obj in data:
# iterate over Telemetry decoders
for tlmdecoder in telemetry_decoders:
decode_demoddata(demoddata=obj, satellite=satellite, tlmdecoder=tlmdecoder)
# Caches stats about satellites and data
def cache_statistics():
"""Populate a django cache with statistics from data in DB
.. seealso:: calculate_statistics
"""
statistics = calculate_statistics()
cache.set('stats_transmitters', statistics, 60 * 60 * 25)
ids = []
sat_stats = {}
satellites = Satellite.objects.filter(satellite_entry__approved=True).values(
'satellite_entry__name', 'satellite_entry__norad_cat_id', 'id', 'associated_satellite',
'satellite_identifier__sat_id'
).annotate(
count=Count('telemetry_data'),
decoded=Count('telemetry_data', filter=Q(telemetry_data__is_decoded=True)),
latest_payload=Max('telemetry_data__timestamp')
)
# Aggregate stats for satellites and their associated ones
for sat in satellites:
# if satellite is merged then add statistics to its association
if sat['associated_satellite']:
if sat['associated_satellite'] in sat_stats:
sat_stats[sat['associated_satellite']]['count'] += sat['count']
sat_stats[sat['associated_satellite']]['decoded'] += sat['decoded']
if sat_stats[sat['associated_satellite']
]['latest_payload'] and sat['latest_payload']:
sat_stats[sat['associated_satellite']]['latest_payload'] = max(
sat_stats[sat['associated_satellite']]['latest_payload'],
sat['latest_payload']
)
else:
sat_stats[sat['associated_satellite']
]['latest_payload'] = sat['latest_payload']
else:
sat_id = sat['associated_satellite']
del sat['associated_satellite']
sat_stats[sat_id] = sat
else:
ids.append(sat['id'])
# if non-merged satellite is already in sat_stats then overwrite
# the name and the NORAD ID as the current ones are from one of its
# associated satellites that have been merged with it.
if sat['id'] in sat_stats:
sat_stats[sat['id']]['satellite_entry__name'] = sat['satellite_entry__name']
sat_stats[sat['id']
]['satellite_entry__norad_cat_id'] = sat['satellite_entry__norad_cat_id']
sat_stats[sat['id']
]['satellite_identifier__sat_id'] = sat['satellite_identifier__sat_id']
sat_stats[sat['id']]['count'] += sat['count']
sat_stats[sat['id']]['decoded'] += sat['decoded']
if sat_stats[sat['id']]['latest_payload'] and sat['latest_payload']:
sat_stats[sat['id']]['latest_payload'] = max(
sat_stats[sat['id']]['latest_payload'], sat['latest_payload']
)
else:
sat_stats[sat['id']]['latest_payload'] = sat['latest_payload']
else:
del sat['associated_satellite']
sat_stats[sat['id']] = sat
for sat_pk in sat_stats:
cache.set(sat_pk, sat_stats[sat_pk], 60 * 60 * 25)
cache.set('satellites_ids', ids, 60 * 60 * 25)
observers = DemodData.objects.values('observer').annotate(
count=Count('observer'), latest_payload=Max('timestamp')
).order_by('-count')
cache.set('stats_observers', observers, 60 * 60 * 25)
def remove_exponent(converted_number):
"""Remove exponent."""
return converted_number.quantize(
Decimal(1)
) if converted_number == converted_number.to_integral() else converted_number.normalize()
def millify(number, precision=0):
"""Humanize number."""
millnames = ['', 'k', 'M', 'B', 'T', 'P', 'E', 'Z', 'Y']
number = float(number)
millidx = max(
0,
min(
len(millnames) - 1, int(math.floor(0 if number == 0 else math.log10(abs(number)) / 3))
)
)
result = '{:.{precision}f}'.format(number / 10**(3 * millidx), precision=precision)
result = remove_exponent(Decimal(result))
return '{0}{dx}'.format(result, dx=millnames[millidx])
def read_influx(norad):
"""Queries influxdb for the last 30d of data points (counted) in 1d resolution.
:param norad: the NORAD ID of the satellite to query influxdb for
:returns: a raw json of the measurement, timestamps, and point counts
"""
client = InfluxDBClient(
settings.INFLUX_HOST,
settings.INFLUX_PORT,
settings.INFLUX_USER,
settings.INFLUX_PASS,
settings.INFLUX_DB,
ssl=settings.INFLUX_SSL,
verify_ssl=settings.INFLUX_VERIFY_SSL
)
# check against injection
if isinstance(norad, int):
# epoch:s to set the return timestamp in unixtime for easier conversion
params = {'epoch': 's'}
results = client.query(
'SELECT count(*) FROM "' + str(norad)
+ '" WHERE time > now() - 30d GROUP BY time(1d) fill(null)',
params=params
)
return results.raw
# no-else-return
return ''
View Git Blame Copy Permalink