add recursion to the decode processes
refactored Refactored db decoding for recursion Should cover all cases for kaitai generated decoder files.environments/stage/deployments/30
Corey Shields
parent
ad94d1bf12
commit
162c1e45d2
|
|
@ -48,7 +48,7 @@ class Cas4(KaitaiStruct):
|
|||
self._read()
|
||||
|
||||
def _read(self):
|
||||
self.u1_pwr_volt = self._io.read_u1()
|
||||
self.pwr_volt = self._io.read_u1()
|
||||
self.pwr_cur = self._io.read_u1()
|
||||
self.convert_volt = self._io.read_u1()
|
||||
self.convert_cur = self._io.read_u1()
|
||||
|
|
@ -115,4 +115,3 @@ class Cas4(KaitaiStruct):
|
|||
self._m_framecounter = self._io.read_u1()
|
||||
self._io.seek(_pos)
|
||||
return self._m_framecounter if hasattr(self, '_m_framecounter') else None
|
||||
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
EPOCH_DATE = datetime.strptime('20000101T000000Z', '%Y%m%dT%H%M%SZ')
|
||||
|
||||
|
||||
def decode_payload(payload, observation_datetime, data_id):
|
||||
dt = payload[:32]
|
||||
datasets = payload[32:]
|
||||
|
||||
# calculate initial datetime
|
||||
seconds = int(dt[24:] + dt[16:24] + dt[8:16] + dt[:8], 2)
|
||||
|
||||
telemetry = []
|
||||
|
||||
while datasets:
|
||||
dataset = datasets[:57]
|
||||
datasets = datasets[57:]
|
||||
|
||||
dataset_datetime = EPOCH_DATE + timedelta(seconds=seconds)
|
||||
seconds += 60
|
||||
satellite_datetime = datetime.strftime(dataset_datetime, '%Y%m%dT%H%M%SZ')
|
||||
|
||||
# mode
|
||||
status = dataset[0]
|
||||
|
||||
# battery voltage
|
||||
u = float(int(dataset[1:9], 2))
|
||||
bat_v = round((u + 60) / 20, 2)
|
||||
|
||||
# battery current
|
||||
u = float(int(dataset[9:17], 2))
|
||||
bat_c = round((u - 127) / 127, 2)
|
||||
|
||||
# 3v3 current
|
||||
u = float(int(dataset[17:25], 2))
|
||||
v3_c = round(u / 40, 2)
|
||||
|
||||
# 5v current
|
||||
u = float(int(dataset[25:33], 2))
|
||||
v5_c = round(u / 40, 2)
|
||||
|
||||
# temperature comms
|
||||
u = float(int(dataset[33:41], 2))
|
||||
comms_t = round((u - 60) / 4, 2)
|
||||
|
||||
# temperature eps
|
||||
u = float(int(dataset[41:49], 2))
|
||||
eps_t = round((u - 60) / 4, 2)
|
||||
|
||||
# temperature battery
|
||||
u = float(int(dataset[49:], 2))
|
||||
batt_t = round((u - 60) / 4, 2)
|
||||
|
||||
data = {
|
||||
'satellite_datetime': satellite_datetime,
|
||||
'observation_datetime': observation_datetime,
|
||||
'data_id': data_id,
|
||||
'demod_data': {
|
||||
'status': status,
|
||||
'bat_v': bat_v,
|
||||
'bat_c': bat_c,
|
||||
'v3_c': v3_c,
|
||||
'v5_c': v5_c,
|
||||
'comms_t': comms_t,
|
||||
'eps_t': eps_t,
|
||||
'batt_t': batt_t
|
||||
}
|
||||
}
|
||||
|
||||
telemetry.append(data)
|
||||
return telemetry
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
# This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
|
||||
|
||||
from pkg_resources import parse_version
|
||||
from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
|
||||
|
||||
|
||||
if parse_version(ks_version) < parse_version('0.7'):
|
||||
raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
|
||||
|
||||
class Siriussat(KaitaiStruct):
|
||||
def __init__(self, _io, _parent=None, _root=None):
|
||||
self._io = _io
|
||||
self._parent = _parent
|
||||
self._root = _root if _root else self
|
||||
self._read()
|
||||
|
||||
def _read(self):
|
||||
self.ax25header = self._io.read_bytes(16)
|
||||
self.u_panel1 = self._io.read_u2le()
|
||||
self.u_panel2 = self._io.read_u2le()
|
||||
self.u_panel3 = self._io.read_u2le()
|
||||
self.i_panel1 = self._io.read_u2le()
|
||||
self.i_panel2 = self._io.read_u2le()
|
||||
self.i_panel3 = self._io.read_u2le()
|
||||
self.i_bat = self._io.read_s2le()
|
||||
self.i_ch1 = self._io.read_u2le()
|
||||
self.i_ch2 = self._io.read_u2le()
|
||||
self.i_ch3 = self._io.read_u2le()
|
||||
self.i_ch4 = self._io.read_u2le()
|
||||
self.t1_pw = self._io.read_s2le()
|
||||
self.t2_pw = self._io.read_s2le()
|
||||
self.t3_pw = self._io.read_s2le()
|
||||
self.t4_pw = self._io.read_s2le()
|
||||
self.u_bat_crit = self._io.read_bits_int(1) != 0
|
||||
self.u_bat_min = self._io.read_bits_int(1) != 0
|
||||
self.heater2_manual = self._io.read_bits_int(1) != 0
|
||||
self.heater1_manual = self._io.read_bits_int(1) != 0
|
||||
self.heater2_on = self._io.read_bits_int(1) != 0
|
||||
self.heater1_on = self._io.read_bits_int(1) != 0
|
||||
self.t_bat_max = self._io.read_bits_int(1) != 0
|
||||
self.t_bat_min = self._io.read_bits_int(1) != 0
|
||||
self.channel_on4 = self._io.read_bits_int(1) != 0
|
||||
self.channel_on3 = self._io.read_bits_int(1) != 0
|
||||
self.channel_on2 = self._io.read_bits_int(1) != 0
|
||||
self.channel_on1 = self._io.read_bits_int(1) != 0
|
||||
self.i_ch_limit4 = self._io.read_bits_int(1) != 0
|
||||
self.i_ch_limit3 = self._io.read_bits_int(1) != 0
|
||||
self.i_ch_limit2 = self._io.read_bits_int(1) != 0
|
||||
self.i_ch_limit1 = self._io.read_bits_int(1) != 0
|
||||
self.reserved0 = self._io.read_bits_int(7)
|
||||
self.charger = self._io.read_bits_int(1) != 0
|
||||
self._io.align_to_byte()
|
||||
self.reserved1 = self._io.read_u1()
|
||||
self.u_bat = self._io.read_s2le()
|
||||
self.reg_tel_id = self._io.read_u4le()
|
||||
self.pss_time = self._io.read_s4le()
|
||||
self.pss_n_reset = self._io.read_u1()
|
||||
self.pss_flags = self._io.read_u1()
|
||||
self.t_amp = self._io.read_s1()
|
||||
self.t_uhf = self._io.read_s1()
|
||||
self.rssi_rx = self._io.read_s1()
|
||||
self.rssi_idle = self._io.read_s1()
|
||||
self.power_forward = self._io.read_s1()
|
||||
self.power_reflected = self._io.read_s1()
|
||||
self.uhf_n_reset = self._io.read_u1()
|
||||
self.uhf_flags = self._io.read_u1()
|
||||
self.uhf_time = self._io.read_s4le()
|
||||
self.uptime = self._io.read_u4le()
|
||||
self.uhf_current = self._io.read_u2le()
|
||||
self.uhf_voltage = self._io.read_s2le()
|
||||
|
||||
|
||||
|
||||
|
|
@ -105,7 +105,8 @@ def cache_statistics():
|
|||
@app.task
|
||||
def reset_decoded_data(norad):
|
||||
"""DESTRUCTIVE: deletes decoded data from db and/or influxdb"""
|
||||
frames = DemodData.objects.filter(satellite__norad_cat_id=norad)
|
||||
frames = DemodData.objects.filter(satellite__norad_cat_id=norad) \
|
||||
.filter(is_decoded=True)
|
||||
for frame in frames:
|
||||
frame.payload_decoded = ''
|
||||
frame.is_decoded = False
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import json
|
||||
import binascii
|
||||
from datetime import datetime, timedelta
|
||||
from db.base.models import Satellite, Transmitter, Mode, DemodData, Telemetry
|
||||
|
|
@ -110,38 +109,13 @@ def calculate_statistics():
|
|||
# decoders with different attributes. This is a hacky way of getting them
|
||||
# to json. We also need to sanitize this for any binary data left over as
|
||||
# it won't export to json.
|
||||
def kaitai_to_json(struct):
|
||||
"""Take a kaitai decode object and send to db as a json object"""
|
||||
structdict = struct.__dict__
|
||||
tojson = {}
|
||||
for key, value in structdict.iteritems():
|
||||
if key != '_root' and \
|
||||
key != '_parent' and \
|
||||
key != '_io': # kaitai objects we want to skip
|
||||
if isinstance(value, basestring): # skip binary values
|
||||
try:
|
||||
value.decode('utf-8')
|
||||
tojson[key] = value
|
||||
except UnicodeError:
|
||||
continue
|
||||
else:
|
||||
tojson[key] = value
|
||||
return json.dumps(tojson)
|
||||
|
||||
|
||||
# send the data from kaitai to influxdb, expects a kaitai object, the satellite
|
||||
# telemetry and demoddata for that observation to tag metadata.
|
||||
def kaitai_to_influx(struct, satellite, telemetry, demoddata):
|
||||
"""Take a kaitai decode object and send to influxdb."""
|
||||
client = InfluxDBClient(settings.INFLUX_HOST, settings.INFLUX_PORT,
|
||||
settings.INFLUX_USER, settings.INFLUX_PASS,
|
||||
settings.INFLUX_DB)
|
||||
structdict = struct.__dict__
|
||||
def kaitai_to_json(structdict, satellite, telemetry, demoddata, json_obj):
|
||||
"""Recursively sends dict string/int data to the given json_obj"""
|
||||
for key, value in structdict.iteritems():
|
||||
if key != '_root' and \
|
||||
key != '_parent' and \
|
||||
key != '_io': # kaitai objects we want to skip
|
||||
influx_tlm = []
|
||||
if type(value) is dict: # recursion
|
||||
kaitai_to_json(value, satellite, telemetry, demoddata, json_obj)
|
||||
else:
|
||||
data = {
|
||||
'time': demoddata.timestamp.strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'measurement': key,
|
||||
|
|
@ -163,13 +137,38 @@ def kaitai_to_influx(struct, satellite, telemetry, demoddata):
|
|||
else:
|
||||
data.update({'fields': {'value': value}})
|
||||
if 'value' in data['fields']:
|
||||
influx_tlm.append(data)
|
||||
client.write_points(influx_tlm)
|
||||
json_obj.append(data)
|
||||
|
||||
|
||||
def kaitai_to_dict(struct):
|
||||
"""Take a kaitai decode object and return a dict object"""
|
||||
structdict = struct.__dict__
|
||||
todict = {}
|
||||
for key, value in structdict.iteritems():
|
||||
if not key.startswith("_"): # kaitai objects and priv vars, skip
|
||||
if isinstance(value, basestring): # skip binary values
|
||||
try:
|
||||
value.decode('utf-8')
|
||||
todict[key] = value
|
||||
except UnicodeError:
|
||||
continue
|
||||
elif hasattr(value, '__dict__'):
|
||||
todict[key] = kaitai_to_dict(value) # recursion
|
||||
else:
|
||||
todict[key] = value
|
||||
return todict
|
||||
|
||||
|
||||
def write_influx(json_obj):
|
||||
"""Take a json object and send to influxdb."""
|
||||
client = InfluxDBClient(settings.INFLUX_HOST, settings.INFLUX_PORT,
|
||||
settings.INFLUX_USER, settings.INFLUX_PASS,
|
||||
settings.INFLUX_DB)
|
||||
client.write_points(json_obj)
|
||||
|
||||
|
||||
def decode_data(norad, period=None):
|
||||
"""Decode data for a satellite, with an option to limit the scope."""
|
||||
|
||||
sat = Satellite.objects.get(norad_cat_id=norad)
|
||||
if sat.has_telemetry_decoders:
|
||||
now = datetime.utcnow()
|
||||
|
|
@ -205,8 +204,10 @@ def decode_data(norad, period=None):
|
|||
if settings.USE_INFLUX:
|
||||
try:
|
||||
frame = decoder_class.from_bytes(bindata)
|
||||
# find kaitai_to_influx in utils.py
|
||||
kaitai_to_influx(frame, sat, tlmdecoder, obj)
|
||||
json_obj = []
|
||||
kaitai_to_json(kaitai_to_dict(frame), sat,
|
||||
tlmdecoder, obj, json_obj)
|
||||
write_influx(json_obj)
|
||||
obj.payload_decoded = 'influxdb'
|
||||
obj.is_decoded = True
|
||||
obj.save()
|
||||
|
|
@ -224,8 +225,10 @@ def decode_data(norad, period=None):
|
|||
obj.save()
|
||||
continue
|
||||
else:
|
||||
# find kaitai_to_json in utils.py
|
||||
obj.payload_decoded = kaitai_to_json(frame)
|
||||
json_obj = []
|
||||
kaitai_to_json(kaitai_to_dict(frame), sat,
|
||||
tlmdecoder, obj, json_obj)
|
||||
obj.payload_decoded = json_obj
|
||||
obj.is_decoded = True
|
||||
obj.save()
|
||||
break
|
||||
|
|
|
|||
|
|
@ -127,7 +127,7 @@ django-filter==1.1.0 \
|
|||
|
||||
# Astronomy
|
||||
https://gitlab.com/librespacefoundation/orbit/-/archive/master/orbit-master.zip \
|
||||
--hash=sha256:cf693a70e221f6217fd8f86343406c96792504d1d331cb1f2c724e6099062850
|
||||
--hash=sha256:c83a69991f13992c2864323caaf410474211f1bc8e53f2d6439d949211e9b29a
|
||||
|
||||
# Deps
|
||||
shortuuid==0.5.0 \
|
||||
|
|
|
|||
Loading…
Reference in New Issue