From 8e1cdaef80b3e09e184e01401b7c497c2cf07191 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Mon, 5 Jul 2021 21:53:43 +0200 Subject: [PATCH 01/11] contrib: Add satnogs waterfall tabulation helper --- .../satnogs_waterfall_tabulation_helper.py | 213 ++++++++++++++++++ contrib/settings.py | 12 + 2 files changed, 225 insertions(+) create mode 100755 contrib/satnogs_waterfall_tabulation_helper.py create mode 100644 contrib/settings.py diff --git a/contrib/satnogs_waterfall_tabulation_helper.py b/contrib/satnogs_waterfall_tabulation_helper.py new file mode 100755 index 0000000..0456656 --- /dev/null +++ b/contrib/satnogs_waterfall_tabulation_helper.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 + +import argparse +import requests +import ephem +import csv + + +from astropy.time import Time +from astropy import constants as const +from datetime import datetime, timedelta +from io import BytesIO +import matplotlib.pyplot as plt +import numpy as np +from PIL import Image + +import settings +from satnogs_api_client import fetch_observation_data, fetch_tle_of_observation + + +def tabulation_helper_dialog(lower_left, upper_right, + bandwidth, duration, + f_center, filename_out, + waterfall_matrix, + epoch_start, site_id, correction_method=None): + # Assumptions: x-axis - freqency, y-axis - time; up-time advance, right-freq advance + + # Derive conversion parameters + x_range = upper_right[0] - lower_left[0] + y_range = lower_left[1] - upper_right[1] + x_offset = int(lower_left[0] + 0.5 * x_range) + y_offset = lower_left[1] + + t_step = duration / y_range + f_step = bandwidth / x_range + + # Highlight center and calibration lines + waterfall_matrix[:,x_offset] = (255,0,0,255) + waterfall_matrix[y_offset,:] = (255,0,0,255) + waterfall_matrix[:,upper_right[0]] = (255,0,0,255) + waterfall_matrix[upper_right[1],:] = (255,0,0,255) + + + markers = {'x': [], 'y': [], 't': [], 'f': []} + + fig, ax = plt.subplots() + ax.imshow(waterfall_matrix) + marker_line, = ax.plot([0], [0], marker='.', c='k', zorder=100) + def on_mouse_press(event): + tb = plt.get_current_fig_manager().toolbar + if not (event.button==1 and event.inaxes and tb.mode == ''): + return + + markers['x'].append(event.xdata) + markers['y'].append(event.ydata) + markers['t'].append(epoch_start - (event.ydata - y_offset) * t_step) + markers['f'].append(f_step * (event.xdata - x_offset)) + print('x:{:.2f} y:{:.2f} --> {:7.0f} --> {} {:7.0f}'.format(markers['x'][-1], + markers['y'][-1], + markers['f'][-1] - f_center, + markers['t'][-1], + markers['f'][-1])) + update_plot_markers() + + def update_plot_markers(): + marker_line.set_xdata(markers['x']) + marker_line.set_ydata(markers['y']) + # ax.scatter(x=[int(event.xdata)], y=[int(event.ydata)], marker='.', c='k', zorder=100) + ax.figure.canvas.draw() + + def undo_track_point_selection(): + markers['x'] = markers['x'][:-1] + markers['y'] = markers['y'][:-1] + markers['t'] = markers['t'][:-1] + markers['f'] = markers['f'][:-1] + update_plot_markers() + + def write_strf_spectrum(filename, site_id, markers): + with open(filename, 'w') as f: + for t,freq in list(sorted(zip(markers['t'],markers['f']), key=lambda x: x[0])): + + if correction_method: + freq_recv = correction_method(t, freq) + else: + freq_recv = freq + line = '{:.6f}\t{:.2f}\t1.0\t{}\n'.format(Time(t).mjd, freq_recv, site_id) + print(line, end='') + f.write(line) + + def save_selected_track_points(): + write_strf_spectrum(filename_out, site_id, markers) + print('Stored {} selected track points in {}.'.format(len(markers['t']), filename_out)) + + def on_key(event): + if event.key == 'u': + undo_track_point_selection() + elif event.key == 'f': + save_selected_track_points() + + fig.canvas.mpl_connect('button_press_event', on_mouse_press) + fig.canvas.mpl_connect('key_press_event', on_key) + plt.tight_layout() + plt.show() + + +def read_sitestxt(path): + sites = {} + with open(path) as fp: + d = csv.DictReader([row for row in fp if not row.startswith('#')], + delimiter=' ', + fieldnames=["no", "id", "lat", "lon", "alt"], + restkey="observer", + skipinitialspace=True) + for line in d: + print(line) + sites[line["no"]] = {'lat': line["lat"], + 'lon': line["lon"], + 'sn': line["id"], + 'alt': line["alt"], + 'name': ' '.join(line["observer"])} + return sites + + +def add_station_to_sitestxt(station): + # Read the sites.txt + sites = read_sitestxt(settings.SITES_TXT) + if '7{}'.format(station['id']) in sites: + # Station is already available + return + else: + with open(settings.SITES_TXT, 'a') as f: + f.write('7{} {} {} {} {} {}\n'.format(station['id'], + 'SN', + station['lat'], + station['lng'], + station['alt'], + station['name'])) + + +def tabulation_helper(observation_id): + # Fetch waterfall image and observation data + observation = fetch_observation_data([observation_id])[0] + tle = fetch_tle_of_observation(observation_id) + result = requests.get(observation['waterfall']) + image = Image.open(BytesIO(result.content)) + waterfall_matrix = np.array(image) + + epoch_start = datetime.strptime(observation['start'], '%Y-%m-%dT%H:%M:%SZ') + epoch_end = datetime.strptime(observation['end'], '%Y-%m-%dT%H:%M:%SZ') + site_id = int("7" + str(observation['ground_station'])) + + station = {'lat': observation['station_lat'], + 'lng': observation['station_lng'], + 'alt': observation['station_alt'], + 'name': "SatNOGS No.{}".format(observation['ground_station']), + 'id': observation['ground_station']} + + # Store TLE to file + with open('{}/{}.txt'.format(settings.TLE_DIR, observation_id), 'w') as f: + f.write('0 OBJECT\n') + for line in tle: + f.write(f'{line}\n') + + # Add SatNOGS station to sites.txt + add_station_to_sitestxt(station) + + # Set image parameters + lower_left = (66, 1553) + upper_right = (686, 13) + + # Set data parameters + bandwidth = 48e3 # Hz + duration = epoch_end - epoch_start + f = observation['transmitter_downlink_low'] + drift = observation['transmitter_downlink_drift'] + if drift: + f_center = f * (1 + drift / 1e9) + else: + f_center = f + filename_out = '{}/{}.dat'.format(settings.OBS_DIR, observation_id) + + # Initialize SGP4 propagator / pyephem + satellite = ephem.readtle('sat', tle[0], tle[1]) + observer = ephem.Observer() + observer.lat = str(station['lat']) + observer.lon = str(station['lng']) + observer.elevation = station['alt'] + + + def remove_doppler_correction(t, freq): + # Remove the doppler correction + observer.date = t + satellite.compute(observer) + v = satellite.range_velocity + df = f_center * v / const.c.value + return f_center + freq - df + + tabulation_helper_dialog(lower_left, upper_right, + bandwidth, duration, + f_center, filename_out, + waterfall_matrix, + epoch_start, site_id, correction_method=remove_doppler_correction) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Interactive helper to tabulate signals from SatNOGS waterfalls.') + parser.add_argument('observation_ids', metavar='ID', type=int, nargs='+', + help='SatNOGS Observation ID') + + args = parser.parse_args() + + for observation_id in args.observation_ids: + tabulation_helper(observation_id) diff --git a/contrib/settings.py b/contrib/settings.py new file mode 100644 index 0000000..ed30b38 --- /dev/null +++ b/contrib/settings.py @@ -0,0 +1,12 @@ +from decouple import config + +# Path to TLE +# Filename convention: {TLE_DIR}/{observation_id}.txt +TLE_DIR = config('SATNOGS_TLE_DIR') + +# absulute frequency measurement storage +# Filename convention: {OBS_DIR}/{observation_id}.txt +OBS_DIR = config('SATNOGS_OBS_DIR') + +# SATTOOLS/STRF/STVID sites.txt file +SITES_TXT = config('SATNOGS_SITES_TXT') From 85d0555b1e6268811c81aa8debed9fd6f919af4c Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Mon, 5 Jul 2021 22:18:30 +0200 Subject: [PATCH 02/11] Add GUIDE.md --- GUIDE.md | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 GUIDE.md diff --git a/GUIDE.md b/GUIDE.md new file mode 100644 index 0000000..3510b9a --- /dev/null +++ b/GUIDE.md @@ -0,0 +1,73 @@ +# Guide for RF doppler analysis from SatNOGS Observation Waterfall Images + +Forum post: https://community.libre.space/t/new-software-satnogs-waterfall-tabulation-helper/4380 + +This tool evolved from the need to tabulate doppler data from the lunar +Change-4 probe, +see . + +## Installation +Install dependencies +``` +pip install -r contrib/requirements.txt +``` + +# First Setup +Choose a folder where TLEs are stored and +a folder where RF doppler observatons (`.dat`-files) are stored. + +For now the paths are configured via environment variables, +so make sure to set them correctly before each usage. + +Example: +``` +# Filename convention: {TLE_DIR}/{observation_id}.txt +SATNOGS_TLE_DIR="./data/tles" + +# absulute frequency measurement storage +# Filename convention: {OBS_DIR}/{observation_id}.txt +SATNOGS_OBS_DIR="./data/obs" + +# SATTOOLS/STRF/STVID sites.txt file +SATNOGS_SITES_TXT="./data/sites.txt" + +mkdir -p $SATNOGS_TLE_DIR +mkdir -p $SATNOGS_OBS_DIR +``` + +## Usage +0. Make sure the (3) env variables are set. +1. Choose SatNOGS Observation ID from network and run the tabulation helper + + ``` + ./contrib/satnogs_waterfall_tabulation_helper.py 1102230 + ``` + + An interactive plot will show up. + Clicking inside the plot will add a signal marker. + If you are finished with adding signal markers, + save the signal markers using the keyboard shortcut `f`. + + Custom keyboard shortcuts: + + - u - undo last signal marker + - f - save the signal markers in an strf-compatible file + + Useful Matplotlib navigation keyboard shortcuts (documentation): + + p - toggle 'Pan/Zoom' modus + o - toggle 'Zoom-to-rect' modus + h - Home/Reset (view) + c - Back (view) + +2. Run rffit for orbit fitting, e.g. + ``` + ./rffit -d $SATNOGS_OBS_DIR/1102230.dat -i 44356 -c $SATNOGS_TLE_DIR/1102230.txt -s 7669 + ``` + +## Known issues +- A site id of the form `7{station_id}` is automatically assigned and written to + the `sites.txt` (e.g. station 669 should get `7669` assigned). + Only SatNOGS stations <999 are supported, as the strf sites.txt parse only allows + 4-digit site ids. In case of problems, choose a free site id and manually correct the + doppler obs (`.dat`-files, last colum). From 3591f7ce78cadccaea728867cc8953e2501929b1 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Mon, 5 Jul 2021 22:18:41 +0200 Subject: [PATCH 03/11] contrib: Add requirements.txt --- contrib/requirements.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 contrib/requirements.txt diff --git a/contrib/requirements.txt b/contrib/requirements.txt new file mode 100644 index 0000000..358f2c7 --- /dev/null +++ b/contrib/requirements.txt @@ -0,0 +1,8 @@ +astropy +ephem +matplotlib +numpy +Pillow +python-decouple +requests +git+https://gitlab.com/librespacefoundation/satnogs/python-satnogs-api.git@e20a7d3c From 1efbc09f6ccdb32e51fdba2f2d446507cedc7fa2 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 4 Sep 2021 00:43:02 +0200 Subject: [PATCH 04/11] Add env-dist configuration example Signed-off-by: Fabian P. Schmidt --- .gitignore | 3 +++ env-dist | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100644 env-dist diff --git a/.gitignore b/.gitignore index 3ca5119..ba41636 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ /rfinfo /rfplot /rfpng +/rfinfo + +.env diff --git a/env-dist b/env-dist new file mode 100644 index 0000000..95114bf --- /dev/null +++ b/env-dist @@ -0,0 +1,8 @@ +# "New" Python STRF environment variables + +SATNOGS_DIR=/my/path/to/satnogs/data + +# SatNOGS Waterfall Tabulation Helper +SATNOGS_TLE_DIR=$SATNOGS_DIR/tles +SATNOGS_OBS_DIR=$SATNOGS_DIR/doppler_obs +SATNOGS_SITES_TXT=$SATNOGS_DIR/sites.txt From 8577efce40d9c80e481182145543788041dfa545 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 4 Sep 2021 00:44:11 +0200 Subject: [PATCH 05/11] contrib: Add SatNOGS artifacts download helpers Signed-off-by: Fabian P. Schmidt --- contrib/download_satnogs_artifact.py | 77 ++++++++++++++++++++++++++ contrib/find_good_satnogs_artifacts.py | 54 ++++++++++++++++++ contrib/settings.py | 5 ++ env-dist | 6 ++ 4 files changed, 142 insertions(+) create mode 100755 contrib/download_satnogs_artifact.py create mode 100755 contrib/find_good_satnogs_artifacts.py diff --git a/contrib/download_satnogs_artifact.py b/contrib/download_satnogs_artifact.py new file mode 100755 index 0000000..50d4447 --- /dev/null +++ b/contrib/download_satnogs_artifact.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import argparse +import tempfile +import sys +import logging +import requests +import settings + +from urllib.parse import urljoin +from pprint import pprint + + +logger = logging.getLogger(__name__) + +def fetch_artifact_metadata(network_obs_id): + url = urljoin(settings.SATNOGS_DB_API_URL, 'artifacts/',) + params = {'network_obs_id': network_obs_id} + headers = {'Authorization': 'Token {0}'.format(settings.SATNOGS_DB_API_TOKEN)} + + response = requests.get(url, + params=params, + headers=headers, + timeout=10) + response.raise_for_status() + + return response.json() + + +def fetch_artifact(url, artifact_filename): + headers = {'Authorization': 'Token {0}'.format(settings.SATNOGS_DB_API_TOKEN)} + + response = requests.get(url, + headers=headers, + stream=True, + timeout=10) + response.raise_for_status() + + with open(artifact_filename, 'wb') as fname: + for chunk in response.iter_content(chunk_size=1024): + fname.write(chunk) + + +def download_artifact(observation_id): + try: + artifact_metadata = fetch_artifact_metadata(network_obs_id=observation_id) + except requests.HTTPError: + print('An error occurred trying to GET artifact metadata from db') + return + + if not len(artifact_metadata): + print('No artifact found in db for network_obs_id {}'.format(network_obs_id)) + return + + print("Artifact Metadata for Observation #{} found.".format(observation_id)) + + try: + artifact_file_url = artifact_metadata[0]['artifact_file'] + artifact_file = tempfile.NamedTemporaryFile(delete=False) + fetch_artifact(artifact_file_url, artifact_file.name) + print("Artifact for Observation #{} saved in '{}'".format(observation_id, artifact_file.name)) + except requests.HTTPError: + print('Download failed for {}'.format(artifact_file_url)) + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Download SatNOGS Artifacts from SatNOGS DB.') + parser.add_argument('observation_ids', metavar='ID', type=int, nargs='+', + help='SatNOGS Observation ID') + + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + for observation_id in args.observation_ids: + download_artifact(observation_id) diff --git a/contrib/find_good_satnogs_artifacts.py b/contrib/find_good_satnogs_artifacts.py new file mode 100755 index 0000000..4fb9083 --- /dev/null +++ b/contrib/find_good_satnogs_artifacts.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import sys +import logging +import requests +import settings + +from urllib.parse import urljoin +from pprint import pprint + +from satnogs_api_client import fetch_observation_data, fetch_tle_of_observation + +logger = logging.getLogger(__name__) + + +def fetch_latest_artifacts_metadata(): + url = urljoin(settings.SATNOGS_DB_API_URL, 'artifacts/',) + params = {} + headers = {'Authorization': 'Token {0}'.format(settings.SATNOGS_DB_API_TOKEN)} + + try: + response = requests.get(url, + params=params, + headers=headers, + timeout=10) + response.raise_for_status() + except (requests.ConnectionError, requests.Timeout, requests.TooManyRedirects): + logger.exception('An error occurred trying to GET artifact metadata from db') + + artifacts_metadata = response.json() + + if not len(artifacts_metadata): + logger.info('No artifacts found in db') + sys.exit(-1) + + return artifacts_metadata + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + # Fetch list of artifacts + artifacts_metadata = fetch_latest_artifacts_metadata() + + observation_ids = [artifact['network_obs_id'] for artifact in artifacts_metadata] + + # Load corresponding obs from network + observations = fetch_observation_data(sorted(observation_ids, reverse=True)) + + # Filter by good status + for observation in observations: + if not observation['vetted_status'] == 'good': + pass + + print("{}/observations/{}/".format(settings.SATNOGS_NETWORK_API_URL[:-5], observation['id'])) diff --git a/contrib/settings.py b/contrib/settings.py index ed30b38..9abbf7b 100644 --- a/contrib/settings.py +++ b/contrib/settings.py @@ -10,3 +10,8 @@ OBS_DIR = config('SATNOGS_OBS_DIR') # SATTOOLS/STRF/STVID sites.txt file SITES_TXT = config('SATNOGS_SITES_TXT') + + +SATNOGS_NETWORK_API_URL = config('SATNOGS_NETWORK_API_URL') +SATNOGS_DB_API_URL = config('SATNOGS_DB_API_URL') +SATNOGS_DB_API_TOKEN = config('SATNOGS_DB_API_TOKEN') diff --git a/env-dist b/env-dist index 95114bf..f0da9a2 100644 --- a/env-dist +++ b/env-dist @@ -6,3 +6,9 @@ SATNOGS_DIR=/my/path/to/satnogs/data SATNOGS_TLE_DIR=$SATNOGS_DIR/tles SATNOGS_OBS_DIR=$SATNOGS_DIR/doppler_obs SATNOGS_SITES_TXT=$SATNOGS_DIR/sites.txt + + +# SatNOGS Artifacts Helpers +SATNOGS_NETWORK_API_URL=https://network.satnogs.org/api/ +SATNOGS_DB_API_URL=https://db.satnogs.org/api/ +SATNOGS_DB_API_TOKEN=your-db-api-token From ef08aa056fb453455381cecb5c0026293aeb5754 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 4 Sep 2021 00:52:39 +0200 Subject: [PATCH 06/11] contrib: Add README Signed-off-by: Fabian P. Schmidt --- contrib/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 contrib/README.md diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000..0e2d6cb --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,15 @@ +# STRF contrib Scripts + +## Installation + +- Create Pyhton virtual environment and install dependencies via pip: + ``` + mkvirtualenv strf + pip insatll -r contrib/requirements.txt + ``` + +- Create initial configuration from `env-dist` + ``` + cp env-dist .env + ``` +- Configure the data paths (they must already exist!) and add your SatNOGS DB API Token in `.env` From 6c1f859c6845dbd009949a92e849d0593787bd22 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 4 Sep 2021 01:03:25 +0200 Subject: [PATCH 07/11] contrib/README: Add artifacts helper section Signed-off-by: Fabian P. Schmidt --- contrib/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/contrib/README.md b/contrib/README.md index 0e2d6cb..c502e0e 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -13,3 +13,21 @@ cp env-dist .env ``` - Configure the data paths (they must already exist!) and add your SatNOGS DB API Token in `.env` + +## Find good observations with SatNOGS artifacts +``` +$ ./contrib/find_good_satnogs_artifacts.py +``` + +## Download SatNOGS Artifacts +``` +$ ./contrib/download_satnogs_artifact.py 2786575 +Artifact Metadata for Observation #2786575 found. +Download failed for https://db-satnogs.freetls.fastly.net/media/artifacts/b4975058-04eb-4ab7-9c40-9bcce76d94db.h5 +``` + +``` +$ ./contrib/download_satnogs_artifact.py 4443137 +Artifact Metadata for Observation #4443137 found. +Artifact for Observation #4443137 saved in '/tmp/tmp1rdpnz_k' +``` From ce46a2cb2a39ed9622b7432ee42ed78186e4ebc0 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 20 Nov 2021 06:39:40 +0100 Subject: [PATCH 08/11] Rename OBS_DIR to SATNOGS_DOPPLER_OBS_DIR Signed-off-by: Fabian P. Schmidt --- GUIDE.md | 10 +++++----- contrib/settings.py | 6 ++++-- env-dist | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/GUIDE.md b/GUIDE.md index 3510b9a..ebc024b 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -25,14 +25,14 @@ Example: SATNOGS_TLE_DIR="./data/tles" # absulute frequency measurement storage -# Filename convention: {OBS_DIR}/{observation_id}.txt -SATNOGS_OBS_DIR="./data/obs" +# Filename convention: {DOPPLER_OBS_DIR}/{observation_id}.txt +SATNOGS_DOPPLER_OBS_DIR="./data/obs" # SATTOOLS/STRF/STVID sites.txt file SATNOGS_SITES_TXT="./data/sites.txt" -mkdir -p $SATNOGS_TLE_DIR -mkdir -p $SATNOGS_OBS_DIR +mkdir -p $SATNOGS_TLE_DIR +mkdir -p $SATNOGS_DOPPLER_OBS_DIR ``` ## Usage @@ -62,7 +62,7 @@ mkdir -p $SATNOGS_OBS_DIR 2. Run rffit for orbit fitting, e.g. ``` - ./rffit -d $SATNOGS_OBS_DIR/1102230.dat -i 44356 -c $SATNOGS_TLE_DIR/1102230.txt -s 7669 + ./rffit -d $SATNOGS_DOPPLER_OBS_DIR/1102230.dat -i 44356 -c $SATNOGS_TLE_DIR/1102230.txt -s 7669 ``` ## Known issues diff --git a/contrib/settings.py b/contrib/settings.py index 9abbf7b..e88ce8e 100644 --- a/contrib/settings.py +++ b/contrib/settings.py @@ -5,8 +5,10 @@ from decouple import config TLE_DIR = config('SATNOGS_TLE_DIR') # absulute frequency measurement storage -# Filename convention: {OBS_DIR}/{observation_id}.txt -OBS_DIR = config('SATNOGS_OBS_DIR') +# Filename convention: {DOPPLER_OBS_DIR}/{observation_id}.dat +DOPPLER_OBS_DIR = config('SATNOGS_DOPPLER_OBS_DIR') +# legacy name: +OBS_DIR = DOPPLER_OBS_DIR # SATTOOLS/STRF/STVID sites.txt file SITES_TXT = config('SATNOGS_SITES_TXT') diff --git a/env-dist b/env-dist index f0da9a2..855c80b 100644 --- a/env-dist +++ b/env-dist @@ -4,7 +4,7 @@ SATNOGS_DIR=/my/path/to/satnogs/data # SatNOGS Waterfall Tabulation Helper SATNOGS_TLE_DIR=$SATNOGS_DIR/tles -SATNOGS_OBS_DIR=$SATNOGS_DIR/doppler_obs +SATNOGS_DOPPLER_OBS_DIR=$SATNOGS_DIR/doppler_obs SATNOGS_SITES_TXT=$SATNOGS_DIR/sites.txt From 098406bdd60e52d6a69446a9a802e98e9439f928 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 20 Nov 2021 06:38:23 +0100 Subject: [PATCH 09/11] Add SatNOGS Artifact Analysis (WIP) Signed-off-by: Fabian P. Schmidt --- contrib/README.md | 41 ++++++- contrib/analyze_artifact.py | 158 +++++++++++++++++++++++++++ contrib/download_satnogs_artifact.py | 18 ++- contrib/download_satnogs_tle.py | 26 +++++ contrib/requirements.txt | 4 +- contrib/test_artifact_download.sh | 7 ++ 6 files changed, 242 insertions(+), 12 deletions(-) create mode 100755 contrib/analyze_artifact.py create mode 100755 contrib/download_satnogs_tle.py create mode 100755 contrib/test_artifact_download.sh diff --git a/contrib/README.md b/contrib/README.md index c502e0e..f29a5cb 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -20,14 +20,43 @@ $ ./contrib/find_good_satnogs_artifacts.py ``` ## Download SatNOGS Artifacts + ``` -$ ./contrib/download_satnogs_artifact.py 2786575 -Artifact Metadata for Observation #2786575 found. -Download failed for https://db-satnogs.freetls.fastly.net/media/artifacts/b4975058-04eb-4ab7-9c40-9bcce76d94db.h5 +$ ./contrib/download_satnogs_artifact.py 4950356 +Artifact Metadata for Observation #4950356 found. +Artifact saved in /home/pi/data/artifacts/4950356.h5 +``` + +## Download SatNOGS Observation TLEs + +To add the TLE used in a specific SatNOGS Observation to your catalog, the following command can be used: +``` +$ ./contrib/download_satnogs_tle.py 4950356 +TLE saved in /home/pi/data/tles/satnogs/4950356.txt +``` + +## Plot SatNOGS Artifacts + +This can be done using [spectranalysis](https://github.com/kerel-fs/spectranalysis). + +## Ultra-Short Analysis Guide +``` +OBSERVATION_ID=4950356 +./contrib/download_satnogs_tle.py $OBSERVATION_ID +./contrib/download_satnogs_artifact.py $OBSERVATION_ID +./contrib/analyze_artifact.py --observation_id $OBSERVATION_ID --site_id 977 +rffit -d "$SATNOGS_DOPPLER_OBS_DIR/$OBSERVATION_ID.dat" -c "$SATNOGS_TLE_DIR/$OBSERVATION_ID.txt" -i 27844 -s 977 ``` ``` -$ ./contrib/download_satnogs_artifact.py 4443137 -Artifact Metadata for Observation #4443137 found. -Artifact for Observation #4443137 saved in '/tmp/tmp1rdpnz_k' +$ ./contrib/download_satnogs_tle.py 4950356 +TLE saved in /mnt/old_home/kerel/c4/satnogs/data/tles/satnogs/4950356.txt +$ ./contrib/download_satnogs_artifact.py 4950356 +Artifact Metadata for Observation #4950356 found. +Artifact saved in /mnt/old_home/kerel/c4/satnogs/data/artifacts/4950356.h5 +$ ./contrib/analyze_artifact.py --observation_id 4950356 --site_id 977 +Load /mnt/old_home/kerel/c4/satnogs/data/artifacts/4950356.h5 +Extract measurements... +Data written in /mnt/old_home/kerel/c4/satnogs/data/doppler_obs/4950356.dat +$ rffit -d ${SATNOGS_DOPPLER_OBS_DIR}/4950356.dat -c ${SATNOGS_TLE_DIR}/4950356.txt -i 27844 -s 977 ``` diff --git a/contrib/analyze_artifact.py b/contrib/analyze_artifact.py new file mode 100755 index 0000000..cce8b20 --- /dev/null +++ b/contrib/analyze_artifact.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 + +from astropy.time import Time + +from datetime import datetime, timedelta + +import argparse +import ephem +import h5py +import json +import os + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd + + +def load_artifact(filename): + hdf5_file = h5py.File(filename, 'r') + if hdf5_file.attrs['artifact_version'] != 2: + print("unsupported artifact version {}".format(hdf5_file.attrs['artifact_version'])) + # return + wf = hdf5_file.get('waterfall') + data = (np.array(wf['data']) * np.array(wf['scale']) + np.array(wf['offset'])) + metadata = json.loads(hdf5_file.attrs['metadata']) + + return hdf5_file, wf, data, metadata + +def extract_peaks(data): + """ + Returns + ------- + points: (time_index, freq_index) + measurements: measurement tuples of relative time in seconds and relative frequency in hertz + """ + + snr = (np.max(data, axis=1) - np.mean(data, axis=1)) / np.std(data, axis=1) + + # Integrate each channel along the whole observation, + # This filter helps under the assumption of minimal Doppler deviaion + snr_integrated = np.zeros(data[0].shape) + for row in data: + snr_integrated += (row - np.mean(row)) / np.std(row) + snr_integrated = pd.Series(snr_integrated/len(snr_integrated)) + + SNR_CUTOFF = 4 + CHANNEL_WINDOW_SIZE = 4 + channel_mask = (snr_integrated.diff().abs() / snr_integrated.diff().std() > SNR_CUTOFF).rolling(CHANNEL_WINDOW_SIZE, min_periods=1).max() + rows=[] + for i, row in enumerate(channel_mask): + if not row: + continue + rows.append(i) + + # Select only maximum values in masked channels + points = [] + for i,x in enumerate(np.argmax(data, axis=1)): + if not x in rows: + continue + points.append([i, x]) + points = np.array(points) + measurements = np.vstack((wf['relative_time'][points[:,0]], + [wf['frequency'][x] for x in points[:,1]])) + return snr, measurements, points + +def plot_measurements(wf, data, measurements, snr): + plt.plot(wf['relative_time'][:], + [wf['frequency'][x] for x in np.argmax(data[:], axis=1)], + '.', + label='all') + plt.plot(measurements[:,0], + measurements[:,1], + '.', + label='automatic channel mask') + plt.plot(wf['relative_time'][2.4 < snr], + [wf['frequency'][x] for x in np.argmax(data[2.4 < snr], axis=1)], + '.', + label="SNR > 2.4") + plt.legend() + plt.title("Observation #4991792 - Maximum Values") + plt.grid() + plt.xlabel('Elapsed Time / s') + plt.ylabel('rel. Frequency / kHz') + plt.show() + + + +def dedoppler(measurements, metadata): + tle = metadata['tle'].split('\n') + start_time = datetime.strptime(wf.attrs['start_time'].decode('ascii'), + '%Y-%m-%dT%H:%M:%S.%fZ') + f_center = float(metadata['frequency']) + + # Initialize SGP4 propagator / pyephem + satellite = ephem.readtle('sat', tle[1], tle[2]) + observer = ephem.Observer() + observer.lat = str(metadata['location']['latitude']) + observer.lon = str(metadata['location']['longitude']) + observer.elevation = metadata['location']['altitude'] + + def remove_doppler_correction(t, freq): + """ + Arguments + --------- + t - float: Time in seconds + freq - float: Relative Frequency in herz + """ + observer.date = t + satellite.compute(observer) + v = satellite.range_velocity + df = f_center * v / ephem.c + return f_center + freq - df*2 + + output = [] + for dt,df in measurements.T: + t = start_time + timedelta(seconds=dt) + f = f_center + df + freq_recv = remove_doppler_correction(t, f) + output.append((t, freq_recv)) + return output + +def save_rffit_data(filename, measurements, site_id): + with open(filename, 'w') as file_out: + for time, freq in measurements: + line = '{:.6f}\t{:.2f}\t1.0\t{}\n'.format(Time(time).mjd, freq, site_id) + file_out.write(line) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Analyze SatNOGS Artifact.') + parser.add_argument('--observation_id', type=str, + help='SatNOGS Observation ID') + parser.add_argument('--filename', type=str, + help='SatNOGS Artifacts File') + parser.add_argument('--output_file', type=str, + help='STRF-compatible output file') + parser.add_argument('--site_id', type=int, required=True, + help='STRF Site ID') + args = parser.parse_args() + + if args.observation_id: + # Use canonic file paths, ignoring `--filename` and `--output_path` + filename = '{}/{}.h5'.format(os.getenv('SATNOGS_ARTIFACTS_DIR'), args.observation_id) + output_file = '{}/{}.dat'.format(os.getenv('SATNOGS_DOPPLER_OBS_DIR'), args.observation_id) + else: + if not any([args.filename, args.output_file]): + print('ERROR: Missing arguments') + filename = args.filename + output_file = args.output_file + + print('Load {}'.format(filename)) + hdf5_file, wf, data, metadata = load_artifact(filename) + + print('Extract measurements...') + snr, measurements, points = extract_peaks(data) + # plot_measurements(wf, data, measurements, snr) + m2 = dedoppler(measurements, metadata) + save_rffit_data(output_file, m2, site_id=args.site_id) + print('Data written in {}'.format(output_file)) diff --git a/contrib/download_satnogs_artifact.py b/contrib/download_satnogs_artifact.py index 50d4447..81c9866 100755 --- a/contrib/download_satnogs_artifact.py +++ b/contrib/download_satnogs_artifact.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import argparse -import tempfile -import sys import logging import requests import settings +import shutil +import sys +import tempfile +import os from urllib.parse import urljoin from pprint import pprint @@ -41,7 +43,7 @@ def fetch_artifact(url, artifact_filename): fname.write(chunk) -def download_artifact(observation_id): +def download_artifact(observation_id, filename): try: artifact_metadata = fetch_artifact_metadata(network_obs_id=observation_id) except requests.HTTPError: @@ -57,8 +59,13 @@ def download_artifact(observation_id): try: artifact_file_url = artifact_metadata[0]['artifact_file'] artifact_file = tempfile.NamedTemporaryFile(delete=False) + fetch_artifact(artifact_file_url, artifact_file.name) - print("Artifact for Observation #{} saved in '{}'".format(observation_id, artifact_file.name)) + + artifact_file.close() + shutil.copy(artifact_file.name, filename) + os.remove(artifact_file.name) + print("Artifact saved in {}".format(filename)) except requests.HTTPError: print('Download failed for {}'.format(artifact_file_url)) return @@ -74,4 +81,5 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) for observation_id in args.observation_ids: - download_artifact(observation_id) + download_artifact(observation_id, + '{}/{}.h5'.format(os.getenv('SATNOGS_ARTIFACTS_DIR'), observation_id)) diff --git a/contrib/download_satnogs_tle.py b/contrib/download_satnogs_tle.py new file mode 100755 index 0000000..8cc460a --- /dev/null +++ b/contrib/download_satnogs_tle.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +import argparse +import os + +from satnogs_api_client import fetch_observation_data + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Download TLE from a specific Observation in SatNOGS Network.') + parser.add_argument('observation_id', type=int, + help='SatNOGS Observation ID') + args = parser.parse_args() + + obs = fetch_observation_data([args.observation_id])[0] + + filename = '{}/{}.txt'.format(os.getenv('SATNOGS_TLE_DIR'), args.observation_id) + + with open(filename, 'w') as f: + f.write(obs['tle0']) + f.write('\n') + f.write(obs['tle1']) + f.write('\n') + f.write(obs['tle2']) + f.write('\n') + print("TLE saved in {}".format(filename)) diff --git a/contrib/requirements.txt b/contrib/requirements.txt index 358f2c7..eda36ad 100644 --- a/contrib/requirements.txt +++ b/contrib/requirements.txt @@ -1,8 +1,10 @@ astropy -ephem matplotlib numpy Pillow python-decouple requests git+https://gitlab.com/librespacefoundation/satnogs/python-satnogs-api.git@e20a7d3c +h5py~=3.6.0 +pandas~=1.3.4 +ephem~=4.3.1 diff --git a/contrib/test_artifact_download.sh b/contrib/test_artifact_download.sh new file mode 100755 index 0000000..395253f --- /dev/null +++ b/contrib/test_artifact_download.sh @@ -0,0 +1,7 @@ +#!/usr/bin/bash + +DB_API_TOKEN="4f20a493a3f5fd85074d61db944365bf94987613" +URL="https://db-satnogs.freetls.fastly.net/media/artifacts/b4975058-04eb-4ab7-9c40-9bcce76d94db.h5" +echo "Authorization: Token $DB_API_TOKEN" +curl -H "Authorization: Token $DB_API_TOKEN" "$URL" + From 720fba5dfaf085d54536654b13d9a9078720accd Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 20 Nov 2021 08:20:22 +0100 Subject: [PATCH 10/11] Always plot measurement results Signed-off-by: Fabian P. Schmidt --- contrib/analyze_artifact.py | 50 +++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/contrib/analyze_artifact.py b/contrib/analyze_artifact.py index cce8b20..fff9333 100755 --- a/contrib/analyze_artifact.py +++ b/contrib/analyze_artifact.py @@ -9,6 +9,7 @@ import ephem import h5py import json import os +import sys import matplotlib.pyplot as plt import numpy as np @@ -52,32 +53,40 @@ def extract_peaks(data): continue rows.append(i) - # Select only maximum values in masked channels - points = [] + points_all = [] + points_filtered = [] for i,x in enumerate(np.argmax(data, axis=1)): - if not x in rows: - continue - points.append([i, x]) - points = np.array(points) - measurements = np.vstack((wf['relative_time'][points[:,0]], - [wf['frequency'][x] for x in points[:,1]])) - return snr, measurements, points + points_all.append([i, x]) + if x in rows: + points_filtered.append([i, x]) -def plot_measurements(wf, data, measurements, snr): + points_all = np.array(points_all) + points_filtered = np.array(points_filtered) + + if len(points_filtered) == 0: + print("WARNING: No measurement passed filter, loosening requirements now.") + points_filtered = points_all + measurements = None + + measurements = np.vstack((wf['relative_time'][points_filtered[:,0]], + [wf['frequency'][x] for x in points_filtered[:,1]])) + return snr, measurements, points_filtered + +def plot_measurements_all(wf, data): plt.plot(wf['relative_time'][:], [wf['frequency'][x] for x in np.argmax(data[:], axis=1)], '.', label='all') - plt.plot(measurements[:,0], - measurements[:,1], + +def plot_measurements_selected(measurements): + plt.plot(measurements[0,:], + measurements[1,:], '.', - label='automatic channel mask') - plt.plot(wf['relative_time'][2.4 < snr], - [wf['frequency'][x] for x in np.argmax(data[2.4 < snr], axis=1)], - '.', - label="SNR > 2.4") + label='filtered') + +def plot_legend(observation_id): plt.legend() - plt.title("Observation #4991792 - Maximum Values") + plt.title("Observation #{} - Maximum Values".format(observation_id)) plt.grid() plt.xlabel('Elapsed Time / s') plt.ylabel('rel. Frequency / kHz') @@ -152,7 +161,10 @@ if __name__ == '__main__': print('Extract measurements...') snr, measurements, points = extract_peaks(data) - # plot_measurements(wf, data, measurements, snr) + plot_measurements_all(wf, data) + plot_measurements_selected(measurements) + plot_legend(args.observation_id) + m2 = dedoppler(measurements, metadata) save_rffit_data(output_file, m2, site_id=args.site_id) print('Data written in {}'.format(output_file)) From c76a0cbf8a787fbad57f5e9dafdd8774b8e5ab2d Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 20 Nov 2021 08:24:00 +0100 Subject: [PATCH 11/11] Add docs on config variables Signed-off-by: Fabian P. Schmidt --- config.sh | 1 + contrib/README.md | 3 +++ env-dist | 9 ++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 120000 config.sh diff --git a/config.sh b/config.sh new file mode 120000 index 0000000..3a968fd --- /dev/null +++ b/config.sh @@ -0,0 +1 @@ +../sattools/config.sh \ No newline at end of file diff --git a/contrib/README.md b/contrib/README.md index f29a5cb..5c8c371 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -40,7 +40,10 @@ TLE saved in /home/pi/data/tles/satnogs/4950356.txt This can be done using [spectranalysis](https://github.com/kerel-fs/spectranalysis). ## Ultra-Short Analysis Guide + ``` +source .env + OBSERVATION_ID=4950356 ./contrib/download_satnogs_tle.py $OBSERVATION_ID ./contrib/download_satnogs_artifact.py $OBSERVATION_ID diff --git a/env-dist b/env-dist index 855c80b..4a841c0 100644 --- a/env-dist +++ b/env-dist @@ -1,14 +1,17 @@ # "New" Python STRF environment variables -SATNOGS_DIR=/my/path/to/satnogs/data +# SATNOGS_DIR is used by config.sh for STRF configuration +SATNOGS_DIR=/home/pi/satnogs_data # SatNOGS Waterfall Tabulation Helper -SATNOGS_TLE_DIR=$SATNOGS_DIR/tles -SATNOGS_DOPPLER_OBS_DIR=$SATNOGS_DIR/doppler_obs +export SATNOGS_TLE_DIR=/home/pi/satnogs_data/tles/satnogs +export SATNOGS_ARTIFACTS_DIR=/home/pi/satnogs_data/artifacts +export SATNOGS_DOPPLER_OBS_DIR=/home/pi/satnogs_data/doppler_obs SATNOGS_SITES_TXT=$SATNOGS_DIR/sites.txt # SatNOGS Artifacts Helpers SATNOGS_NETWORK_API_URL=https://network.satnogs.org/api/ SATNOGS_DB_API_URL=https://db.satnogs.org/api/ + SATNOGS_DB_API_TOKEN=your-db-api-token