From 8577efce40d9c80e481182145543788041dfa545 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 4 Sep 2021 00:44:11 +0200 Subject: [PATCH] contrib: Add SatNOGS artifacts download helpers Signed-off-by: Fabian P. Schmidt --- contrib/download_satnogs_artifact.py | 77 ++++++++++++++++++++++++++ contrib/find_good_satnogs_artifacts.py | 54 ++++++++++++++++++ contrib/settings.py | 5 ++ env-dist | 6 ++ 4 files changed, 142 insertions(+) create mode 100755 contrib/download_satnogs_artifact.py create mode 100755 contrib/find_good_satnogs_artifacts.py diff --git a/contrib/download_satnogs_artifact.py b/contrib/download_satnogs_artifact.py new file mode 100755 index 0000000..50d4447 --- /dev/null +++ b/contrib/download_satnogs_artifact.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +import argparse +import tempfile +import sys +import logging +import requests +import settings + +from urllib.parse import urljoin +from pprint import pprint + + +logger = logging.getLogger(__name__) + +def fetch_artifact_metadata(network_obs_id): + url = urljoin(settings.SATNOGS_DB_API_URL, 'artifacts/',) + params = {'network_obs_id': network_obs_id} + headers = {'Authorization': 'Token {0}'.format(settings.SATNOGS_DB_API_TOKEN)} + + response = requests.get(url, + params=params, + headers=headers, + timeout=10) + response.raise_for_status() + + return response.json() + + +def fetch_artifact(url, artifact_filename): + headers = {'Authorization': 'Token {0}'.format(settings.SATNOGS_DB_API_TOKEN)} + + response = requests.get(url, + headers=headers, + stream=True, + timeout=10) + response.raise_for_status() + + with open(artifact_filename, 'wb') as fname: + for chunk in response.iter_content(chunk_size=1024): + fname.write(chunk) + + +def download_artifact(observation_id): + try: + artifact_metadata = fetch_artifact_metadata(network_obs_id=observation_id) + except requests.HTTPError: + print('An error occurred trying to GET artifact metadata from db') + return + + if not len(artifact_metadata): + print('No artifact found in db for network_obs_id {}'.format(network_obs_id)) + return + + print("Artifact Metadata for Observation #{} found.".format(observation_id)) + + try: + artifact_file_url = artifact_metadata[0]['artifact_file'] + artifact_file = tempfile.NamedTemporaryFile(delete=False) + fetch_artifact(artifact_file_url, artifact_file.name) + print("Artifact for Observation #{} saved in '{}'".format(observation_id, artifact_file.name)) + except requests.HTTPError: + print('Download failed for {}'.format(artifact_file_url)) + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Download SatNOGS Artifacts from SatNOGS DB.') + parser.add_argument('observation_ids', metavar='ID', type=int, nargs='+', + help='SatNOGS Observation ID') + + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + for observation_id in args.observation_ids: + download_artifact(observation_id) diff --git a/contrib/find_good_satnogs_artifacts.py b/contrib/find_good_satnogs_artifacts.py new file mode 100755 index 0000000..4fb9083 --- /dev/null +++ b/contrib/find_good_satnogs_artifacts.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import sys +import logging +import requests +import settings + +from urllib.parse import urljoin +from pprint import pprint + +from satnogs_api_client import fetch_observation_data, fetch_tle_of_observation + +logger = logging.getLogger(__name__) + + +def fetch_latest_artifacts_metadata(): + url = urljoin(settings.SATNOGS_DB_API_URL, 'artifacts/',) + params = {} + headers = {'Authorization': 'Token {0}'.format(settings.SATNOGS_DB_API_TOKEN)} + + try: + response = requests.get(url, + params=params, + headers=headers, + timeout=10) + response.raise_for_status() + except (requests.ConnectionError, requests.Timeout, requests.TooManyRedirects): + logger.exception('An error occurred trying to GET artifact metadata from db') + + artifacts_metadata = response.json() + + if not len(artifacts_metadata): + logger.info('No artifacts found in db') + sys.exit(-1) + + return artifacts_metadata + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + + # Fetch list of artifacts + artifacts_metadata = fetch_latest_artifacts_metadata() + + observation_ids = [artifact['network_obs_id'] for artifact in artifacts_metadata] + + # Load corresponding obs from network + observations = fetch_observation_data(sorted(observation_ids, reverse=True)) + + # Filter by good status + for observation in observations: + if not observation['vetted_status'] == 'good': + pass + + print("{}/observations/{}/".format(settings.SATNOGS_NETWORK_API_URL[:-5], observation['id'])) diff --git a/contrib/settings.py b/contrib/settings.py index ed30b38..9abbf7b 100644 --- a/contrib/settings.py +++ b/contrib/settings.py @@ -10,3 +10,8 @@ OBS_DIR = config('SATNOGS_OBS_DIR') # SATTOOLS/STRF/STVID sites.txt file SITES_TXT = config('SATNOGS_SITES_TXT') + + +SATNOGS_NETWORK_API_URL = config('SATNOGS_NETWORK_API_URL') +SATNOGS_DB_API_URL = config('SATNOGS_DB_API_URL') +SATNOGS_DB_API_TOKEN = config('SATNOGS_DB_API_TOKEN') diff --git a/env-dist b/env-dist index 95114bf..f0da9a2 100644 --- a/env-dist +++ b/env-dist @@ -6,3 +6,9 @@ SATNOGS_DIR=/my/path/to/satnogs/data SATNOGS_TLE_DIR=$SATNOGS_DIR/tles SATNOGS_OBS_DIR=$SATNOGS_DIR/doppler_obs SATNOGS_SITES_TXT=$SATNOGS_DIR/sites.txt + + +# SatNOGS Artifacts Helpers +SATNOGS_NETWORK_API_URL=https://network.satnogs.org/api/ +SATNOGS_DB_API_URL=https://db.satnogs.org/api/ +SATNOGS_DB_API_TOKEN=your-db-api-token