From 0cbf664991b112111dad45dde98cb21d9690a2e5 Mon Sep 17 00:00:00 2001 From: "Fabian P. Schmidt" Date: Sat, 2 Nov 2019 18:48:21 +0100 Subject: [PATCH] Refactor caching into a CacheManager module and add CACHE_DIR setting This commit contains mostly re-location of code, but some logic changes as well: - Added the CACHE_DIR setting (previously it was hardcoded) - Fixed the "cache last updated"/tnow timestamp re-using the args.starttime variable. Previously, if a user invoked the script with the starttime argument in the far future, the caches "last_update" timestamp would've used this future date as a reference (and thus the cache wouldn't be updated until this startdate passed) --- cache.py | 138 +++++++++++++++++++++++++++++++++++++ schedule_single_station.py | 104 ++++++---------------------- settings.py | 1 + test_cache.py | 29 ++++++++ utils.py | 23 ------- 5 files changed, 190 insertions(+), 105 deletions(-) create mode 100644 cache.py create mode 100755 test_cache.py diff --git a/cache.py b/cache.py new file mode 100644 index 0000000..dfa340c --- /dev/null +++ b/cache.py @@ -0,0 +1,138 @@ +import os +import logging +from datetime import datetime + +from satellite_tle import fetch_tles +from utils import get_active_transmitter_info, \ + get_satellite_info, \ + get_transmitter_stats + + +class CacheManager: + def __init__(self, + ground_station_id, + ground_station_antennas, + cache_dir, + cache_age, + max_norad_cat_id): + self.ground_station_id = ground_station_id + self.ground_station_antennas = ground_station_antennas + self.cache_dir = cache_dir + self.cache_age = cache_age + self.max_norad_cat_id = max_norad_cat_id + + self.transmitters_file = os.path.join(self.cache_dir, "transmitters_%d.txt" % self.ground_station_id) + self.tles_file = os.path.join(self.cache_dir, "tles_%d.txt" % self.ground_station_id) + self.last_update_file = os.path.join(self.cache_dir, "last_update_%d.txt" % ground_station_id) + + # Create cache + if not os.path.isdir(self.cache_dir): + os.mkdir(self.cache_dir) + + def last_update(self): + try: + with open(self.last_update_file, "r") as f: + line = f.readline() + return datetime.strptime(line.strip(), "%Y-%m-%dT%H:%M:%S") + except IOError: + return None + + def update_needed(self): + tnow = datetime.now() + + # Get last update + tlast = self.last_update() + + if tlast is None or (tnow - tlast).total_seconds() > self.cache_age * 3600: + return True + if not os.path.isfile(self.transmitters_file): + return True + if not os.path.isfile(self.tles_file): + return True + return False + + def update(self, force=False): + if not force and not self.update_needed(): + # Cache is valid, skip the update + return + + logging.info('Updating transmitters and TLEs for station') + tnow = datetime.now() + + # Store current time + with open(self.last_update_file, "w") as fp: + fp.write(tnow.strftime("%Y-%m-%dT%H:%M:%S") + "\n") + + # Get active transmitters in frequency range of each antenna + transmitters = {} + for antenna in self.ground_station_antennas: + for transmitter in get_active_transmitter_info(antenna["frequency"], + antenna["frequency_max"]): + transmitters[transmitter['uuid']] = transmitter + + # Get satellites which are alive + alive_norad_cat_ids = get_satellite_info() + + # Extract NORAD IDs from transmitters + norad_cat_ids = sorted( + set([ + transmitter["norad_cat_id"] for transmitter in transmitters.values() + if transmitter["norad_cat_id"] < self.max_norad_cat_id and + transmitter["norad_cat_id"] in alive_norad_cat_ids + ])) + + # Store transmitters + fp = open(self.transmitters_file, "w") + logging.info("Requesting transmitter success rates.") + transmitters_stats = get_transmitter_stats() + for transmitter in transmitters_stats: + uuid = transmitter["uuid"] + # Skip absent transmitters + if uuid not in transmitters.keys(): + continue + # Skip dead satellites + if transmitters[uuid]["norad_cat_id"] not in alive_norad_cat_ids: + continue + + fp.write( + "%05d %s %d %d %d %s\n" % + (transmitters[uuid]["norad_cat_id"], uuid, transmitter["stats"]["success_rate"], + transmitter["stats"]["good_count"], transmitter["stats"]["total_count"], transmitters[uuid]["mode"])) + + logging.info("Transmitter success rates received!") + fp.close() + + # Get TLEs + tles = fetch_tles(norad_cat_ids) + + # Store TLEs + with open(self.tles_file, "w") as f: + for norad_cat_id, (source, tle) in tles.items(): + f.write("%s\n%s\n%s\n" % (tle[0], tle[1], tle[2])) + + def read_tles(self): + with open(self.tles_file, "r") as f: + lines = f.readlines() + for i in range(0, len(lines), 3): + tle0 = lines[i] + tle1 = lines[i + 1] + tle2 = lines[i + 2] + + if tle1.split(" ")[1] == "": + norad_cat_id = int(tle1.split(" ")[2][:4]) + else: + norad_cat_id = int(tle1.split(" ")[1][:5]) + + yield {'norad_cat_id': norad_cat_id, + 'lines': [tle0, tle1, tle2]} + + def read_transmitters(self): + with open(self.transmitters_file, "r") as f: + for line in f.readlines(): + item = line.split() + yield {"norad_cat_id": int(item[0]), + "uuid": item[1], + "success_rate": float(item[2]) / 100.0, + "good_count": int(item[3]), + "data_count": int(item[4]), + "mode": item[5]} diff --git a/schedule_single_station.py b/schedule_single_station.py index 7ac4ec9..e19a242 100755 --- a/schedule_single_station.py +++ b/schedule_single_station.py @@ -3,24 +3,20 @@ from __future__ import division import requests import ephem from datetime import datetime, timedelta -from satellite_tle import fetch_tles import os import lxml.html import argparse import logging -from utils import get_active_transmitter_info, \ - get_transmitter_stats, \ - get_groundstation_info, \ +from utils import get_groundstation_info, \ get_scheduled_passes_from_network, \ schedule_observation, \ read_priorities_transmitters, \ - get_satellite_info, \ - update_needed, \ get_priority_passes from auto_scheduler import Twolineelement, Satellite from auto_scheduler.pass_predictor import find_passes from auto_scheduler.schedulers import ordered_scheduler, \ report_efficiency +from cache import CacheManager import settings from tqdm import tqdm import sys @@ -142,7 +138,6 @@ def main(): min_priority = 1.0 else: min_priority = args.min_priority - cache_dir = "/tmp/cache" schedule = not args.dryrun only_priority = args.only_priority priority_filename = args.priorities @@ -156,68 +151,14 @@ def main(): ground_station = get_groundstation_info(ground_station_id, args.allow_testing) if not ground_station: sys.exit() - - # Create cache - if not os.path.isdir(cache_dir): - os.mkdir(cache_dir) - # Update logic - update = update_needed(tnow, ground_station_id, cache_dir) - - # Update - if update: - logging.info('Updating transmitters and TLEs for station') - # Store current time - with open(os.path.join(cache_dir, "last_update_%d.txt" % ground_station_id), "w") as fp: - fp.write(tnow.strftime("%Y-%m-%dT%H:%M:%S") + "\n") - - # Get active transmitters in frequency range of each antenna - transmitters = {} - for antenna in ground_station['antenna']: - for transmitter in get_active_transmitter_info(antenna["frequency"], - antenna["frequency_max"]): - transmitters[transmitter['uuid']] = transmitter - - # Get satellites which are alive - alive_norad_cat_ids = get_satellite_info() - - # Get NORAD IDs - norad_cat_ids = sorted( - set([ - transmitter["norad_cat_id"] for transmitter in transmitters.values() - if transmitter["norad_cat_id"] < settings.MAX_NORAD_CAT_ID and - transmitter["norad_cat_id"] in alive_norad_cat_ids - ])) - - # Store transmitters - fp = open(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id), "w") - logging.info("Requesting transmitter success rates.") - transmitters_stats = get_transmitter_stats() - for transmitter in transmitters_stats: - uuid = transmitter["uuid"] - # Skip absent transmitters - if uuid not in transmitters.keys(): - continue - # Skip dead satellites - if transmitters[uuid]["norad_cat_id"] not in alive_norad_cat_ids: - continue - - fp.write( - "%05d %s %d %d %d %s\n" % - (transmitters[uuid]["norad_cat_id"], uuid, transmitter["stats"]["success_rate"], - transmitter["stats"]["good_count"], transmitter["stats"]["total_count"], transmitters[uuid]["mode"])) - - logging.info("Transmitter success rates received!") - fp.close() - - # Get TLEs - tles = fetch_tles(norad_cat_ids) - - # Store TLEs - fp = open(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id), "w") - for norad_cat_id, (source, tle) in tles.items(): - fp.write("%s\n%s\n%s\n" % (tle[0], tle[1], tle[2])) - fp.close() + # Create or update the transmitter & TLE cache + cache = CacheManager(ground_station_id, + ground_station['antenna'], + settings.CACHE_DIR, + settings.CACHE_AGE, + settings.MAX_NORAD_CAT_ID) + cache.update() # Set observer observer = ephem.Observer() @@ -255,23 +196,22 @@ def main(): min_pass_duration = settings.MIN_PASS_DURATION # Read tles - with open(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id), "r") as f: - lines = f.readlines() - tles = [ - Twolineelement(lines[i], lines[i + 1], lines[i + 2]) for i in range(0, len(lines), 3) - ] + tles = list(cache.read_tles()) # Read transmitters + transmitters = cache.read_transmitters() + + # Extract satellites from receivable transmitters satellites = [] - with open(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id), "r") as f: - lines = f.readlines() - for line in lines: - item = line.split() - norad_cat_id, uuid, success_rate, good_count, data_count, mode = int( - item[0]), item[1], float(item[2]) / 100.0, int(item[3]), int(item[4]), item[5] - for tle in tles: - if tle.id == norad_cat_id: - satellites.append(Satellite(tle, uuid, success_rate, good_count, data_count, mode)) + for transmitter in transmitters: + for tle in tles: + if tle['norad_cat_id'] == transmitter['norad_cat_id']: + satellites.append(Satellite(Twolineelement(*tle['lines']), + transmitter['uuid'], + transmitter['success_rate'], + transmitter['good_count'], + transmitter['data_count'], + transmitter['mode'])) # Find passes passes = [] diff --git a/settings.py b/settings.py index a69c544..fb68013 100644 --- a/settings.py +++ b/settings.py @@ -3,6 +3,7 @@ from decouple import config # Basic settings DB_BASE_URL = config('DB_BASE_URL', default='https://db.satnogs.org') NETWORK_BASE_URL = config('NETWORK_BASE_URL', default='https://network.satnogs.org') +CACHE_DIR = config('CACHE_DIR', default='/tmp/cache') CACHE_AGE = config('CACHE_AGE', default=24) # In hours MAX_NORAD_CAT_ID = config('MAX_NORAD_CAT_ID', default=90000) MIN_PASS_DURATION = config('MIN_PASS_DURATION', default=2) # In minutes diff --git a/test_cache.py b/test_cache.py new file mode 100755 index 0000000..f6465c6 --- /dev/null +++ b/test_cache.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +from datetime import datetime +import logging +import settings + +from cache import CacheManager +from utils import get_groundstation_info + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") + + ground_station_id = 2 + ground_station = get_groundstation_info(ground_station_id, allow_testing=True) + print(ground_station) + + cache = CacheManager(ground_station_id, + ground_station['antenna'], + settings.CACHE_DIR, + settings.CACHE_AGE, + settings.MAX_NORAD_CAT_ID) + print(cache.last_update()) + print(cache.update_needed()) + + cache.update(force=True) + + print(cache.last_update()) diff --git a/utils.py b/utils.py index 3a28762..b4e94aa 100644 --- a/utils.py +++ b/utils.py @@ -197,29 +197,6 @@ def get_groundstation_info(ground_station_id, allow_testing): return {} -def get_last_update(fname): - try: - fp = open(fname, "r") - line = fp.readline() - fp.close() - return datetime.strptime(line.strip(), "%Y-%m-%dT%H:%M:%S") - except IOError: - return None - - -def update_needed(tnow, ground_station_id, cache_dir): - # Get last update - tlast = get_last_update(os.path.join(cache_dir, "last_update_%d.txt" % ground_station_id)) - - if tlast is None or (tnow - tlast).total_seconds() > settings.CACHE_AGE * 3600: - return True - if not os.path.isfile(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id)): - return True - if not os.path.isfile(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id)): - return True - return False - - def schedule_observation(session, norad_cat_id, uuid, ground_station_id, starttime, endtime): obsURL = '{}/observations/new/'.format(settings.NETWORK_BASE_URL) # Observation URL