Refactor caching into a CacheManager module and add CACHE_DIR setting

This commit contains mostly re-location of code, but some
logic changes as well:
- Added the CACHE_DIR setting (previously it was hardcoded)
- Fixed the "cache last updated"/tnow timestamp re-using the
  args.starttime variable. Previously, if a user invoked the script
  with the starttime argument in the far future, the caches
  "last_update" timestamp would've used this future date as a reference
  (and thus the cache wouldn't be updated until this startdate passed)

cache.py

@@ -0,0 +1,138 @@
+import os
+import logging
+from datetime import datetime
+
+from satellite_tle import fetch_tles
+from utils import get_active_transmitter_info, \
+                  get_satellite_info, \
+                  get_transmitter_stats
+
+
+class CacheManager:
+    def __init__(self,
+                 ground_station_id,
+                 ground_station_antennas,
+                 cache_dir,
+                 cache_age,
+                 max_norad_cat_id):
+        self.ground_station_id = ground_station_id
+        self.ground_station_antennas = ground_station_antennas
+        self.cache_dir = cache_dir
+        self.cache_age = cache_age
+        self.max_norad_cat_id = max_norad_cat_id
+
+        self.transmitters_file = os.path.join(self.cache_dir, "transmitters_%d.txt" % self.ground_station_id)
+        self.tles_file = os.path.join(self.cache_dir, "tles_%d.txt" % self.ground_station_id)
+        self.last_update_file = os.path.join(self.cache_dir, "last_update_%d.txt" % ground_station_id)
+
+        # Create cache
+        if not os.path.isdir(self.cache_dir):
+            os.mkdir(self.cache_dir)
+
+    def last_update(self):
+        try:
+            with open(self.last_update_file, "r") as f:
+                line = f.readline()
+            return datetime.strptime(line.strip(), "%Y-%m-%dT%H:%M:%S")
+        except IOError:
+            return None
+
+    def update_needed(self):
+        tnow = datetime.now()
+
+        # Get last update
+        tlast = self.last_update()
+
+        if tlast is None or (tnow - tlast).total_seconds() > self.cache_age * 3600:
+            return True
+        if not os.path.isfile(self.transmitters_file):
+            return True
+        if not os.path.isfile(self.tles_file):
+            return True
+        return False
+
+    def update(self, force=False):
+        if not force and not self.update_needed():
+            # Cache is valid, skip the update
+            return
+
+        logging.info('Updating transmitters and TLEs for station')
+        tnow = datetime.now()
+
+        # Store current time
+        with open(self.last_update_file, "w") as fp:
+            fp.write(tnow.strftime("%Y-%m-%dT%H:%M:%S") + "\n")
+
+        # Get active transmitters in frequency range of each antenna
+        transmitters = {}
+        for antenna in self.ground_station_antennas:
+            for transmitter in get_active_transmitter_info(antenna["frequency"],
+                                                           antenna["frequency_max"]):
+                transmitters[transmitter['uuid']] = transmitter
+
+        # Get satellites which are alive
+        alive_norad_cat_ids = get_satellite_info()
+
+        # Extract NORAD IDs from transmitters
+        norad_cat_ids = sorted(
+            set([
+                transmitter["norad_cat_id"] for transmitter in transmitters.values()
+                if transmitter["norad_cat_id"] < self.max_norad_cat_id and
+                transmitter["norad_cat_id"] in alive_norad_cat_ids
+            ]))
+
+        # Store transmitters
+        fp = open(self.transmitters_file, "w")
+        logging.info("Requesting transmitter success rates.")
+        transmitters_stats = get_transmitter_stats()
+        for transmitter in transmitters_stats:
+            uuid = transmitter["uuid"]
+            # Skip absent transmitters
+            if uuid not in transmitters.keys():
+                continue
+            # Skip dead satellites
+            if transmitters[uuid]["norad_cat_id"] not in alive_norad_cat_ids:
+                continue
+
+            fp.write(
+                "%05d %s %d %d %d %s\n" %
+                (transmitters[uuid]["norad_cat_id"], uuid, transmitter["stats"]["success_rate"],
+                 transmitter["stats"]["good_count"], transmitter["stats"]["total_count"], transmitters[uuid]["mode"]))
+
+        logging.info("Transmitter success rates received!")
+        fp.close()
+
+        # Get TLEs
+        tles = fetch_tles(norad_cat_ids)
+
+        # Store TLEs
+        with open(self.tles_file, "w") as f:
+            for norad_cat_id, (source, tle) in tles.items():
+                f.write("%s\n%s\n%s\n" % (tle[0], tle[1], tle[2]))
+
+    def read_tles(self):
+        with open(self.tles_file, "r") as f:
+            lines = f.readlines()
+            for i in range(0, len(lines), 3):
+                tle0 = lines[i]
+                tle1 = lines[i + 1]
+                tle2 = lines[i + 2]
+
+                if tle1.split(" ")[1] == "":
+                    norad_cat_id = int(tle1.split(" ")[2][:4])
+                else:
+                    norad_cat_id = int(tle1.split(" ")[1][:5])
+
+                yield {'norad_cat_id': norad_cat_id,
+                       'lines': [tle0, tle1, tle2]}
+
+    def read_transmitters(self):
+        with open(self.transmitters_file, "r") as f:
+            for line in f.readlines():
+                item = line.split()
+                yield {"norad_cat_id": int(item[0]),
+                       "uuid": item[1],
+                       "success_rate": float(item[2]) / 100.0,
+                       "good_count": int(item[3]),
+                       "data_count": int(item[4]),
+                       "mode": item[5]}

schedule_single_station.py

@@ -3,24 +3,20 @@ from __future__ import division
 import requests
 import ephem
 from datetime import datetime, timedelta
-from satellite_tle import fetch_tles
 import os
 import lxml.html
 import argparse
 import logging
-from utils import get_active_transmitter_info, \
+from utils import get_groundstation_info, \
-                  get_transmitter_stats, \
-                  get_groundstation_info, \
                   get_scheduled_passes_from_network, \
                   schedule_observation, \
                   read_priorities_transmitters, \
-                  get_satellite_info, \
-                  update_needed, \
                   get_priority_passes
 from auto_scheduler import Twolineelement, Satellite
 from auto_scheduler.pass_predictor import find_passes
 from auto_scheduler.schedulers import ordered_scheduler, \
                                       report_efficiency
+from cache import CacheManager
 import settings
 from tqdm import tqdm
 import sys
@@ -142,7 +138,6 @@ def main():
         min_priority = 1.0
     else:
         min_priority = args.min_priority
-    cache_dir = "/tmp/cache"
     schedule = not args.dryrun
     only_priority = args.only_priority
     priority_filename = args.priorities
@@ -156,68 +151,14 @@ def main():
     ground_station = get_groundstation_info(ground_station_id, args.allow_testing)
     if not ground_station:
         sys.exit()
-    
-    # Create cache
-    if not os.path.isdir(cache_dir):
-        os.mkdir(cache_dir)
 
-    # Update logic
+    # Create or update the transmitter & TLE cache
-    update = update_needed(tnow, ground_station_id, cache_dir)
+    cache = CacheManager(ground_station_id,
-
+                         ground_station['antenna'],
-    # Update
+                         settings.CACHE_DIR,
-    if update:
+                         settings.CACHE_AGE,
-        logging.info('Updating transmitters and TLEs for station')
+                         settings.MAX_NORAD_CAT_ID)
-        # Store current time
+    cache.update()
-        with open(os.path.join(cache_dir, "last_update_%d.txt" % ground_station_id), "w") as fp:
-            fp.write(tnow.strftime("%Y-%m-%dT%H:%M:%S") + "\n")
-
-        # Get active transmitters in frequency range of each antenna
-        transmitters = {}
-        for antenna in ground_station['antenna']:
-            for transmitter in get_active_transmitter_info(antenna["frequency"],
-                                                           antenna["frequency_max"]):
-                transmitters[transmitter['uuid']] = transmitter
-
-        # Get satellites which are alive
-        alive_norad_cat_ids = get_satellite_info()
-
-        # Get NORAD IDs
-        norad_cat_ids = sorted(
-            set([
-                transmitter["norad_cat_id"] for transmitter in transmitters.values()
-                if transmitter["norad_cat_id"] < settings.MAX_NORAD_CAT_ID and
-                transmitter["norad_cat_id"] in alive_norad_cat_ids
-            ]))
-
-        # Store transmitters
-        fp = open(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id), "w")
-        logging.info("Requesting transmitter success rates.")
-        transmitters_stats = get_transmitter_stats()
-        for transmitter in transmitters_stats:
-            uuid = transmitter["uuid"]
-            # Skip absent transmitters
-            if uuid not in transmitters.keys():
-                continue
-            # Skip dead satellites
-            if transmitters[uuid]["norad_cat_id"] not in alive_norad_cat_ids:
-                continue
-
-            fp.write(
-                "%05d %s %d %d %d %s\n" %
-                (transmitters[uuid]["norad_cat_id"], uuid, transmitter["stats"]["success_rate"],
-                 transmitter["stats"]["good_count"], transmitter["stats"]["total_count"], transmitters[uuid]["mode"]))
-
-        logging.info("Transmitter success rates received!")
-        fp.close()
-
-        # Get TLEs
-        tles = fetch_tles(norad_cat_ids)
-
-        # Store TLEs
-        fp = open(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id), "w")
-        for norad_cat_id, (source, tle) in tles.items():
-            fp.write("%s\n%s\n%s\n" % (tle[0], tle[1], tle[2]))
-        fp.close()
 
     # Set observer
     observer = ephem.Observer()
@@ -255,23 +196,22 @@ def main():
     min_pass_duration = settings.MIN_PASS_DURATION
 
     # Read tles
-    with open(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id), "r") as f:
+    tles = list(cache.read_tles())
-        lines = f.readlines()
-        tles = [
-            Twolineelement(lines[i], lines[i + 1], lines[i + 2]) for i in range(0, len(lines), 3)
-        ]
 
     # Read transmitters
+    transmitters = cache.read_transmitters()
+
+    # Extract satellites from receivable transmitters
     satellites = []
-    with open(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id), "r") as f:
+    for transmitter in transmitters:
-        lines = f.readlines()
+        for tle in tles:
-        for line in lines:
+            if tle['norad_cat_id'] == transmitter['norad_cat_id']:
-            item = line.split()
+                satellites.append(Satellite(Twolineelement(*tle['lines']),
-            norad_cat_id, uuid, success_rate, good_count, data_count, mode = int(
+                                            transmitter['uuid'],
-                item[0]), item[1], float(item[2]) / 100.0, int(item[3]), int(item[4]), item[5]
+                                            transmitter['success_rate'],
-            for tle in tles:
+                                            transmitter['good_count'],
-                if tle.id == norad_cat_id:
+                                            transmitter['data_count'],
-                    satellites.append(Satellite(tle, uuid, success_rate, good_count, data_count, mode))
+                                            transmitter['mode']))
 
     # Find passes
     passes = []

settings.py

@@ -3,6 +3,7 @@ from decouple import config
 # Basic settings
 DB_BASE_URL = config('DB_BASE_URL', default='https://db.satnogs.org')
 NETWORK_BASE_URL = config('NETWORK_BASE_URL', default='https://network.satnogs.org')
+CACHE_DIR = config('CACHE_DIR', default='/tmp/cache')
 CACHE_AGE = config('CACHE_AGE', default=24)  # In hours
 MAX_NORAD_CAT_ID = config('MAX_NORAD_CAT_ID', default=90000)
 MIN_PASS_DURATION = config('MIN_PASS_DURATION', default=2)  # In minutes

test_cache.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+import logging
+import settings
+
+from cache import CacheManager
+from utils import get_groundstation_info
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO,
+                        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+    ground_station_id = 2
+    ground_station = get_groundstation_info(ground_station_id, allow_testing=True)
+    print(ground_station)
+
+    cache = CacheManager(ground_station_id,
+                         ground_station['antenna'],
+                         settings.CACHE_DIR,
+                         settings.CACHE_AGE,
+                         settings.MAX_NORAD_CAT_ID)
+    print(cache.last_update())
+    print(cache.update_needed())
+
+    cache.update(force=True)
+
+    print(cache.last_update())

utils.py

@@ -197,29 +197,6 @@ def get_groundstation_info(ground_station_id, allow_testing):
         return {}
 
 
-def get_last_update(fname):
-    try:
-        fp = open(fname, "r")
-        line = fp.readline()
-        fp.close()
-        return datetime.strptime(line.strip(), "%Y-%m-%dT%H:%M:%S")
-    except IOError:
-        return None
-
-
-def update_needed(tnow, ground_station_id, cache_dir):
-    # Get last update
-    tlast = get_last_update(os.path.join(cache_dir, "last_update_%d.txt" % ground_station_id))
-
-    if tlast is None or (tnow - tlast).total_seconds() > settings.CACHE_AGE * 3600:
-        return True
-    if not os.path.isfile(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id)):
-        return True
-    if not os.path.isfile(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id)):
-        return True
-    return False
-
-
 def schedule_observation(session, norad_cat_id, uuid, ground_station_id, starttime, endtime):
 
     obsURL = '{}/observations/new/'.format(settings.NETWORK_BASE_URL)  # Observation URL