Refactor caching into a CacheManager module and add CACHE_DIR setting
This commit contains mostly re-location of code, but some logic changes as well: - Added the CACHE_DIR setting (previously it was hardcoded) - Fixed the "cache last updated"/tnow timestamp re-using the args.starttime variable. Previously, if a user invoked the script with the starttime argument in the far future, the caches "last_update" timestamp would've used this future date as a reference (and thus the cache wouldn't be updated until this startdate passed)merge-requests/49/head
Fabian P. Schmidt
parent
ea84f8a0e7
commit
0cbf664991
|
|
@ -0,0 +1,138 @@
|
|||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from satellite_tle import fetch_tles
|
||||
from utils import get_active_transmitter_info, \
|
||||
get_satellite_info, \
|
||||
get_transmitter_stats
|
||||
|
||||
|
||||
class CacheManager:
|
||||
def __init__(self,
|
||||
ground_station_id,
|
||||
ground_station_antennas,
|
||||
cache_dir,
|
||||
cache_age,
|
||||
max_norad_cat_id):
|
||||
self.ground_station_id = ground_station_id
|
||||
self.ground_station_antennas = ground_station_antennas
|
||||
self.cache_dir = cache_dir
|
||||
self.cache_age = cache_age
|
||||
self.max_norad_cat_id = max_norad_cat_id
|
||||
|
||||
self.transmitters_file = os.path.join(self.cache_dir, "transmitters_%d.txt" % self.ground_station_id)
|
||||
self.tles_file = os.path.join(self.cache_dir, "tles_%d.txt" % self.ground_station_id)
|
||||
self.last_update_file = os.path.join(self.cache_dir, "last_update_%d.txt" % ground_station_id)
|
||||
|
||||
# Create cache
|
||||
if not os.path.isdir(self.cache_dir):
|
||||
os.mkdir(self.cache_dir)
|
||||
|
||||
def last_update(self):
|
||||
try:
|
||||
with open(self.last_update_file, "r") as f:
|
||||
line = f.readline()
|
||||
return datetime.strptime(line.strip(), "%Y-%m-%dT%H:%M:%S")
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
def update_needed(self):
|
||||
tnow = datetime.now()
|
||||
|
||||
# Get last update
|
||||
tlast = self.last_update()
|
||||
|
||||
if tlast is None or (tnow - tlast).total_seconds() > self.cache_age * 3600:
|
||||
return True
|
||||
if not os.path.isfile(self.transmitters_file):
|
||||
return True
|
||||
if not os.path.isfile(self.tles_file):
|
||||
return True
|
||||
return False
|
||||
|
||||
def update(self, force=False):
|
||||
if not force and not self.update_needed():
|
||||
# Cache is valid, skip the update
|
||||
return
|
||||
|
||||
logging.info('Updating transmitters and TLEs for station')
|
||||
tnow = datetime.now()
|
||||
|
||||
# Store current time
|
||||
with open(self.last_update_file, "w") as fp:
|
||||
fp.write(tnow.strftime("%Y-%m-%dT%H:%M:%S") + "\n")
|
||||
|
||||
# Get active transmitters in frequency range of each antenna
|
||||
transmitters = {}
|
||||
for antenna in self.ground_station_antennas:
|
||||
for transmitter in get_active_transmitter_info(antenna["frequency"],
|
||||
antenna["frequency_max"]):
|
||||
transmitters[transmitter['uuid']] = transmitter
|
||||
|
||||
# Get satellites which are alive
|
||||
alive_norad_cat_ids = get_satellite_info()
|
||||
|
||||
# Extract NORAD IDs from transmitters
|
||||
norad_cat_ids = sorted(
|
||||
set([
|
||||
transmitter["norad_cat_id"] for transmitter in transmitters.values()
|
||||
if transmitter["norad_cat_id"] < self.max_norad_cat_id and
|
||||
transmitter["norad_cat_id"] in alive_norad_cat_ids
|
||||
]))
|
||||
|
||||
# Store transmitters
|
||||
fp = open(self.transmitters_file, "w")
|
||||
logging.info("Requesting transmitter success rates.")
|
||||
transmitters_stats = get_transmitter_stats()
|
||||
for transmitter in transmitters_stats:
|
||||
uuid = transmitter["uuid"]
|
||||
# Skip absent transmitters
|
||||
if uuid not in transmitters.keys():
|
||||
continue
|
||||
# Skip dead satellites
|
||||
if transmitters[uuid]["norad_cat_id"] not in alive_norad_cat_ids:
|
||||
continue
|
||||
|
||||
fp.write(
|
||||
"%05d %s %d %d %d %s\n" %
|
||||
(transmitters[uuid]["norad_cat_id"], uuid, transmitter["stats"]["success_rate"],
|
||||
transmitter["stats"]["good_count"], transmitter["stats"]["total_count"], transmitters[uuid]["mode"]))
|
||||
|
||||
logging.info("Transmitter success rates received!")
|
||||
fp.close()
|
||||
|
||||
# Get TLEs
|
||||
tles = fetch_tles(norad_cat_ids)
|
||||
|
||||
# Store TLEs
|
||||
with open(self.tles_file, "w") as f:
|
||||
for norad_cat_id, (source, tle) in tles.items():
|
||||
f.write("%s\n%s\n%s\n" % (tle[0], tle[1], tle[2]))
|
||||
|
||||
def read_tles(self):
|
||||
with open(self.tles_file, "r") as f:
|
||||
lines = f.readlines()
|
||||
for i in range(0, len(lines), 3):
|
||||
tle0 = lines[i]
|
||||
tle1 = lines[i + 1]
|
||||
tle2 = lines[i + 2]
|
||||
|
||||
if tle1.split(" ")[1] == "":
|
||||
norad_cat_id = int(tle1.split(" ")[2][:4])
|
||||
else:
|
||||
norad_cat_id = int(tle1.split(" ")[1][:5])
|
||||
|
||||
yield {'norad_cat_id': norad_cat_id,
|
||||
'lines': [tle0, tle1, tle2]}
|
||||
|
||||
def read_transmitters(self):
|
||||
with open(self.transmitters_file, "r") as f:
|
||||
for line in f.readlines():
|
||||
item = line.split()
|
||||
yield {"norad_cat_id": int(item[0]),
|
||||
"uuid": item[1],
|
||||
"success_rate": float(item[2]) / 100.0,
|
||||
"good_count": int(item[3]),
|
||||
"data_count": int(item[4]),
|
||||
"mode": item[5]}
|
||||
|
|
@ -3,24 +3,20 @@ from __future__ import division
|
|||
import requests
|
||||
import ephem
|
||||
from datetime import datetime, timedelta
|
||||
from satellite_tle import fetch_tles
|
||||
import os
|
||||
import lxml.html
|
||||
import argparse
|
||||
import logging
|
||||
from utils import get_active_transmitter_info, \
|
||||
get_transmitter_stats, \
|
||||
get_groundstation_info, \
|
||||
from utils import get_groundstation_info, \
|
||||
get_scheduled_passes_from_network, \
|
||||
schedule_observation, \
|
||||
read_priorities_transmitters, \
|
||||
get_satellite_info, \
|
||||
update_needed, \
|
||||
get_priority_passes
|
||||
from auto_scheduler import Twolineelement, Satellite
|
||||
from auto_scheduler.pass_predictor import find_passes
|
||||
from auto_scheduler.schedulers import ordered_scheduler, \
|
||||
report_efficiency
|
||||
from cache import CacheManager
|
||||
import settings
|
||||
from tqdm import tqdm
|
||||
import sys
|
||||
|
|
@ -142,7 +138,6 @@ def main():
|
|||
min_priority = 1.0
|
||||
else:
|
||||
min_priority = args.min_priority
|
||||
cache_dir = "/tmp/cache"
|
||||
schedule = not args.dryrun
|
||||
only_priority = args.only_priority
|
||||
priority_filename = args.priorities
|
||||
|
|
@ -156,68 +151,14 @@ def main():
|
|||
ground_station = get_groundstation_info(ground_station_id, args.allow_testing)
|
||||
if not ground_station:
|
||||
sys.exit()
|
||||
|
||||
# Create cache
|
||||
if not os.path.isdir(cache_dir):
|
||||
os.mkdir(cache_dir)
|
||||
|
||||
# Update logic
|
||||
update = update_needed(tnow, ground_station_id, cache_dir)
|
||||
|
||||
# Update
|
||||
if update:
|
||||
logging.info('Updating transmitters and TLEs for station')
|
||||
# Store current time
|
||||
with open(os.path.join(cache_dir, "last_update_%d.txt" % ground_station_id), "w") as fp:
|
||||
fp.write(tnow.strftime("%Y-%m-%dT%H:%M:%S") + "\n")
|
||||
|
||||
# Get active transmitters in frequency range of each antenna
|
||||
transmitters = {}
|
||||
for antenna in ground_station['antenna']:
|
||||
for transmitter in get_active_transmitter_info(antenna["frequency"],
|
||||
antenna["frequency_max"]):
|
||||
transmitters[transmitter['uuid']] = transmitter
|
||||
|
||||
# Get satellites which are alive
|
||||
alive_norad_cat_ids = get_satellite_info()
|
||||
|
||||
# Get NORAD IDs
|
||||
norad_cat_ids = sorted(
|
||||
set([
|
||||
transmitter["norad_cat_id"] for transmitter in transmitters.values()
|
||||
if transmitter["norad_cat_id"] < settings.MAX_NORAD_CAT_ID and
|
||||
transmitter["norad_cat_id"] in alive_norad_cat_ids
|
||||
]))
|
||||
|
||||
# Store transmitters
|
||||
fp = open(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id), "w")
|
||||
logging.info("Requesting transmitter success rates.")
|
||||
transmitters_stats = get_transmitter_stats()
|
||||
for transmitter in transmitters_stats:
|
||||
uuid = transmitter["uuid"]
|
||||
# Skip absent transmitters
|
||||
if uuid not in transmitters.keys():
|
||||
continue
|
||||
# Skip dead satellites
|
||||
if transmitters[uuid]["norad_cat_id"] not in alive_norad_cat_ids:
|
||||
continue
|
||||
|
||||
fp.write(
|
||||
"%05d %s %d %d %d %s\n" %
|
||||
(transmitters[uuid]["norad_cat_id"], uuid, transmitter["stats"]["success_rate"],
|
||||
transmitter["stats"]["good_count"], transmitter["stats"]["total_count"], transmitters[uuid]["mode"]))
|
||||
|
||||
logging.info("Transmitter success rates received!")
|
||||
fp.close()
|
||||
|
||||
# Get TLEs
|
||||
tles = fetch_tles(norad_cat_ids)
|
||||
|
||||
# Store TLEs
|
||||
fp = open(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id), "w")
|
||||
for norad_cat_id, (source, tle) in tles.items():
|
||||
fp.write("%s\n%s\n%s\n" % (tle[0], tle[1], tle[2]))
|
||||
fp.close()
|
||||
# Create or update the transmitter & TLE cache
|
||||
cache = CacheManager(ground_station_id,
|
||||
ground_station['antenna'],
|
||||
settings.CACHE_DIR,
|
||||
settings.CACHE_AGE,
|
||||
settings.MAX_NORAD_CAT_ID)
|
||||
cache.update()
|
||||
|
||||
# Set observer
|
||||
observer = ephem.Observer()
|
||||
|
|
@ -255,23 +196,22 @@ def main():
|
|||
min_pass_duration = settings.MIN_PASS_DURATION
|
||||
|
||||
# Read tles
|
||||
with open(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id), "r") as f:
|
||||
lines = f.readlines()
|
||||
tles = [
|
||||
Twolineelement(lines[i], lines[i + 1], lines[i + 2]) for i in range(0, len(lines), 3)
|
||||
]
|
||||
tles = list(cache.read_tles())
|
||||
|
||||
# Read transmitters
|
||||
transmitters = cache.read_transmitters()
|
||||
|
||||
# Extract satellites from receivable transmitters
|
||||
satellites = []
|
||||
with open(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id), "r") as f:
|
||||
lines = f.readlines()
|
||||
for line in lines:
|
||||
item = line.split()
|
||||
norad_cat_id, uuid, success_rate, good_count, data_count, mode = int(
|
||||
item[0]), item[1], float(item[2]) / 100.0, int(item[3]), int(item[4]), item[5]
|
||||
for tle in tles:
|
||||
if tle.id == norad_cat_id:
|
||||
satellites.append(Satellite(tle, uuid, success_rate, good_count, data_count, mode))
|
||||
for transmitter in transmitters:
|
||||
for tle in tles:
|
||||
if tle['norad_cat_id'] == transmitter['norad_cat_id']:
|
||||
satellites.append(Satellite(Twolineelement(*tle['lines']),
|
||||
transmitter['uuid'],
|
||||
transmitter['success_rate'],
|
||||
transmitter['good_count'],
|
||||
transmitter['data_count'],
|
||||
transmitter['mode']))
|
||||
|
||||
# Find passes
|
||||
passes = []
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from decouple import config
|
|||
# Basic settings
|
||||
DB_BASE_URL = config('DB_BASE_URL', default='https://db.satnogs.org')
|
||||
NETWORK_BASE_URL = config('NETWORK_BASE_URL', default='https://network.satnogs.org')
|
||||
CACHE_DIR = config('CACHE_DIR', default='/tmp/cache')
|
||||
CACHE_AGE = config('CACHE_AGE', default=24) # In hours
|
||||
MAX_NORAD_CAT_ID = config('MAX_NORAD_CAT_ID', default=90000)
|
||||
MIN_PASS_DURATION = config('MIN_PASS_DURATION', default=2) # In minutes
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import settings
|
||||
|
||||
from cache import CacheManager
|
||||
from utils import get_groundstation_info
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.INFO,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
|
||||
ground_station_id = 2
|
||||
ground_station = get_groundstation_info(ground_station_id, allow_testing=True)
|
||||
print(ground_station)
|
||||
|
||||
cache = CacheManager(ground_station_id,
|
||||
ground_station['antenna'],
|
||||
settings.CACHE_DIR,
|
||||
settings.CACHE_AGE,
|
||||
settings.MAX_NORAD_CAT_ID)
|
||||
print(cache.last_update())
|
||||
print(cache.update_needed())
|
||||
|
||||
cache.update(force=True)
|
||||
|
||||
print(cache.last_update())
|
||||
23
utils.py
23
utils.py
|
|
@ -197,29 +197,6 @@ def get_groundstation_info(ground_station_id, allow_testing):
|
|||
return {}
|
||||
|
||||
|
||||
def get_last_update(fname):
|
||||
try:
|
||||
fp = open(fname, "r")
|
||||
line = fp.readline()
|
||||
fp.close()
|
||||
return datetime.strptime(line.strip(), "%Y-%m-%dT%H:%M:%S")
|
||||
except IOError:
|
||||
return None
|
||||
|
||||
|
||||
def update_needed(tnow, ground_station_id, cache_dir):
|
||||
# Get last update
|
||||
tlast = get_last_update(os.path.join(cache_dir, "last_update_%d.txt" % ground_station_id))
|
||||
|
||||
if tlast is None or (tnow - tlast).total_seconds() > settings.CACHE_AGE * 3600:
|
||||
return True
|
||||
if not os.path.isfile(os.path.join(cache_dir, "transmitters_%d.txt" % ground_station_id)):
|
||||
return True
|
||||
if not os.path.isfile(os.path.join(cache_dir, "tles_%d.txt" % ground_station_id)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def schedule_observation(session, norad_cat_id, uuid, ground_station_id, starttime, endtime):
|
||||
|
||||
obsURL = '{}/observations/new/'.format(settings.NETWORK_BASE_URL) # Observation URL
|
||||
|
|
|
|||
Loading…
Reference in New Issue