diff --git a/README.md b/README.md index 49200bd..1b9484d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,24 @@ -# SatnogsMap +#Satnogs-Map -Uses Satnogs data to generate a map of all stations on Production and all sats currently observed. +Uses CesiumJS and a python backend to make a very nice map view of the satnogs network. + +Launch using python satnogs.py -Currently hosted at satnogs.minecraft16.ml \ No newline at end of file +Access the basic map at + +http://127.0.0.1:5001 when launched + +Access an auto rotating view at + +http://127.0.0.1:5001/rotating + +add a speed arg to the url to change the speed. + +Eaxmple + +http://127.0.0.1:5001/rotating?speed=60 + +Access station specific views at + +http://127.0.0.1:5001/station/ replacing with the id of the station on the network. diff --git a/requirments.txt b/requirments.txt index 1a226d3..9fb41b9 100644 --- a/requirments.txt +++ b/requirments.txt @@ -1,4 +1,6 @@ requests==2.21.0 flask==1.0.2 apscheduler==3.5.3 -satellitetle==0.5.1 \ No newline at end of file +satellitetle==0.5.1 +tqdm==4.31.1 +skyfield==1.10 diff --git a/satnogs.py b/satnogs.py index a35d1b0..0c7d98b 100644 --- a/satnogs.py +++ b/satnogs.py @@ -1,197 +1,298 @@ -from datetime import datetime , timedelta +from datetime import datetime, timedelta import requests -from flask import Flask , render_template,redirect,url_for +from tqdm import tqdm +from flask import Flask , render_template,redirect,url_for, request import json +from collections import defaultdict, Counter import random from apscheduler.schedulers.background import BackgroundScheduler -from satnogs_api_client import fetch_satellites, DB_BASE_URL,fetch_tle_of_observation +from satnogs_api_client import fetch_satellites +from satnogs_api_client import DB_BASE_URL, get_paginated_endpoint from satellite_tle import fetch_tles +from skyfield.api import EarthSatellite, utc, load scheduler = BackgroundScheduler() app = Flask(__name__) +ts = load.timescale() -Passes = [] -Occuring_sats = {} + +broken = defaultdict(set) + + +Usage = Counter() +Sats = defaultdict(list) +Passes = defaultdict(list) Stations = [] -TLEs = {} -Transmitters = {} - -class Pass: - id = 0 - start = None - end = None - ground_station = None - satellite = None - transmitter = None - norad = 0 - +TLEs = defaultdict(list) +Transmitters = defaultdict(dict) +StationsPasses = defaultdict(list) +SatDescrip = defaultdict(str) +CZML = [] + + +def getFuture(): + print("Getting future Passes") + global Sats + global TLEs + observations = defaultdict(dict) + start = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S%z') + end = (datetime.utcnow() + timedelta(hours=4, minutes=30)) + end = end.strftime('%Y-%m-%dT%H:%M:%S%z') + passes = get_paginated_endpoint("https://network.satnogs.org/api/jobs/") + obs = get_paginated_endpoint("https://network.satnogs.org/api/observations/?end="+end+"&format=json&start="+start) + for x in tqdm(obs): + observations[x["id"]] = x + Sats = defaultdict(list) + for x in tqdm(passes): + if x["id"] in observations: + try: + start = datetime.strptime(x["start"], '%Y-%m-%dT%H:%M:%Sz') + start = start.replace(tzinfo=utc) + end = datetime.strptime(x["end"], '%Y-%m-%dT%H:%M:%Sz') + end = end.replace(tzinfo=utc) + # "transmitter":Transmitters[observations[x["id"]]["norad_cat_id"]][x["transmitter"]] + Sats[observations[x["id"]]["norad_cat_id"]].append({"station": x["ground_station"], "transmitter": Transmitters[observations[x["id"]]["norad_cat_id"]][x["transmitter"]], "start": start, "end": end, "id": x["id"]}) + TLEs[observations[x["id"]]["norad_cat_id"]] = [x["tle0"], x["tle1"], x["tle2"]] + StationsPasses[x["ground_station"]].append({"norad": observations[x["id"]]["norad_cat_id"], "transmitter": Transmitters[observations[x["id"]]["norad_cat_id"]][x["transmitter"]], "start": start, "end": end, "id": x["id"]}) + except Exception as e: + print("Error on observation number: " + str(x["id"]) + " " + str(e)) + broken[observations[x["id"]]["norad_cat_id"]].update([x["transmitter"]]) + print(str(len(Sats))+" Future passes found.") -def getActive(): - start = (datetime.utcnow() - timedelta(0,0,0,0,20)).strftime('%Y-%m-%dT%H:%M:%S%z') - end = (datetime.utcnow() + timedelta(0,0,0,0,30)).strftime('%Y-%m-%dT%H:%M:%S%z') - - passesR = requests.get("https://network.satnogs.org/api/observations/?end="+end+"&format=json&start="+start) - passes = passesR.json() - if passesR.links.has_key("next"): - while passesR.links.has_key("next"): - passesR = requests.get(passesR.links["next"]["url"]) - passes += passesR.json() - ground_stations = {} - for x in passes: - if datetime.strptime(x["start"],'%Y-%m-%dT%H:%M:%Sz') > datetime.utcnow() or datetime.strptime(x["end"],'%Y-%m-%dT%H:%M:%Sz') < datetime.utcnow(): - passes.remove(x) - else: - if ground_stations.has_key(x["ground_station"]): - ground_stations[x["ground_station"]].append(x) - else: - ground_stations[x["ground_station"]] = [] - ground_stations[x["ground_station"]].append(x) - passes = [] - for x in ground_stations: - start = datetime.utcnow() - current = {"start":datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S%z')+"z"} - for y in ground_stations[x]: - if datetime.strptime(y["start"],'%Y-%m-%dT%H:%M:%Sz') < datetime.strptime(current["start"],'%Y-%m-%dT%H:%M:%Sz'): - current = y - passes.append(current) - Passes =[] - for x in passes: - temp = Pass() - temp.id = x["id"] - temp.start = datetime.strptime(x["start"],'%Y-%m-%dT%H:%M:%Sz') - temp.end = datetime.strptime(x["end"],'%Y-%m-%dT%H:%M:%Sz') - temp.ground_station = x["ground_station"] - temp.transmitter = x["transmitter"] - temp.norad = str(x["norad_cat_id"]) - try: - temp.satellite = requests.get("https://db.satnogs.org/api/satellites/"+str(x["norad_cat_id"])).json() - except: - temp.satellite = {"name":""} - Passes.append(temp) - - - - return Passes - - def GetGroundStations(): - stationsR = requests.get("https://network.satnogs.org/api/stations/") - stations = stationsR.json() - while stationsR.links.has_key("next"): - stationsR = requests.get(stationsR.links["next"]["url"]) - stations += stationsR.json() - - for x in stations: - if x["last_seen"] == None: - stations.remove(x) - continue - - if datetime.strptime(x["last_seen"],'%Y-%m-%dT%H:%M:%Sz') < (datetime.utcnow()- timedelta(10,0,0,0)): - stations.remove(x) - for x in stations: - if x["last_seen"] == None: - stations.remove(x) - continue - - if datetime.strptime(x["last_seen"],'%Y-%m-%dT%H:%M:%Sz') < (datetime.utcnow()- timedelta(10,0,0,0)): - stations.remove(x) - - + print("Getting Ground Stations") + stations = get_paginated_endpoint("https://network.satnogs.org/api/stations/") return stations - -@scheduler.scheduled_job('interval',days=5) -def updateTransmitters(): - global Transmitters - print "Updating Transmitters" - temp = requests.get("https://db.satnogs.org/api/transmitters/").json() - for x in temp: - if str(x["norad_cat_id"]) in Transmitters.keys(): - Transmitters[str(x["norad_cat_id"])][x["uuid"]] = [x["description"],"#"+str("%06x" % random.randint(0, 0xFFFFFF))] - else: - Transmitters[str(x["norad_cat_id"])]={} - Transmitters[str(x["norad_cat_id"])][x["uuid"]] = [x["description"],"#"+str("%06x" % random.randint(0, 0xFFFFFF))] - #print Transmitters - -@scheduler.scheduled_job('interval',minutes=3) -def updatePasses(): - global Passes - global Occuring_sats - print "Updating Passes" - Passes = getActive() - Occuring_sats = {} - for x in Passes: - if x.satellite['norad_cat_id'] not in TLEs.keys(): - q = fetch_tle_of_observation(x.id) - TLEs[ x.norad ] = [str(x.satellite["name"]),str(q[0]),str(q[1])] - Occuring_sats[x.norad] = TLEs[x.norad] - -@scheduler.scheduled_job('interval',hours=1) -def updateStations(): - global Stations - print "Updating Stations" - Stations = GetGroundStations() - -@scheduler.scheduled_job('interval',days=1) + + def updateTLE(): - print "Updating TLE" - global TlEs - sats = fetch_satellites(None,DB_BASE_URL) + print("Updating TLE") + global TLEs + sats = fetch_satellites(None, DB_BASE_URL) satnogs_db_norad_ids = set(sat['norad_cat_id'] for sat in sats if sat['status'] != 're-entered') # Remove satellites with temporary norad ids temporary_norad_ids = set(filter(lambda norad_id: norad_id >= 99900, satnogs_db_norad_ids)) satnogs_db_norad_ids = satnogs_db_norad_ids - temporary_norad_ids # Fetch TLEs for the satellites of interest - + tles = fetch_tles(satnogs_db_norad_ids) TLEs = {} - for norad_id, (source, tle) in tles.items(): - TLEs[norad_id] = [str(tle[0]),str(tle[1]),str(tle[2])] + for norad_id, (source, tle) in tqdm(tles.items()): + TLEs[norad_id] = [str(tle[0]), str(tle[1]), str(tle[2])] print('\nTLEs for {} of {} requested satellites found ({} satellites with temporary norad ids skipped).'.format(len(tles), len(satnogs_db_norad_ids), len(temporary_norad_ids))) +@scheduler.scheduled_job('interval', days=5) +def updateTransmitters(): + global Transmitters + print("Updating Transmitters") + temp = requests.get("https://db.satnogs.org/api/transmitters/").json() + for x in tqdm(temp): + Transmitters[x["norad_cat_id"]][x["uuid"]] = [x["description"], [random.randint(0, 255), random.randint(0, 255), random.randint(0, 255), 255]] + + for x in Transmitters.keys(): + for y in Transmitters[x].keys(): + SatDescrip[x] += '
'+Transmitters[x][y][0]+'
' + + +@scheduler.scheduled_job('interval', hours=1) +def updatePasses(): + getFuture() + + +@scheduler.scheduled_job('interval', hours=1) +def updateStations(): + global Stations + print("Updating Stations") + Stations = GetGroundStations() + + +@scheduler.scheduled_job('interval', minutes=30) +def updateCZML(): + global CZML + CZML = [] + doc = {} + doc["id"] = "document" + doc["name"] = "sats" + doc["version"] = "1.0" + doc["clock"] = {} + doc["clock"]["interval"] = "0000-00-00T00:00:00Z/9999-12-31T24:00:00Z" + doc["clock"]["step"] = "SYSTEM_CLOCK" + CZML.append(doc) + + for x in tqdm(Stations): + color = [0, 230, 64, 255] + if x["status"] == "Testing": + color = [248, 148, 6, 255] + if x["status"] == "Offline": + color = [255, 0, 0, 50] + + station = {} + station["id"] = str(x["id"]) + station["name"] = x["name"] + station["point"] = {} + station["show"] = True + station["point"]["color"] = {} + station["point"]["color"]["rgba"] = color + station["point"]["outlineColor"] = {} + station["point"]["outlineColor"]["rgba"] = [255, 255, 255, color[3]] + station["point"]["outlineWidth"] = 2.0 + station["position"] = {} + station["point"]["pixelSize"] = 7.0 + station["position"]["cartographicDegrees"] = [x["lng"], x['lat'], x["altitude"]] + station["description"] = "ID: "+str(x["id"]) + "
Total Observations: " + station["description"] += str(x["observations"]) + "
Status: " + x["status"] + "
QTH: " + station["description"] += x["qthlocator"] + "
Description: " + x["description"] + CZML.append(station) + + for x in tqdm(Sats.keys()): + for y in Sats[x]: + sat = {} + sat["id"] = str(y["id"]) + sat["name"] = TLEs[x][0] + sat["show"] = True + sat["billboard"] = {"image": "static/sat.png", "scale": 0.50} + sat["position"] = {} + sat["position"]["cartographicDegrees"] = [] + sat["description"] = SatDescrip[x] + temp = y["start"] + satObj = EarthSatellite(TLEs[x][1], TLEs[x][2], TLEs[x][0]) + time = 0 + while temp <= y["end"] + timedelta(minutes=1): + subpoint = satObj.at(ts.utc(temp)).subpoint() + lat = subpoint.latitude.degrees + lng = subpoint.longitude.degrees + elevation = subpoint.elevation.m + sat["position"]["cartographicDegrees"].extend([time, lng, lat, elevation]) + time += 60 + temp = temp + timedelta(minutes=1) + sat["position"]["interpolationAlgorithm"] = "LAGRANGE" + sat["position"]["interpolationDegree"] = 5 + sat["position"]["epoch"] = (y["start"].isoformat()+"Z").replace("+00:00", "") + sat["path"] = {"show": {"interval": (y["start"].isoformat()+"Z").replace("+00:00", "") + "/" + ((y["end"] + timedelta(minutes=1)).isoformat()+"Z").replace("+00:00", ""), "boolean": True}, "width": 2, "material": {"solidColor": {"color": {"rgba": [0, 255, 0, 255]}}}, "leadTime": 100000, "trailTime": 100000} + CZML.append(sat) + for x in tqdm(Sats.keys()): + for y in Sats[x]: + sat = {} + sat["id"] = str(y["id"])+"Link" + sat["polyline"] = {"show": {"interval": (y["start"].isoformat()+"Z").replace("+00:00", "")+"/"+((y["end"]+timedelta(minutes=1)).isoformat()+"Z").replace("+00:00", ""), "boolean": True}, "width": 2, "material": {"solidColor": {"color": {"rgba": y["transmitter"][1]}}}, "followSurface": False, "positions": {"references": [str(y["id"])+"#position", str(y["station"]) + "#position"]}} + CZML.append(sat) @app.route("/") -def map_view(): - return render_template("map.html") - -@app.route('/active_stations') -def api_active_stations(): - sations = [] +def index(): + Usage.update([str(request.headers.get("CF-IPCountry"))]) + return render_template("index.html",url="/czml") + +@app.route("/rotating") +def rotating(): + speed = request.args.get('speed', default = '30', type = str) + Usage.update([str(request.headers.get("CF-IPCountry"))]) + return render_template("rotating.html",url="/czml",speed=speed) + +@app.route("/station/") +def index_station(station_id): + Usage.update(request.headers.get("CF-IPCountry")) + return render_template("index.html",url="/czml/"+str(station_id)) + +@app.route('/future_sats') +def api_future_sats(): + return json.dumps(Sats) + + +@app.route("/czml") +def api_czml(): + return json.dumps(CZML) + + +@app.route("/broken") +def api_broken(): + output = defaultdict(list) + for x in broken.keys(): + output[x]=list(broken[x]) + return json.dumps(output) + +@app.route("/czml/") +def api_station(station_id): + czml = [] + doc = {} + doc["id"] = "document" + doc["name"] = "sats" + doc["version"] = "1.0" + doc["clock"] = {} + doc["clock"]["interval"] = "0000-00-00T00:00:00Z/9999-12-31T24:00:00Z" + doc["clock"]["currentTime"] = datetime.utcnow().isoformat() + "Z" + doc["clock"]["step"] = "SYSTEM_CLOCK" + czml.append(doc) + for x in Stations: - sations.append({'id':x['id'],'name':x['name'],'lat_lng':[x["lat"],x['lng']]}) - return json.dumps(sations) - -@app.route('/stations_from_sat/') -def api_occuring_observations(norad): - obs = [] - trans = [] - for x in Passes: - if x.norad == norad: - obs.append([x.ground_station,Transmitters[norad][x.transmitter][1]]) - trans.append(x.transmitter) - #print Transmitters[norad].values() - - transList = [] - for x in set(trans): - transList.append(Transmitters[norad][x]) - #print transList,norad - return json.dumps([obs,transList]) - -@app.route('/occuring_sats') -def api_occuring_sats(): - return json.dumps(Occuring_sats) - + if x["id"] == station_id: + color = [0, 230, 64, 255] + if x["status"] == "Testing": + color = [248, 148, 6, 255] + if x["status"] == "Offline": + color = [255, 0, 0, 50] + station = {} + station["id"] = str(x["id"]) + station["name"] = x["name"] + station["point"] = {} + station["show"] = True + station["point"]["color"] = {} + station["point"]["color"]["rgba"] = color + station["point"]["outlineColor"] = {} + station["point"]["outlineColor"]["rgba"] = [255, 255, 255, color[3]] + station["point"]["outlineWidth"] = 2.0 + station["position"] = {} + station["point"]["pixelSize"] = 7.0 + station["position"]["cartographicDegrees"] = [x["lng"], x['lat'], x["altitude"]] + station["description"] = "ID: " + str(x["id"]) + "
Total Observations: " + station["description"] += str(x["observations"]) + "
Status: " + x["status"] + "
QTH: " + station["description"] += x["qthlocator"] + "
Description: " + x["description"] + czml.append(station) + break + + for y in StationsPasses[station_id]: + sat = {} + sat["id"] = str(y["id"]) + sat["name"] = TLEs[y["norad"]][0] + sat["show"] = True + sat["point"] = {} + sat["point"]["color"] = {} + sat["point"]["color"]["rgba"] = [255, 0, 0, 255] + sat["point"]["pixelSize"] = 8.0 + sat["position"] = {} + sat["position"]["cartographicDegrees"] = [] + sat["description"] = SatDescrip[y["norad"]] + temp = y["start"] + satObj = EarthSatellite(TLEs[y["norad"]][1], TLEs[y["norad"]][2], TLEs[y["norad"]][0]) + time = 0 + while temp <= y["end"]+timedelta(minutes=1): + subpoint = satObj.at(ts.utc(temp)).subpoint() + lat = subpoint.latitude.degrees + lng = subpoint.longitude.degrees + elevation = subpoint.elevation.m + sat["position"]["cartographicDegrees"].extend([time, lng, lat, elevation]) + time += 60 + temp = temp + timedelta(minutes=1) + sat["position"]["interpolationAlgorithm"] = "LAGRANGE" + sat["position"]["interpolationDegree"] = 5 + sat["position"]["epoch"] = (y["start"].isoformat() + "Z").replace("+00:00", "") + sat["path"] = {"show": {"interval": (y["start"].isoformat()+"Z").replace("+00:00", "") + "/" + ((y["end"] + timedelta(minutes=1)).isoformat() + "Z").replace("+00:00", ""), "boolean": True}, "width": 2, "material": {"solidColor": {"color": {"rgba": [0, 255, 0, 255]}}}, "leadTime": 100000, "trailTime": 100000} + czml.append(sat) + for y in StationsPasses[station_id]: + sat = {} + sat["id"] = str(y["id"])+"Link" + sat["polyline"] = {"show": {"interval": (y["start"].isoformat()+"Z").replace("+00:00", "") + "/" + ((y["end"] + timedelta(minutes=1)).isoformat()+"Z").replace("+00:00", ""), "boolean": True}, "width": 2, "material": {"solidColor": {"color": {"rgba": y["transmitter"][1]}}}, "followSurface": False, "positions": {"references": [str(y["id"]) + "#position", str(station_id) + "#position"]}} + czml.append(sat) + return json.dumps(czml) - - -updatePasses() updateStations() -updateTLE() updateTransmitters() +getFuture() +updateCZML() scheduler.start() -app.run(use_reloader=False,host = "0.0.0.0",port=5001) \ No newline at end of file +app.run(use_reloader=False, host="0.0.0.0", port=5001) diff --git a/satnogs_api_client.pyc b/satnogs_api_client.pyc deleted file mode 100644 index 91a2e0b..0000000 Binary files a/satnogs_api_client.pyc and /dev/null differ diff --git a/static/Worker.js b/static/Worker.js deleted file mode 100644 index 72c91e0..0000000 --- a/static/Worker.js +++ /dev/null @@ -1,48 +0,0 @@ -self.importScripts("satellite.js"); - -norad = "" -groundStations = [] -TLE = [] - -onmessage = function(e) { - norad = e.data[0] - TLE = e.data[1] - getStations() -} - -setInterval(function(){ - getStations() -}, 20000); - -setInterval(function(){ - var satrec = self.satellite_js.twoline2satrec(TLE[1],TLE[2]); - var gmst = self.satellite_js.gstime(new Date()); - var positionAndVelocity = self.satellite_js.propagate(satrec, new Date()); - var positionEci = positionAndVelocity.position - var positionGd = self.satellite_js.eciToGeodetic(positionEci, gmst) - var longitude = positionGd.longitude - var latitude = positionGd.latitude - - if (groundStations[1] == undefined){ - groundStations.push([]) - } - - postMessage([norad,TLE[0],[degress(latitude),degress(longitude)],groundStations[0],groundStations[1]]) - - -}, 500); - -function getStations(){ -var xhttp = new XMLHttpRequest(); - xhttp.onreadystatechange = function() { - if (this.readyState == 4 && this.status == 200) { - groundStations = JSON.parse(this.responseText); - } - }; - xhttp.open("GET", "/stations_from_sat/"+norad, true); - xhttp.send(); -} - -function degress (radians) { - return radians * 180 / Math.PI; -}; \ No newline at end of file diff --git a/static/active-station-marker.png b/static/active-station-marker.png deleted file mode 100644 index 0d9450c..0000000 Binary files a/static/active-station-marker.png and /dev/null differ diff --git a/static/satellite-marker-dark.png b/static/sat.png similarity index 100% rename from static/satellite-marker-dark.png rename to static/sat.png diff --git a/static/satellite-marker-light.png b/static/satellite-marker-light.png deleted file mode 100644 index af17f1b..0000000 Binary files a/static/satellite-marker-light.png and /dev/null differ diff --git a/static/satellite.js b/static/satellite.js deleted file mode 100644 index e294cab..0000000 --- a/static/satellite.js +++ /dev/null @@ -1 +0,0 @@ -!function(o,t){"object"==typeof exports&&"undefined"!=typeof module?t(exports):"function"==typeof define&&define.amd?define(["exports"],t):t(o.satellite_js={})}(this,function(o){"use strict";var Ho=Math.PI,Yo=2*Ho,m=Ho/180,t=180/Ho,s=398600.5,Ao=6378.137,Bo=60/Math.sqrt(Ao*Ao*Ao/s),p=1/Bo,Co=.00108262998905,e=-253215306e-14,Uo=-161098761e-14,Do=e/Co,Jo=2/3,a=Object.freeze({pi:Ho,twoPi:Yo,deg2rad:m,rad2deg:t,minutesPerDay:1440,mu:s,earthRadius:Ao,xke:Bo,tumin:p,j2:Co,j3:e,j4:Uo,j3oj2:Do,x2o3:Jo});function l(o,t){for(var s=[31,o%4==0?29:28,31,30,31,30,31,31,30,31,30,31],e=Math.floor(t),a=1,n=0;n+s[a-1]Ho&&(so + + + + + + + + + +
+ + + +Creative Commons License
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License diff --git a/templates/map.html b/templates/map.html deleted file mode 100644 index 3b9b0ab..0000000 --- a/templates/map.html +++ /dev/null @@ -1,195 +0,0 @@ - - - - - - - - - - - -
-
. -
-
- -
-
-
- -
-
-
-
- - - - - -Creative Commons License
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License diff --git a/templates/rotating.html b/templates/rotating.html new file mode 100644 index 0000000..ad2dec6 --- /dev/null +++ b/templates/rotating.html @@ -0,0 +1,69 @@ + + + + + + + + + + +
+ + + + +Creative Commons License
This work is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License