ml server
parent
4a46d037d3
commit
45c0fcd5df
|
|
@ -0,0 +1,287 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# wut-web --- What U Think? Web App: SatNOGS Observation AI, makes predictions.\n",
|
||||
"#\n",
|
||||
"# https://spacecruft.org/spacecruft/satnogs-wut\n",
|
||||
"#\n",
|
||||
"# GPLv3+\n",
|
||||
"\n",
|
||||
"#from collections import defaultdict\n",
|
||||
"#import PIL as pil\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import random\n",
|
||||
"import tempfile\n",
|
||||
"import shutil\n",
|
||||
"import tensorflow as tf\n",
|
||||
"import ipywidgets as wg\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from IPython.display import display, Image\n",
|
||||
"from IPython.utils import text\n",
|
||||
"from PIL import Image as im\n",
|
||||
"from tensorflow.python.keras.models import load_model\n",
|
||||
"from tensorflow.python.keras.preprocessing.image import ImageDataGenerator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"display(Image(filename='/srv/satnogs/satnogs-wut/pics/spacecruft-bk.png'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%HTML\n",
|
||||
"<H1><B>wut?<B></H1>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"wut? --- What U Think? SatNOGS Observation AI.\")\n",
|
||||
"print(\"\")\n",
|
||||
"print(\"wut is an AI that rates SatNOGS Observations good or bad.\")\n",
|
||||
"print(\"The training model was built from DUV transmissions recorded by the\")\n",
|
||||
"print(\"SatNOGS network in December, 2019.\")\n",
|
||||
"print(\"The plan is to have models of all SatNOGS modes (65 at present),\")\n",
|
||||
"print(\"and you can enter an arbitrary Observation ID and the AI will return a rating.\")\n",
|
||||
"print(\"\")\n",
|
||||
"print(\"Source Code:\")\n",
|
||||
"print(\"https://spacecruft.org/spacecruft/satnogs-wut\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"IMG_HEIGHT = 416\n",
|
||||
"IMG_WIDTH = 804\n",
|
||||
"batch_size = 32\n",
|
||||
"minobsid = 1292461\n",
|
||||
"#maxobsid = 1470525\n",
|
||||
"maxobsid = 1591638\n",
|
||||
"base_dir = ('/srv/wut/data')\n",
|
||||
"sample_dir = ('/srv/wut/data/test/unvetted')\n",
|
||||
"model_file = os.path.join(base_dir, 'models', 'wut-DUV-201912.tf')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"model = load_model(model_file)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def gen_image(test_data_gen,test_dir):\n",
|
||||
" test_image_gen = ImageDataGenerator(rescale=1./255);\n",
|
||||
" test_data_gen = test_image_gen.flow_from_directory(batch_size=1,\n",
|
||||
" directory=test_dir,\n",
|
||||
" target_size=(IMG_HEIGHT, IMG_WIDTH),\n",
|
||||
" shuffle=True,\n",
|
||||
" class_mode='binary')\n",
|
||||
" return test_data_gen"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def rm_image_tmp(test_dir):\n",
|
||||
" #print('Not removed:', test_dir)\n",
|
||||
" shutil.rmtree(test_dir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture --no-stderr --no-stdout\n",
|
||||
"def gen_image_tmp(obs_waterfalltmp):\n",
|
||||
" tmp_dir = tempfile.mkdtemp()\n",
|
||||
" test_dir = os.path.join(tmp_dir)\n",
|
||||
" os.makedirs(test_dir + '/unvetted', exist_ok=True)\n",
|
||||
" shutil.copy(obs_waterfalltmp, test_dir + '/unvetted/') \n",
|
||||
" \n",
|
||||
" img = im.open(obs_waterfalltmp).resize( (100,200))\n",
|
||||
" display(img)\n",
|
||||
"\n",
|
||||
" return test_dir"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def obs_wutsay(test_data_gen):\n",
|
||||
" prediction = model.predict(\n",
|
||||
" x=test_data_gen,\n",
|
||||
" verbose=0)\n",
|
||||
" predictions=[]\n",
|
||||
" prediction_bool = (prediction >0.8)\n",
|
||||
" predictions = prediction_bool.astype(int)\n",
|
||||
" \n",
|
||||
" return prediction_bool"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_obs_dict(datObs):\n",
|
||||
" obsjsonfile=('/srv/satnogs/download/' + format(datObs) + '/' + format(datObs) + '.json')\n",
|
||||
" with open(obsjsonfile) as f:\n",
|
||||
" content = f.read()\n",
|
||||
" data = json.loads(content)\n",
|
||||
" res = {x : data[x] for x in range(len(data))}\n",
|
||||
" res2 = dict(enumerate(data))\n",
|
||||
" obs_dict=(res2[0])\n",
|
||||
" \n",
|
||||
" return obs_dict"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_obs_var(var, datObs):\n",
|
||||
" obs_dict=get_obs_dict(datObs);\n",
|
||||
" obs_var=(obs_dict[(var)])\n",
|
||||
" \n",
|
||||
" return obs_var"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def doallthethings(datObs):\n",
|
||||
"\n",
|
||||
" obs_waterfall=get_obs_var('waterfall', datObs) \n",
|
||||
" obs_waterfallpic=os.path.basename(obs_waterfall)\n",
|
||||
" obs_waterfalltmp = os.path.join('/srv/satnogs/download', str(get_obs_var('id', datObs)), obs_waterfallpic)\n",
|
||||
"\n",
|
||||
" test_dir=gen_image_tmp(obs_waterfalltmp);\n",
|
||||
" test_data_gen=gen_image(obs_waterfalltmp, test_dir);\n",
|
||||
" \n",
|
||||
" prediction_bool=obs_wutsay(test_data_gen);\n",
|
||||
"\n",
|
||||
" print()\n",
|
||||
" print('Observation ID: ', get_obs_var('id', datObs))\n",
|
||||
" print('Encoding: ', get_obs_var('transmitter_mode', datObs))\n",
|
||||
" print('Human rating: ', get_obs_var('vetted_status', datObs))\n",
|
||||
" if prediction_bool[0] == False:\n",
|
||||
" rating = 'bad'\n",
|
||||
" else:\n",
|
||||
" rating = 'good'\n",
|
||||
" print('wut AI rating: %s' % (rating)) \n",
|
||||
" print()\n",
|
||||
" if get_obs_var('transmitter_mode', datObs) == 'DUV':\n",
|
||||
" print(\"Using DUV training model.\")\n",
|
||||
" else:\n",
|
||||
" print(\"NOTE: wut has not been trained on\", get_obs_var('transmitter_mode', datObs), \"encodings.\")\n",
|
||||
" print('https://network.satnogs.org/observations/' + str(get_obs_var('id', datObs)))\n",
|
||||
" #!cat $obsjsonfile\n",
|
||||
" rm_image_tmp(test_dir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def wutObs(datObs):\n",
|
||||
" if int(datObs) > ( minobsid - 1 ) and int(datObs) < ( maxobsid + 1):\n",
|
||||
" doallthethings(datObs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def display_results():\n",
|
||||
" print('Enter an Observation ID between', minobsid, 'and', maxobsid)\n",
|
||||
" wutObs_slide = wg.IntText(value='1292461')\n",
|
||||
" wg.interact(wutObs, datObs=wutObs_slide)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"display_results()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# wut-web --- What U Think? Web App: SatNOGS Observation AI, makes predictions.\n",
|
||||
"#\n",
|
||||
"# https://spacecruft.org/spacecruft/satnogs-wut\n",
|
||||
"#\n",
|
||||
"# GPLv3+\n",
|
||||
"\n",
|
||||
"#from collections import defaultdict\n",
|
||||
"#import PIL as pil\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import random\n",
|
||||
"import tempfile\n",
|
||||
"import shutil\n",
|
||||
"import tensorflow as tf\n",
|
||||
"import ipywidgets as wg\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from IPython.display import display, Image\n",
|
||||
"from IPython.utils import text\n",
|
||||
"from PIL import Image as im\n",
|
||||
"from tensorflow.python.keras.models import load_model\n",
|
||||
"from tensorflow.python.keras.preprocessing.image import ImageDataGenerator"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"display(Image(filename='/srv/satnogs/satnogs-wut/pics/spacecruft-bk.png'))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%HTML\n",
|
||||
"<H1><B>wut?<B></H1>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(\"wut? --- What U Think? SatNOGS Observation AI.\")\n",
|
||||
"print(\"\")\n",
|
||||
"print(\"wut is an AI that rates SatNOGS Observations good or bad.\")\n",
|
||||
"print(\"The training model was built from DUV transmissions recorded by the\")\n",
|
||||
"print(\"SatNOGS network in December, 2019.\")\n",
|
||||
"print(\"The plan is to have models of all SatNOGS modes (65 at present),\")\n",
|
||||
"print(\"and you can enter an arbitrary Observation ID and the AI will return a rating.\")\n",
|
||||
"print(\"\")\n",
|
||||
"print(\"Source Code:\")\n",
|
||||
"print(\"https://spacecruft.org/spacecruft/satnogs-wut\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"IMG_HEIGHT = 416\n",
|
||||
"IMG_WIDTH = 804\n",
|
||||
"batch_size = 32\n",
|
||||
"minobsid = 1292461\n",
|
||||
"#maxobsid = 1470525\n",
|
||||
"maxobsid = 1591638\n",
|
||||
"base_dir = ('/srv/wut/data')\n",
|
||||
"sample_dir = ('/srv/wut/data/test/unvetted')\n",
|
||||
"model_file = os.path.join(base_dir, 'models', 'wut-DUV-201912.tf')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"model = load_model(model_file)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def gen_image(test_data_gen,test_dir):\n",
|
||||
" test_image_gen = ImageDataGenerator(rescale=1./255);\n",
|
||||
" test_data_gen = test_image_gen.flow_from_directory(batch_size=1,\n",
|
||||
" directory=test_dir,\n",
|
||||
" target_size=(IMG_HEIGHT, IMG_WIDTH),\n",
|
||||
" shuffle=True,\n",
|
||||
" class_mode='binary')\n",
|
||||
" return test_data_gen"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def rm_image_tmp(test_dir):\n",
|
||||
" #print('Not removed:', test_dir)\n",
|
||||
" shutil.rmtree(test_dir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture --no-stderr --no-stdout\n",
|
||||
"def gen_image_tmp(obs_waterfalltmp):\n",
|
||||
" tmp_dir = tempfile.mkdtemp()\n",
|
||||
" test_dir = os.path.join(tmp_dir)\n",
|
||||
" os.makedirs(test_dir + '/unvetted', exist_ok=True)\n",
|
||||
" shutil.copy(obs_waterfalltmp, test_dir + '/unvetted/') \n",
|
||||
" \n",
|
||||
" img = im.open(obs_waterfalltmp).resize( (100,200))\n",
|
||||
" display(img)\n",
|
||||
"\n",
|
||||
" return test_dir"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def obs_wutsay(test_data_gen):\n",
|
||||
" prediction = model.predict(\n",
|
||||
" x=test_data_gen,\n",
|
||||
" verbose=0)\n",
|
||||
" predictions=[]\n",
|
||||
" prediction_bool = (prediction >0.8)\n",
|
||||
" predictions = prediction_bool.astype(int)\n",
|
||||
" \n",
|
||||
" return prediction_bool"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_obs_dict(datObs):\n",
|
||||
" obsjsonfile=('/srv/satnogs/download/' + format(datObs) + '/' + format(datObs) + '.json')\n",
|
||||
" with open(obsjsonfile) as f:\n",
|
||||
" content = f.read()\n",
|
||||
" data = json.loads(content)\n",
|
||||
" res = {x : data[x] for x in range(len(data))}\n",
|
||||
" res2 = dict(enumerate(data))\n",
|
||||
" obs_dict=(res2[0])\n",
|
||||
" \n",
|
||||
" return obs_dict"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def get_obs_var(var, datObs):\n",
|
||||
" obs_dict=get_obs_dict(datObs);\n",
|
||||
" obs_var=(obs_dict[(var)])\n",
|
||||
" \n",
|
||||
" return obs_var"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def doallthethings(datObs):\n",
|
||||
"\n",
|
||||
" obs_waterfall=get_obs_var('waterfall', datObs) \n",
|
||||
" obs_waterfallpic=os.path.basename(obs_waterfall)\n",
|
||||
" obs_waterfalltmp = os.path.join('/srv/satnogs/download', str(get_obs_var('id', datObs)), obs_waterfallpic)\n",
|
||||
"\n",
|
||||
" test_dir=gen_image_tmp(obs_waterfalltmp);\n",
|
||||
" test_data_gen=gen_image(obs_waterfalltmp, test_dir);\n",
|
||||
" \n",
|
||||
" prediction_bool=obs_wutsay(test_data_gen);\n",
|
||||
"\n",
|
||||
" print()\n",
|
||||
" print('Observation ID: ', get_obs_var('id', datObs))\n",
|
||||
" print('Encoding: ', get_obs_var('transmitter_mode', datObs))\n",
|
||||
" print('Human rating: ', get_obs_var('vetted_status', datObs))\n",
|
||||
" if prediction_bool[0] == False:\n",
|
||||
" rating = 'bad'\n",
|
||||
" else:\n",
|
||||
" rating = 'good'\n",
|
||||
" print('wut AI rating: %s' % (rating)) \n",
|
||||
" print()\n",
|
||||
" if get_obs_var('transmitter_mode', datObs) == 'DUV':\n",
|
||||
" print(\"Using DUV training model.\")\n",
|
||||
" else:\n",
|
||||
" print(\"NOTE: wut has not been trained on\", get_obs_var('transmitter_mode', datObs), \"encodings.\")\n",
|
||||
" print('https://network.satnogs.org/observations/' + str(get_obs_var('id', datObs)))\n",
|
||||
" #!cat $obsjsonfile\n",
|
||||
" rm_image_tmp(test_dir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def wutObs(datObs):\n",
|
||||
" if int(datObs) > ( minobsid - 1 ) and int(datObs) < ( maxobsid + 1):\n",
|
||||
" doallthethings(datObs)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%capture\n",
|
||||
"def display_results():\n",
|
||||
" print('Enter an Observation ID between', minobsid, 'and', maxobsid)\n",
|
||||
" wutObs_slide = wg.IntText(value='1292461')\n",
|
||||
" wg.interact(wutObs, datObs=wutObs_slide)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"display_results()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
Loading…
Reference in New Issue