spacecruft
/
satnogs-wut
0
0
Fork 0
Code Issues 6 Releases Activity

clean on wut-predict jupyter draft

master
ml server 2020-01-16 19:25:44 -07:00
parent a45c228d49
commit 998b9ad154
4 changed files with 9 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

330
jupyter/wut-predict.ipynb
View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -24,7 +24,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -164,51 +164,9 @@
"metadata": {},
"outputs": [],
"source": [
"train_dir = os.path.join('data/', 'train')\n",
"val_dir = os.path.join('data/', 'val')\n",
"test_dir = os.path.join('data/', 'test')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train_good_dir = os.path.join(train_dir, 'good')\n",
"train_bad_dir = os.path.join(train_dir, 'bad')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"val_good_dir = os.path.join(val_dir, 'good')\n",
"val_bad_dir = os.path.join(val_dir, 'bad')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"num_train_good = len(os.listdir(train_good_dir))\n",
"num_train_bad = len(os.listdir(train_bad_dir))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"num_val_good = len(os.listdir(val_good_dir))\n",
"num_val_bad = len(os.listdir(val_bad_dir))"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -218,55 +176,6 @@
"num_test = len(os.listdir(test_dir))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"total_train = num_train_good + num_train_bad\n",
"total_val = num_val_good + num_val_bad"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print('total training good images:', num_train_good)\n",
"print('total training bad images:', num_train_bad)\n",
"print(\"--\")\n",
"print(\"Total training images:\", total_train)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print('total validation good images:', num_val_good)\n",
"print('total validation bad images:', num_val_bad)\n",
"print(\"--\")\n",
"print(\"Total validation images:\", total_val)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"Reduce training and validation set when testing\")\n",
"total_train = 300\n",
"total_val = 150\n",
"print(\"Train =\")\n",
"print(total_train)\n",
"print(\"Validation =\")\n",
"print(total_val)"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -295,28 +204,6 @@
"#IMG_WIDTH = 1606"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train_image_generator = ImageDataGenerator(\n",
" rescale=1./255\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"val_image_generator = ImageDataGenerator(\n",
" rescale=1./255\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -328,31 +215,6 @@
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,\n",
" directory=train_dir,\n",
" shuffle=True,\n",
" target_size=(IMG_HEIGHT, IMG_WIDTH),\n",
" class_mode='binary')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"val_data_gen = val_image_generator.flow_from_directory(batch_size=batch_size,\n",
" directory=val_dir,\n",
" target_size=(IMG_HEIGHT, IMG_WIDTH),\n",
" class_mode='binary')"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -374,24 +236,6 @@
" class_mode='binary')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sample_train_images, _ = next(train_data_gen)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sample_val_images, _ = next(val_data_gen)"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -418,24 +262,6 @@
" plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plotImages(sample_train_images[0:3])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plotImages(sample_val_images[0:3])"
]
},
{
"cell_type": "code",
"execution_count": null,
@ -447,159 +273,11 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"model = Sequential([\n",
" Conv2D(16, 3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),\n",
" MaxPooling2D(),\n",
" Conv2D(32, 3, padding='same', activation='relu'),\n",
" MaxPooling2D(),\n",
" Conv2D(64, 3, padding='same', activation='relu'),\n",
" MaxPooling2D(),\n",
" Flatten(),\n",
" Dense(512, activation='relu'),\n",
" Dense(1, activation='sigmoid')\n",
"])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.compile(optimizer='adam',\n",
" loss='binary_crossentropy',\n",
" metrics=['accuracy'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"Image.LOAD_TRUNCATED_IMAGES = True"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"history = model.fit_generator(\n",
" train_data_gen,\n",
" steps_per_epoch=total_train // batch_size,\n",
" epochs=epochs,\n",
" validation_data=val_data_gen,\n",
" validation_steps=total_val // batch_size,\n",
" verbose=1\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"acc = history.history['accuracy']\n",
"val_acc = history.history['val_accuracy']\n",
"\n",
"loss = history.history['loss']\n",
"val_loss = history.history['val_loss']\n",
"\n",
"epochs_range = range(epochs)\n",
"\n",
"plt.figure(figsize=(8, 8))\n",
"plt.subplot(1, 2, 1)\n",
"plt.plot(epochs_range, acc, label='Training Accuracy')\n",
"plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n",
"plt.legend(loc='lower right')\n",
"plt.title('Training and Validation Accuracy')\n",
"\n",
"plt.subplot(1, 2, 2)\n",
"plt.plot(epochs_range, loss, label='Training Loss')\n",
"plt.plot(epochs_range, val_loss, label='Validation Loss')\n",
"plt.legend(loc='upper right')\n",
"plt.title('Training and Validation Loss')\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"TRAINING info\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_dir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_good_dir)\n",
"print(train_bad_dir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_image_generator)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_data_gen)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#print(sample_train_images)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(history)"
"# load .h5 file here plz"
]
},
{

2
wut-audio-archive
View File

@ -24,7 +24,7 @@
# https://archive.org/details/satnogs?sort=-publicdate
# Run:
# wut-audio-archive 1292461 1368693
# wut-audio-archive 1346458 1368693
# wut-audio-archive 1348064 1368693
#
# XXX Should check input is sane...

2
wut-rm-random
View File

@ -20,7 +20,7 @@
KEEP=100
# this is so bad no one should ever run it again
exit 0
#exit 0
# XXX Delete data in this directory! XXX
cd data/test/unvetted/ || exit

6
wut-water-range
View File

@ -16,7 +16,7 @@
# So to get mostly all of the observations in December, 2019, run:
# wut-water-range 1292461 1470525
# Resume:
# wut-water-range 1355760 1470525
# wut-water-range 1359119 1470525
#
# XXX Should check input is sane...
@ -38,7 +38,7 @@ while [ $OBSID -lt $OBSIDMAX ]
--http2 --ipv4 \
--silent \
--output $OBSID.json \
"$APIURL/observations/?id=$OBSID&ground_station=&satellite__norad_cat_id=&transmitter_uuid=&transmitter_mode=&transmitter_type=&vetted_status=&vetted_user=&start=&end=" && sleep `echo $((0 + RANDOM % 3))`
"$APIURL/observations/?id=$OBSID&ground_station=&satellite__norad_cat_id=&transmitter_uuid=&transmitter_mode=&transmitter_type=&vetted_status=&vetted_user=&start=&end=" && sleep `echo $((0 + RANDOM % 2))`
WATERURL=`cat $OBSID.json | jq --compact-output '.[0] | {waterfall}' | cut -f 2- -d : | sed -e 's/}//g' -e 's/http:/https:/g' -e 's/"//g'`
WATERFILE=`basename $WATERURL`
[ ! -f "$WATERFILE" ] && \
@ -48,7 +48,7 @@ while [ $OBSID -lt $OBSIDMAX ]
--continue-at - \
--remote-time \
--output $WATERFILE \
$WATERURL && sleep `echo $((0 + RANDOM % 11))`
$WATERURL && sleep `echo $((0 + RANDOM % 3))`
cd ..
let OBSID=$OBSID+1
done