slightly better jupyter

master 0.26
ml server 2020-01-15 12:48:12 -07:00
parent 1a30e049be
commit 7c7e5fabda
3 changed files with 196 additions and 195 deletions

View File

@ -21,7 +21,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -37,24 +37,16 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Start\n"
]
}
],
"outputs": [],
"source": [
"print(\"Start\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -63,7 +55,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -72,7 +64,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -81,7 +73,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -98,7 +90,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -107,7 +99,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -123,7 +115,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -137,7 +129,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -147,7 +139,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -160,7 +152,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -170,24 +162,16 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Python import done\n"
]
}
],
"outputs": [],
"source": [
"print(\"Python import done\")"
]
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -199,17 +183,9 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"datagen\n"
]
}
],
"outputs": [],
"source": [
"# https://keras.io/preprocessing/image/\n",
"# TODO:\n",
@ -245,18 +221,18 @@
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"train_dir = os.path.join('data/', 'train')\n",
"val_dir = os.path.join('data/', 'val')\n",
"test_dir = os.path.join('data/', 'test')"
"test_dir = os.path.join('data/', 'test/unvetted')"
]
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -267,7 +243,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -278,7 +254,17 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#data/test/unvetted/waterfall.png\n",
"test_img = os.path.join(test_dir, 'waterfall.png')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -289,7 +275,7 @@
},
{
"cell_type": "code",
"execution_count": 20,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -300,7 +286,7 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -309,7 +295,7 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -321,20 +307,9 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total training good images: 5754\n",
"total training bad images: 1355\n",
"--\n",
"Total training images: 7109\n"
]
}
],
"outputs": [],
"source": [
"print('total training good images:', num_train_good)\n",
"print('total training bad images:', num_train_bad)\n",
@ -345,20 +320,9 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total validation good images: 5735\n",
"total validation bad images: 1364\n",
"--\n",
"Total validation images: 7099\n"
]
}
],
"outputs": [],
"source": [
"print('total validation good images:', num_val_good)\n",
"print('total validation bad images:', num_val_bad)\n",
@ -369,24 +333,13 @@
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Train =\n",
"7109\n",
"Validation =\n",
"7099\n"
]
}
],
"outputs": [],
"source": [
"#print(\"Reduce training and validation set\")\n",
"#total_train = 250\n",
"#total_val = 200\n",
"print(\"Reduce training and validation set\")\n",
"total_train = 50\n",
"total_val = 50\n",
"print(\"Train =\")\n",
"print(total_train)\n",
"print(\"Validation =\")\n",
@ -395,26 +348,27 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(Image('data/test/unvetted/waterfall.png'))"
"print(test_img)\n",
"display(Image(test_img))"
]
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"batch_size = 256\n",
"epochs = 32"
"batch_size = 16\n",
"epochs = 2"
]
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -428,7 +382,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -439,7 +393,7 @@
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -450,17 +404,9 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found 7109 images belonging to 2 classes.\n"
]
}
],
"outputs": [],
"source": [
"train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,\n",
" directory=train_dir,\n",
@ -471,17 +417,9 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Found 7099 images belonging to 2 classes.\n"
]
}
],
"outputs": [],
"source": [
"val_data_gen = val_image_generator.flow_from_directory(batch_size=batch_size,\n",
" directory=val_dir,\n",
@ -491,7 +429,7 @@
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -500,17 +438,25 @@
},
{
"cell_type": "code",
"execution_count": 34,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sample_val_images, _ = next(val_data_gen)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#sample_val_images, _ = next(val_data_gen)\n",
"#sample_test_images, _ = next(test_data_gen)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -527,7 +473,7 @@
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -536,16 +482,16 @@
},
{
"cell_type": "code",
"execution_count": 37,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#plotImages(sample_val_images[:3])"
"plotImages(sample_val_images[:3])"
]
},
{
"cell_type": "code",
"execution_count": 38,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -564,7 +510,7 @@
},
{
"cell_type": "code",
"execution_count": 39,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -575,49 +521,16 @@
},
{
"cell_type": "code",
"execution_count": 40,
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Model: \"sequential\"\n",
"_________________________________________________________________\n",
"Layer (type) Output Shape Param # \n",
"=================================================================\n",
"conv2d (Conv2D) (None, 416, 804, 16) 448 \n",
"_________________________________________________________________\n",
"max_pooling2d (MaxPooling2D) (None, 208, 402, 16) 0 \n",
"_________________________________________________________________\n",
"conv2d_1 (Conv2D) (None, 208, 402, 32) 4640 \n",
"_________________________________________________________________\n",
"max_pooling2d_1 (MaxPooling2 (None, 104, 201, 32) 0 \n",
"_________________________________________________________________\n",
"conv2d_2 (Conv2D) (None, 104, 201, 64) 18496 \n",
"_________________________________________________________________\n",
"max_pooling2d_2 (MaxPooling2 (None, 52, 100, 64) 0 \n",
"_________________________________________________________________\n",
"flatten (Flatten) (None, 332800) 0 \n",
"_________________________________________________________________\n",
"dense (Dense) (None, 512) 170394112 \n",
"_________________________________________________________________\n",
"dense_1 (Dense) (None, 1) 513 \n",
"=================================================================\n",
"Total params: 170,418,209\n",
"Trainable params: 170,418,209\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n"
]
}
],
"outputs": [],
"source": [
"model.summary()"
]
},
{
"cell_type": "code",
"execution_count": 41,
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@ -628,20 +541,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/32\n",
"27/27 [==============================] - 3214s 119s/step - loss: 2.8449 - accuracy: 0.8087 - val_loss: 2.9460 - val_accuracy: 0.8079\n",
"Epoch 2/32\n",
"27/27 [==============================] - 3196s 118s/step - loss: 2.8857 - accuracy: 0.8121 - val_loss: 2.9460 - val_accuracy: 0.8079\n",
"Epoch 3/32\n",
"13/27 [=============>................] - ETA: 22:18 - loss: 2.9072 - accuracy: 0.8104"
]
}
],
"outputs": [],
"source": [
"history = model.fit_generator(\n",
" train_data_gen,\n",
@ -664,6 +564,7 @@
"loss = history.history['loss']\n",
"val_loss = history.history['val_loss']\n",
"\n",
"''\n",
"epochs_range = range(epochs)\n",
"\n",
"plt.figure(figsize=(8, 8))\n",
@ -693,7 +594,86 @@
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
"source": [
"print(train_dir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_good_dir)\n",
"print(train_bad_dir)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_image_generator)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(train_data_gen)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(sample_train_images,)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(history)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_generator.reset()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_datagen=ImageDataGenerator(rescale=1./255.)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_generator=test_datagen.flow_from_directory(\n",
" directory=\"data/test/\",\n",
" target_size=(IMG_HEIGHT, IMG_WIDTH),\n",
" class_mode='binary'\n",
")"
]
},
{
"cell_type": "code",
@ -702,22 +682,40 @@
"outputs": [],
"source": [
"# https://keras.io/models/sequential/\n",
"print(\"predict\")\n",
"\n",
"#prediction = model.predict(x=test_it, batch_size=None, verbose=1, steps=None, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False)\n",
"#prediction = model.predict(x=test_it, batch_size=None, verbose=1, steps=None, use_multiprocessing=True)\n",
"\n",
"prediction = model.predict(\n",
"\tx=test_dir,\n",
"\tbatch_size=None,\n",
"\tverbose=1,\n",
"\tsteps=None,\n",
"\tcallbacks=None,\n",
"\tmax_queue_size=10,\n",
"\tworkers=16,\n",
"\tuse_multiprocessing=True)"
"print(\"predict\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pred=model.predict_generator(test_generator,\n",
"steps=4,\n",
"verbose=1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"prediction = model.predict(\n",
" x=test_generator,\n",
" verbose=2\n",
")\n",
"print(\"end predict\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,

View File

@ -24,6 +24,7 @@
# https://archive.org/details/satnogs?sort=-publicdate
# Run:
# wut-audio-archive 1292461 1368693
# wut-audio-archive 1346458 1368693
#
# XXX Should check input is sane...

View File

@ -15,6 +15,8 @@
#
# So to get mostly all of the observations in December, 2019, run:
# wut-water-range 1292461 1470525
# Resume:
# wut-water-range 1355760 1470525
#
# XXX Should check input is sane...
@ -36,7 +38,7 @@ while [ $OBSID -lt $OBSIDMAX ]
--http2 --ipv4 \
--silent \
--output $OBSID.json \
"$APIURL/observations/?id=$OBSID&ground_station=&satellite__norad_cat_id=&transmitter_uuid=&transmitter_mode=&transmitter_type=&vetted_status=&vetted_user=&start=&end=" && sleep `echo $((0 + RANDOM % 6))`
"$APIURL/observations/?id=$OBSID&ground_station=&satellite__norad_cat_id=&transmitter_uuid=&transmitter_mode=&transmitter_type=&vetted_status=&vetted_user=&start=&end=" && sleep `echo $((0 + RANDOM % 3))`
WATERURL=`cat $OBSID.json | jq --compact-output '.[0] | {waterfall}' | cut -f 2- -d : | sed -e 's/}//g' -e 's/http:/https:/g' -e 's/"//g'`
WATERFILE=`basename $WATERURL`
[ ! -f "$WATERFILE" ] && \