parent
9996882ec1
commit
9d44337a61
|
@ -39,6 +39,7 @@ The following scripts are in the repo:
|
||||||
* `wut-dl-sort-txmode` --- Populate `data/` dir with waterfalls from `download/` using selected encoding.
|
* `wut-dl-sort-txmode` --- Populate `data/` dir with waterfalls from `download/` using selected encoding.
|
||||||
* `wut-files` --- Tells you about what files you have in `downloads/` and `data/`.
|
* `wut-files` --- Tells you about what files you have in `downloads/` and `data/`.
|
||||||
* `wut-ml` --- Main machine learning Python script using Tensorflow and Keras.
|
* `wut-ml` --- Main machine learning Python script using Tensorflow and Keras.
|
||||||
|
* `wut-ml.ipynb` --- Machine learning Python script using Tensorflow and Keras in a Jupyter Notebook.
|
||||||
* `wut-ml-load` --- Machine learning Python script using Tensorflow and Keras, load `data/wut.h5`.
|
* `wut-ml-load` --- Machine learning Python script using Tensorflow and Keras, load `data/wut.h5`.
|
||||||
* `wut-ml-save` --- Machine learning Python script using Tensorflow and Keras, save `data/wut.h5`.
|
* `wut-ml-save` --- Machine learning Python script using Tensorflow and Keras, save `data/wut.h5`.
|
||||||
* `wut-obs` --- Download the JSON for an observation ID.
|
* `wut-obs` --- Download the JSON for an observation ID.
|
||||||
|
|
|
@ -0,0 +1,387 @@
|
||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 1,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"#!/usr/bin/python3\n",
|
||||||
|
"# wut-ml\n",
|
||||||
|
"#\n",
|
||||||
|
"# Vet a SatNOGS image using machine learning (guessing).\n",
|
||||||
|
"# It will vet the image located at test/unvetted/waterfall.png.\n",
|
||||||
|
"#\n",
|
||||||
|
"# Note, there is an issue to fix where it will vet everything\n",
|
||||||
|
"# under the data/test directory, so fix that. For now, just delete\n",
|
||||||
|
"# everything else. :)\n",
|
||||||
|
"#\n",
|
||||||
|
"# Usage:\n",
|
||||||
|
"# wut-ml\n",
|
||||||
|
"# Example:\n",
|
||||||
|
"# wut-ml"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 2,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"import done\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import tensorflow.python.keras\n",
|
||||||
|
"from tensorflow.python.keras import Sequential\n",
|
||||||
|
"from tensorflow.python.keras.layers import Activation, Dropout, Flatten, Dense\n",
|
||||||
|
"from tensorflow.python.keras.preprocessing.image import ImageDataGenerator\n",
|
||||||
|
"from tensorflow.python.keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D\n",
|
||||||
|
"from tensorflow.python.keras import optimizers\n",
|
||||||
|
"from tensorflow.python.keras.preprocessing import image\n",
|
||||||
|
"from tensorflow.python.keras.models import load_model\n",
|
||||||
|
"from tensorflow.python.keras.preprocessing.image import load_img\n",
|
||||||
|
"from tensorflow.python.keras.preprocessing.image import img_to_array\n",
|
||||||
|
"\n",
|
||||||
|
"# XXX\n",
|
||||||
|
"from tensorflow.python.keras.models import Model\n",
|
||||||
|
"from tensorflow.python.keras.layers import Input, concatenate\n",
|
||||||
|
"#from tensorflow.python.keras.optimizers import Adam\n",
|
||||||
|
"\n",
|
||||||
|
"# XXX Plot\n",
|
||||||
|
"from tensorflow.python.keras.utils import plot_model\n",
|
||||||
|
"from tensorflow.python.keras.callbacks import ModelCheckpoint\n",
|
||||||
|
"## for visualizing \n",
|
||||||
|
"import matplotlib.pyplot as plt, numpy as np\n",
|
||||||
|
"from sklearn.decomposition import PCA\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"import done\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 3,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"datagen\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# https://keras.io/preprocessing/image/\n",
|
||||||
|
"# TODO:\n",
|
||||||
|
"# * Pre-process image\n",
|
||||||
|
"print(\"datagen\")\n",
|
||||||
|
"datagen = ImageDataGenerator(\n",
|
||||||
|
"\tfeaturewise_center=False,\n",
|
||||||
|
"\tsamplewise_center=False,\n",
|
||||||
|
"\tfeaturewise_std_normalization=False,\n",
|
||||||
|
"\tsamplewise_std_normalization=False,\n",
|
||||||
|
"\tzca_whitening=False,\n",
|
||||||
|
"\tzca_epsilon=1e-06,\n",
|
||||||
|
"\trescale=1./255,\n",
|
||||||
|
"\tshear_range=0.0,\n",
|
||||||
|
"\tzoom_range=0.0,\n",
|
||||||
|
"\trotation_range=0,\n",
|
||||||
|
"\twidth_shift_range=0.0,\n",
|
||||||
|
"\theight_shift_range=0.0,\n",
|
||||||
|
"\tbrightness_range=None,\n",
|
||||||
|
"\tchannel_shift_range=0.0,\n",
|
||||||
|
"\tfill_mode='nearest',\n",
|
||||||
|
"\tcval=0.0,\n",
|
||||||
|
"\thorizontal_flip=False,\n",
|
||||||
|
"\tvertical_flip=False,\n",
|
||||||
|
"\tpreprocessing_function=None,\n",
|
||||||
|
"\tdata_format='channels_last',\n",
|
||||||
|
"\tvalidation_split=0.0,\n",
|
||||||
|
"\tdtype='float32')"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 4,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"datagen.flow\n",
|
||||||
|
"Found 761 images belonging to 4 classes.\n",
|
||||||
|
"Found 741 images belonging to 3 classes.\n",
|
||||||
|
"Found 1 images belonging to 1 classes.\n",
|
||||||
|
"train_it.next()\n",
|
||||||
|
"Batch shape=(32, 256, 256, 3), min=0.000, max=1.000\n",
|
||||||
|
"Batch shape=(32, 256, 256, 3), min=0.000, max=1.000\n",
|
||||||
|
"Batch shape=(1, 256, 256, 3), min=0.000, max=1.000\n",
|
||||||
|
"input shape\n",
|
||||||
|
"(256, 256, 3)\n",
|
||||||
|
"Height 256 Width 256\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print(\"datagen.flow\")\n",
|
||||||
|
"train_it = datagen.flow_from_directory('data/train/', class_mode='binary')\n",
|
||||||
|
"val_it = datagen.flow_from_directory('data/val/', class_mode='binary')\n",
|
||||||
|
"test_it = datagen.flow_from_directory('data/test/', class_mode='binary')\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"train_it.next()\")\n",
|
||||||
|
"trainX, trainY = train_it.next()\n",
|
||||||
|
"print('Batch shape=%s, min=%.3f, max=%.3f' % (trainX.shape, trainX.min(), trainX.max()))\n",
|
||||||
|
"valX, valY = val_it.next()\n",
|
||||||
|
"print('Batch shape=%s, min=%.3f, max=%.3f' % (valX.shape, valX.min(), valX.max()))\n",
|
||||||
|
"testX, testY = test_it.next()\n",
|
||||||
|
"print('Batch shape=%s, min=%.3f, max=%.3f' % (testX.shape, testX.min(), testX.max()))\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"input shape\")\n",
|
||||||
|
"input_shape=trainX.shape[1:]\n",
|
||||||
|
"print(input_shape)\n",
|
||||||
|
"\n",
|
||||||
|
"#img_width=823\n",
|
||||||
|
"#img_height=1606\n",
|
||||||
|
"img_width=256\n",
|
||||||
|
"img_height=256\n",
|
||||||
|
"print(\"Height\", img_height, \"Width\", img_width)\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 5,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"Sequential\n",
|
||||||
|
"add\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# https://keras.io/models/sequential/\n",
|
||||||
|
"# https://keras.io/getting-started/sequential-model-guide/\n",
|
||||||
|
"print(\"Sequential\")\n",
|
||||||
|
"model = Sequential()\n",
|
||||||
|
"\n",
|
||||||
|
"print(\"add\")\n",
|
||||||
|
"# Other data to consider adding:\n",
|
||||||
|
"# * JSON metadata\n",
|
||||||
|
"# * TLE\n",
|
||||||
|
"# * Audio File (ogg)\n",
|
||||||
|
"# https://www.tensorflow.org/io/api_docs/python/tfio/ffmpeg/AudioDataset\n",
|
||||||
|
"# * Decoded Data (HEX, ASCII, PNG)\n",
|
||||||
|
"# Data from external sources to consider adding:\n",
|
||||||
|
"# * Weather\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 6,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"convolution 2 deeeee\n",
|
||||||
|
"Activation relu\n",
|
||||||
|
"Pooling\n",
|
||||||
|
"Convolution2D\n",
|
||||||
|
"Activation relu\n",
|
||||||
|
"Pooling\n",
|
||||||
|
"Convolution2D\n",
|
||||||
|
"Activation relu\n",
|
||||||
|
"Pooling\n",
|
||||||
|
"Flatten\n",
|
||||||
|
"Dense\n",
|
||||||
|
"Activation relu\n",
|
||||||
|
"Dropout\n",
|
||||||
|
"Dense\n",
|
||||||
|
"Activation softmax\n",
|
||||||
|
"compile\n",
|
||||||
|
"fit\n",
|
||||||
|
"Train for 24 steps, validate for 24 steps\n",
|
||||||
|
"24/24 - 60s - loss: 0.0000e+00 - accuracy: 0.1209 - val_loss: 0.0000e+00 - val_accuracy: 0.0013\n",
|
||||||
|
"fit done\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"print(\"convolution 2 deeeee\")\n",
|
||||||
|
"# https://keras.io/layers/convolutional/\n",
|
||||||
|
"#model.add(Convolution2D(32, 3, 3, input_shape=trainX.shape[1:]))\n",
|
||||||
|
"#model.add(Convolution2D(32, 3, 3, input_shape=(255,255,3)))\n",
|
||||||
|
"model.add(Convolution2D(32, 3, 3, input_shape=(img_width, img_height,3)))\n",
|
||||||
|
"# https://keras.io/activations/\n",
|
||||||
|
"print(\"Activation relu\")\n",
|
||||||
|
"model.add(Activation('relu'))\n",
|
||||||
|
"# https://keras.io/layers/pooling/\n",
|
||||||
|
"print(\"Pooling\")\n",
|
||||||
|
"model.add(MaxPooling2D(pool_size=(2, 2)))\n",
|
||||||
|
"print(\"Convolution2D\")\n",
|
||||||
|
"model.add(Convolution2D(32, 3, 3))\n",
|
||||||
|
"print(\"Activation relu\")\n",
|
||||||
|
"model.add(Activation('relu'))\n",
|
||||||
|
"print(\"Pooling\")\n",
|
||||||
|
"model.add(MaxPooling2D(pool_size=(2, 2)))\n",
|
||||||
|
"print(\"Convolution2D\")\n",
|
||||||
|
"model.add(Convolution2D(64, 3, 3))\n",
|
||||||
|
"print(\"Activation relu\")\n",
|
||||||
|
"model.add(Activation('relu'))\n",
|
||||||
|
"print(\"Pooling\")\n",
|
||||||
|
"model.add(MaxPooling2D(pool_size=(2, 2)))\n",
|
||||||
|
"# https://keras.io/layers/core/\n",
|
||||||
|
"print(\"Flatten\")\n",
|
||||||
|
"model.add(Flatten())\n",
|
||||||
|
"# https://keras.io/layers/core/\n",
|
||||||
|
"print(\"Dense\")\n",
|
||||||
|
"model.add(Dense(64))\n",
|
||||||
|
"print(\"Activation relu\")\n",
|
||||||
|
"model.add(Activation('relu'))\n",
|
||||||
|
"# https://keras.io/layers/core/\n",
|
||||||
|
"print(\"Dropout\")\n",
|
||||||
|
"model.add(Dropout(0.5))\n",
|
||||||
|
"print(\"Dense\")\n",
|
||||||
|
"model.add(Dense(1))\n",
|
||||||
|
"print(\"Activation softmax\")\n",
|
||||||
|
"model.add(Activation('softmax'))\n",
|
||||||
|
"\n",
|
||||||
|
"# https://keras.io/models/sequential/\n",
|
||||||
|
"print(\"compile\")\n",
|
||||||
|
"model.compile(\n",
|
||||||
|
"\tloss='categorical_crossentropy',\n",
|
||||||
|
"\tloss_weights=None,\n",
|
||||||
|
"\tsample_weight_mode=None,\n",
|
||||||
|
"\tweighted_metrics=None,\n",
|
||||||
|
"\ttarget_tensors=None,\t\n",
|
||||||
|
"\toptimizer='rmsprop',\n",
|
||||||
|
"\tmetrics=['accuracy'])\n",
|
||||||
|
"\n",
|
||||||
|
"# https://keras.io/models/sequential/\n",
|
||||||
|
"print(\"fit\")\n",
|
||||||
|
"model.fit(\n",
|
||||||
|
"\tx=train_it,\n",
|
||||||
|
"\ty=None,\n",
|
||||||
|
"\tbatch_size=None,\n",
|
||||||
|
"\tepochs=1,\n",
|
||||||
|
"\tverbose=2,\n",
|
||||||
|
"\tcallbacks=None,\n",
|
||||||
|
"\tvalidation_split=0.0,\n",
|
||||||
|
"\tvalidation_data=val_it,\n",
|
||||||
|
"\tshuffle=True,\n",
|
||||||
|
"\tclass_weight=None,\n",
|
||||||
|
"\tsample_weight=None,\n",
|
||||||
|
"\tinitial_epoch=0,\n",
|
||||||
|
"\tsteps_per_epoch=None,\n",
|
||||||
|
"\tvalidation_steps=None,\n",
|
||||||
|
"\tvalidation_freq=1,\n",
|
||||||
|
"\tmax_queue_size=10,\n",
|
||||||
|
"\tworkers=16,\n",
|
||||||
|
"\tuse_multiprocessing=True)\n",
|
||||||
|
"print(\"fit done\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 15,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# evaluate(x=None, y=None, batch_size=None, verbose=1, sample_weight=None, steps=None, callbacks=None, max_queue_size=10, workers=1, use_multiprocessing=False)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 16,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# TODO:\n",
|
||||||
|
"# * Generate output to visualize training/validating/testing.\n",
|
||||||
|
"\n",
|
||||||
|
"# Plot, fail\n",
|
||||||
|
"#print(\"plot\")\n",
|
||||||
|
"#plot_model(test_it,dpi=72)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": 17,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [
|
||||||
|
{
|
||||||
|
"name": "stdout",
|
||||||
|
"output_type": "stream",
|
||||||
|
"text": [
|
||||||
|
"predict\n",
|
||||||
|
"1/1 - 0s\n",
|
||||||
|
"[[1.]]\n",
|
||||||
|
"Observation: bad\n"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source": [
|
||||||
|
"# https://keras.io/models/sequential/\n",
|
||||||
|
"print(\"predict\")\n",
|
||||||
|
"\n",
|
||||||
|
"prediction = model.predict(\n",
|
||||||
|
"\tx=test_it,\n",
|
||||||
|
"\tbatch_size=None,\n",
|
||||||
|
"\tverbose=2,\n",
|
||||||
|
"\tsteps=None,\n",
|
||||||
|
"\tcallbacks=None,\n",
|
||||||
|
"\tmax_queue_size=10,\n",
|
||||||
|
"\tworkers=16,\n",
|
||||||
|
"\tuse_multiprocessing=True)\n",
|
||||||
|
"\n",
|
||||||
|
"print(prediction)\n",
|
||||||
|
"\n",
|
||||||
|
"if prediction[0][0] == 1:\n",
|
||||||
|
" rating = 'bad'\n",
|
||||||
|
"else:\n",
|
||||||
|
" rating = 'good'\n",
|
||||||
|
"print('Observation: %s' % (rating))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3",
|
||||||
|
"language": "python",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.3"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 4
|
||||||
|
}
|
Loading…
Reference in New Issue