From 181ea45f15cd93adb0a5a9dfce6aa699298d6fb9 Mon Sep 17 00:00:00 2001 From: jebba Date: Mon, 24 Jan 2022 18:47:55 -0700 Subject: [PATCH] Adapt wut? for witzit. --- README.md | 20 +- notebooks/witzit-predict.ipynb | 20 +- notebooks/witzit-train.ipynb | 34 +- notebooks/witzit.ipynb | 758 --------------------------------- 4 files changed, 37 insertions(+), 795 deletions(-) delete mode 100644 notebooks/witzit.ipynb diff --git a/README.md b/README.md index e216e97..8af0161 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ pip install --user --upgrade -r requirements.txt Development is most easily done under Jupyter with Tensorboard for training models. These files are in the `notebooks/` directory. -* `witzit.ipynb` --- witzit Jupyter notebook. * `witzit-predict.ipynb` --- witzit Jupyter notebook, prediction application. * `witzit-train.ipynb` --- witzit Jupyter notebook, training application. @@ -55,6 +54,25 @@ a main original archive of 10,000 samples and you want to process just 1,000 of them, they would be copied to the `data/` directory. +Data is also stored here, which can also be deleted/moved by scripts: + +`/srv/witzit/` + + +Each element sample will be stored under here: + +`/srv/witzit/data/element/` + + +Each element model will be stored under here: + +`/srv/witzit/data/models/` + + +Temporary logs during training may be written to the gitignored +`logs/` directory. + + # Usage HOWTO USE. Getting closer... diff --git a/notebooks/witzit-predict.ipynb b/notebooks/witzit-predict.ipynb index ed92c02..bc3bc34 100644 --- a/notebooks/witzit-predict.ipynb +++ b/notebooks/witzit-predict.ipynb @@ -33,24 +33,6 @@ "# https://spacecruft.org/spacecruft/satnogs-wut" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# GPLv3+" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Built using Jupyter, Tensorflow, Keras" - ] - }, { "cell_type": "code", "execution_count": null, @@ -183,7 +165,7 @@ "metadata": {}, "outputs": [], "source": [ - "model = load_model('data/models/witzit-DUV.tf')" + "model = load_model('data/models/witzit-al.tf')" ] }, { diff --git a/notebooks/witzit-train.ipynb b/notebooks/witzit-train.ipynb index 48a28ad..5b94fca 100644 --- a/notebooks/witzit-train.ipynb +++ b/notebooks/witzit-train.ipynb @@ -76,7 +76,7 @@ "metadata": {}, "outputs": [], "source": [ - "ENCODING='GMSK'\n", + "ELEMENT='al'\n", "batch_size = 128\n", "epochs = 4\n", "IMG_WIDTH = 416\n", @@ -89,9 +89,9 @@ "metadata": {}, "outputs": [], "source": [ - "train_dir = os.path.join('/srv/satnogs/data/txmodes', ENCODING )\n", - "train_dir = os.path.join('/srv/satnogs/data/txmodes', ENCODING, 'train')\n", - "val_dir = os.path.join('/srv/satnogs/data/txmodes', ENCODING, 'val')\n", + "train_dir = os.path.join('/srv/witzit/data/element', ELEMENT )\n", + "train_dir = os.path.join('/srv/witzit/data/element', ELEMENT, 'train')\n", + "val_dir = os.path.join('/srv/witzit/data/element', ELEMENT, 'val')\n", "train_good_dir = os.path.join(train_dir, 'good')\n", "train_bad_dir = os.path.join(train_dir, 'bad')\n", "val_good_dir = os.path.join(val_dir, 'good')\n", @@ -112,19 +112,19 @@ "source": [ "print('total training good images:', num_train_good)\n", "print('total training bad images:', num_train_bad)\n", - "#print(\"--\")\n", + "print(\"--\")\n", "print(\"Total training images:\", total_train)\n", - "#print('total validation good images:', num_val_good)\n", - "#print('total validation bad images:', num_val_bad)\n", - "#print(\"--\")\n", - "#print(\"Total validation images:\", total_val)\n", - "#print(\"Reduce training and validation set when testing\")\n", + "print('total validation good images:', num_val_good)\n", + "print('total validation bad images:', num_val_bad)\n", + "print(\"--\")\n", + "print(\"Total validation images:\", total_val)\n", + "print(\"Reduce training and validation set when testing\")\n", "total_train = 100\n", "total_val = 100\n", - "#print(\"Train =\")\n", - "#print(total_train)\n", - "#print(\"Validation =\")\n", - "#print(total_val)" + "print(\"Train =\")\n", + "print(total_train)\n", + "print(\"Validation =\")\n", + "print(total_val)" ] }, { @@ -371,7 +371,7 @@ "metadata": {}, "outputs": [], "source": [ - "model.save('/srv/satnogs/data/models/GMSK/witzit-GMSK-201912.h5')" + "model.save('/srv/witzit/data/models/al/witzit-al.h5')" ] }, { @@ -380,7 +380,7 @@ "metadata": {}, "outputs": [], "source": [ - "model.save('/srv/satnogs/data/models/GMSK/witzit-GMSK-201912.tf')" + "model.save('/srv/witzit/data/models/al/witzit-al.tf')" ] }, { @@ -398,7 +398,7 @@ "metadata": {}, "outputs": [], "source": [ - "plot_model(model, show_shapes=True, show_layer_names=True, expand_nested=True, dpi=72, to_file='/srv/satnogs/data/models/GMSK/plot_model.png')" + "plot_model(model, show_shapes=True, show_layer_names=True, expand_nested=True, dpi=72, to_file='/srv/witzit/data/models/al/plot_model.png')" ] }, { diff --git a/notebooks/witzit.ipynb b/notebooks/witzit.ipynb deleted file mode 100644 index 2f90aef..0000000 --- a/notebooks/witzit.ipynb +++ /dev/null @@ -1,758 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# witzit --- What In The Zap Is That? AI categorization of spectra from LIBS/XRF analyzers\n", - "#\n", - "# Copyright (C) 2019-2022, Jeff Moe\n", - "#\n", - "# This program is free software: you can redistribute it and/or modify\n", - "# it under the terms of the GNU General Public License as published by\n", - "# the Free Software Foundation, either version 3 of the License, or\n", - "# (at your option) any later version.\n", - "#\n", - "# This program is distributed in the hope that it will be useful,\n", - "# but WITHOUT ANY WARRANTY; without even the implied warranty of\n", - "# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n", - "# GNU General Public License for more details.\n", - "#\n", - "# You should have received a copy of the GNU General Public License\n", - "# along with this program. If not, see .\n", - "#\n", - "# https://spacecruft.org/spacecruft/witzit\n", - "#\n", - "# Based on wut?\n", - "# https://spacecruft.org/spacecruft/satnogs-wut" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Start\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow.python.keras" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.python.keras import Sequential\n", - "from tensorflow.python.keras.layers import Activation, Dropout, Flatten, Dense\n", - "from tensorflow.python.keras.preprocessing.image import ImageDataGenerator\n", - "from tensorflow.python.keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D\n", - "from tensorflow.python.keras import optimizers\n", - "from tensorflow.python.keras.preprocessing import image\n", - "from tensorflow.python.keras.models import load_model\n", - "from tensorflow.python.keras.preprocessing.image import load_img\n", - "from tensorflow.python.keras.preprocessing.image import img_to_array" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from tensorflow.python.keras.models import Model\n", - "from tensorflow.python.keras.layers import Input, concatenate" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Visualization\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from sklearn.decomposition import PCA" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Seaborn pip dependency\n", - "import seaborn as sns" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Interact\n", - "# https://ipywidgets.readthedocs.io/en/stable/examples/Using%20Interact.html\n", - "from __future__ import print_function\n", - "from ipywidgets import interact, interactive, fixed, interact_manual\n", - "import ipywidgets as widgets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Display Images\n", - "from IPython.display import display, Image" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Python import done\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_dir = os.path.join('data/', 'train')\n", - "val_dir = os.path.join('data/', 'val')\n", - "test_dir = os.path.join('data/', 'test')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_good_dir = os.path.join(train_dir, 'good')\n", - "train_bad_dir = os.path.join(train_dir, 'bad')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "val_good_dir = os.path.join(val_dir, 'good')\n", - "val_bad_dir = os.path.join(val_dir, 'bad')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "num_train_good = len(os.listdir(train_good_dir))\n", - "num_train_bad = len(os.listdir(train_bad_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "num_val_good = len(os.listdir(val_good_dir))\n", - "num_val_bad = len(os.listdir(val_bad_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "num_test = len(os.listdir(test_dir))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "total_train = num_train_good + num_train_bad\n", - "total_val = num_val_good + num_val_bad" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print('total training good images:', num_train_good)\n", - "print('total training bad images:', num_train_bad)\n", - "print(\"--\")\n", - "print(\"Total training images:\", total_train)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print('total validation good images:', num_val_good)\n", - "print('total validation bad images:', num_val_bad)\n", - "print(\"--\")\n", - "print(\"Total validation images:\", total_val)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"Reduce training and validation set when testing\")\n", - "#total_train = 500\n", - "#total_val = 500\n", - "print(\"Train =\")\n", - "print(total_train)\n", - "print(\"Validation =\")\n", - "print(total_val)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Good results\n", - "#batch_size = 128\n", - "#epochs = 6\n", - "#\n", - "# Testing, faster more inaccurate results\n", - "batch_size = 96\n", - "epochs = 3" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Half size\n", - "IMG_HEIGHT = 416\n", - "IMG_WIDTH= 804\n", - "# Full size, machine barfs probably needs more RAM\n", - "#IMG_HEIGHT = 832\n", - "#IMG_WIDTH = 1606" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_image_generator = ImageDataGenerator(\n", - " rescale=1./255\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "val_image_generator = ImageDataGenerator(\n", - " rescale=1./255\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_image_generator = ImageDataGenerator(\n", - " rescale=1./255\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_data_gen = train_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=train_dir,\n", - " shuffle=True,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "val_data_gen = val_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=val_dir,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(test_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_data_gen = test_image_generator.flow_from_directory(batch_size=batch_size,\n", - " directory=test_dir,\n", - " target_size=(IMG_HEIGHT, IMG_WIDTH),\n", - " class_mode='binary')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sample_train_images, _ = next(train_data_gen)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sample_val_images, _ = next(val_data_gen)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sample_test_images, _ = next(test_data_gen)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This function will plot images in the form of a grid with 1 row and 3 columns where images are placed in each column.\n", - "def plotImages(images_arr):\n", - " fig, axes = plt.subplots(1, 3, figsize=(20,20))\n", - " axes = axes.flatten()\n", - " for img, ax in zip( images_arr, axes):\n", - " ax.imshow(img)\n", - " ax.axis('off')\n", - " plt.tight_layout()\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plotImages(sample_train_images[0:3])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plotImages(sample_val_images[0:3])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plotImages(sample_test_images[0:3])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model = Sequential([\n", - " Conv2D(16, 3, padding='same', activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),\n", - " MaxPooling2D(),\n", - " Conv2D(32, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Conv2D(64, 3, padding='same', activation='relu'),\n", - " MaxPooling2D(),\n", - " Flatten(),\n", - " Dense(512, activation='relu'),\n", - " Dense(1, activation='sigmoid')\n", - "])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.compile(optimizer='adam',\n", - " loss='binary_crossentropy',\n", - " metrics=['accuracy'])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "Image.LOAD_TRUNCATED_IMAGES = True" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "history = model.fit_generator(\n", - " train_data_gen,\n", - " steps_per_epoch=total_train // batch_size,\n", - " epochs=epochs,\n", - " validation_data=val_data_gen,\n", - " validation_steps=total_val // batch_size,\n", - " verbose=1\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "acc = history.history['accuracy']\n", - "val_acc = history.history['val_accuracy']\n", - "\n", - "loss = history.history['loss']\n", - "val_loss = history.history['val_loss']\n", - "\n", - "epochs_range = range(epochs)\n", - "\n", - "plt.figure(figsize=(8, 8))\n", - "plt.subplot(1, 2, 1)\n", - "plt.plot(epochs_range, acc, label='Training Accuracy')\n", - "plt.plot(epochs_range, val_acc, label='Validation Accuracy')\n", - "plt.legend(loc='lower right')\n", - "plt.title('Training and Validation Accuracy')\n", - "\n", - "plt.subplot(1, 2, 2)\n", - "plt.plot(epochs_range, loss, label='Training Loss')\n", - "plt.plot(epochs_range, val_loss, label='Validation Loss')\n", - "plt.legend(loc='upper right')\n", - "plt.title('Training and Validation Loss')\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(\"TRAINING info\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(train_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(train_good_dir)\n", - "print(train_bad_dir)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(train_image_generator)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(train_data_gen)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#print(sample_train_images)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(history)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "model.save('data/witzit-CW.h5')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# https://keras.io/models/sequential/\n", - "print(\"predict\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pred=model.predict_generator(test_data_gen,\n", - "steps=1,\n", - "verbose=1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prediction = model.predict(\n", - " x=test_data_gen,\n", - " verbose=2\n", - ")\n", - "print(\"end predict\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predictions=[]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Show prediction score\n", - "print(prediction)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "prediction_bool = (prediction >0.8)\n", - "print(prediction_bool)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predictions = prediction_bool.astype(int)\n", - "print(predictions)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Make final prediction\n", - "# XXX, display name, display all of them with mini waterfall, etc.\n", - "if prediction_bool[0] == False:\n", - " rating = 'bad'\n", - "else:\n", - " rating = 'good'\n", - "print('Observation: %s' % (rating))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if prediction_bool[1] == False:\n", - " rating = 'bad'\n", - "else:\n", - " rating = 'good'\n", - "print('Observation: %s' % (rating))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if prediction_bool[2] == False:\n", - " rating = 'bad'\n", - "else:\n", - " rating = 'good'\n", - "print('Observation: %s' % (rating))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# The End" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.2" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -}