info on data fiels
ml server
parent
9c09213915
commit
eb3831f97d
|
|
@ -58,9 +58,9 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%matplotlib inline\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import seaborn as sns\n",
|
||||
"#%matplotlib inline\n",
|
||||
"#import matplotlib.pyplot as plt\n",
|
||||
"#import seaborn as sns\n",
|
||||
"#from sklearn.decomposition import PCA\n",
|
||||
"#from ipywidgets import interact, interactive, fixed, interact_manual\n",
|
||||
"#import ipywidgets as widgets\n",
|
||||
|
|
@ -236,20 +236,20 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"sample_train_images, _ = next(train_data_gen)\n",
|
||||
"sample_val_images, _ = next(val_data_gen)\n",
|
||||
"# This function will plot images in the form of a grid with 1 row and 3 columns where images are placed in each column.\n",
|
||||
"def plotImages(images_arr):\n",
|
||||
" fig, axes = plt.subplots(1, 3, figsize=(20,20))\n",
|
||||
" axes = axes.flatten()\n",
|
||||
" for img, ax in zip( images_arr, axes):\n",
|
||||
" ax.imshow(img)\n",
|
||||
" ax.axis('off')\n",
|
||||
" plt.tight_layout()\n",
|
||||
" plt.show()\n",
|
||||
" \n",
|
||||
"plotImages(sample_train_images[0:3])\n",
|
||||
"plotImages(sample_val_images[0:3])"
|
||||
"#sample_train_images, _ = next(train_data_gen)\n",
|
||||
"#sample_val_images, _ = next(val_data_gen)\n",
|
||||
"## This function will plot images in the form of a grid with 1 row and 3 columns where images are placed in each column.\n",
|
||||
"#def plotImages(images_arr):\n",
|
||||
"# fig, axes = plt.subplots(1, 3, figsize=(20,20))\n",
|
||||
"# axes = axes.flatten()\n",
|
||||
"# for img, ax in zip( images_arr, axes):\n",
|
||||
"# ax.imshow(img)\n",
|
||||
"# ax.axis('off')\n",
|
||||
"# plt.tight_layout()\n",
|
||||
"# plt.show()\n",
|
||||
"# \n",
|
||||
"#plotImages(sample_train_images[0:3])\n",
|
||||
"#plotImages(sample_val_images[0:3])"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -258,12 +258,12 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%load_ext tensorboard\n",
|
||||
"!rm -rf ./clusterlogs/\n",
|
||||
"#%load_ext tensorboard\n",
|
||||
"#!rm -rf ./clusterlogs/\n",
|
||||
"#log_dir=\"clusterlogs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
|
||||
"log_dir=\"clusterlogs\"\n",
|
||||
"#log_dir=\"clusterlogs\"\n",
|
||||
"#tensorboard_callback = tensorflow.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)\n",
|
||||
"tensorboard_callback = tensorflow.keras.callbacks.TensorBoard(log_dir=log_dir)\n",
|
||||
"#tensorboard_callback = tensorflow.keras.callbacks.TensorBoard(log_dir=log_dir)\n",
|
||||
"#%tensorboard --logdir clusterlogs --port 6006"
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ while [ $OBSID -lt $OBSIDMAX ]
|
|||
do cd $OBSID
|
||||
VET=`cat $OBSID.json | jq --compact-output '.[0] | {vetted_status}' | cut -f 2 -d ":" | sed -e 's/}//g' -e 's/"//g'`
|
||||
TX=`cat $OBSID.json | jq --compact-output '.[0] | {transmitter_uuid}' | cut -f 2 -d ":" | sed -e 's/}//g' -e 's/"//g'`
|
||||
if [ $OBSTX = $TX ] ; then
|
||||
if [ "$OBSTX" = "$TX" ] ; then
|
||||
RAND_DIR=`echo $((0 + RANDOM % 2))`
|
||||
if [ $RAND_DIR = 1 ] ; then
|
||||
CLASS_DIR="train"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
# wut-files
|
||||
#
|
||||
# Tells you about what files you have in downloads/ and data/
|
||||
#
|
||||
# Usage:
|
||||
# wut-files
|
||||
# Example:
|
||||
# wut-files
|
||||
|
||||
TRAIN=`find data/train -type f | wc -l`
|
||||
echo
|
||||
echo "Training Files: $TRAIN"
|
||||
VAL=`find data/val -type f | wc -l`
|
||||
echo "Validation Files: $VAL"
|
||||
TEST=`find data/test -type f | wc -l`
|
||||
echo "Testing Files: $TEST"
|
||||
echo
|
||||
TRAINGOOD=`find data/train/good/ -name '*.png' | wc -l`
|
||||
echo "Training Good: $TRAINGOOD"
|
||||
TRAINBAD=`find data/train/bad/ -name '*.png' | wc -l`
|
||||
echo "Training Bad: $TRAINBAD"
|
||||
TRAINFAILED=`find data/train/failed/ -name '*.png' | wc -l`
|
||||
echo "Training Failed: $TRAINFAILED"
|
||||
echo
|
||||
VALGOOD=`find data/val/good/ -name '*.png' | wc -l`
|
||||
echo "Validation Good: $VALGOOD"
|
||||
VALBAD=`find data/val/bad/ -name '*.png' | wc -l`
|
||||
echo "Validation Bad: $VALBAD"
|
||||
VALFAILED=`find data/val/failed/ -name '*.png' | wc -l`
|
||||
echo "Validation Failed: $VALFAILED"
|
||||
echo
|
||||
TESTWATER=`find data/test/ -name '*.png' | wc -l`
|
||||
echo "Testing waterfalls: $TESTWATER"
|
||||
echo
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ while [ $OBSID -lt $OBSIDMAX ]
|
|||
--http2 --ipv4 \
|
||||
--silent \
|
||||
--output $OBSID.json \
|
||||
"$APIURL/observations/?id=$OBSID&ground_station=&satellite__norad_cat_id=&transmitter_uuid=&transmitter_mode=&transmitter_type=&vetted_status=&vetted_user=&start=&end=" && sleep `echo $((0 + RANDOM % 2))`
|
||||
"$APIURL/observations/?id=$OBSID&ground_station=&satellite__norad_cat_id=&transmitter_uuid=&transmitter_mode=&transmitter_type=&vetted_status=&vetted_user=&start=&end=" && sleep `echo $((0 + RANDOM % 1))`
|
||||
WATERURL=`cat $OBSID.json | jq --compact-output '.[0] | {waterfall}' | cut -f 2- -d : | sed -e 's/}//g' -e 's/http:/https:/g' -e 's/"//g'`
|
||||
WATERFILE=`basename $WATERURL`
|
||||
[ ! -f "$WATERFILE" ] && \
|
||||
|
|
@ -48,7 +48,7 @@ while [ $OBSID -lt $OBSIDMAX ]
|
|||
--continue-at - \
|
||||
--remote-time \
|
||||
--output $WATERFILE \
|
||||
$WATERURL && sleep `echo $((0 + RANDOM % 3))`
|
||||
$WATERURL && sleep `echo $((0 + RANDOM % 1))`
|
||||
cd ..
|
||||
let OBSID=$OBSID+1
|
||||
done
|
||||
|
|
|
|||
Loading…
Reference in New Issue