From d102e6dacf912d4ea001c64dbe8a7da5d19f8743 Mon Sep 17 00:00:00 2001 From: ml server Date: Sun, 26 Jan 2020 17:27:12 -0700 Subject: [PATCH] update paths to /srv/satnogs --- wut | 2 ++ wut-audio-archive | 2 +- wut-dl-sort | 2 ++ wut-dl-sort-tx | 2 ++ wut-dl-sort-txmode | 2 ++ wut-files | 2 ++ wut-files-data | 2 ++ wut-ml | 6 +++--- wut-ml-auto | 6 +++--- wut-ml-load | 2 +- wut-ml-save | 6 +++--- wut-obs | 2 +- wut-review-staging | 2 +- wut-rm-random | 2 +- wut-water | 2 +- wut-water-range | 2 +- wut-worker-mas.py | 10 +++++----- 17 files changed, 33 insertions(+), 21 deletions(-) diff --git a/wut b/wut index dcd5e92..16240a6 100755 --- a/wut +++ b/wut @@ -9,6 +9,8 @@ # Example: # wut 1456893 +cd /srv/satnogs + OBSID="$1" rm -rf data/test diff --git a/wut-audio-archive b/wut-audio-archive index c6bb49f..e046fa9 100755 --- a/wut-audio-archive +++ b/wut-audio-archive @@ -29,7 +29,7 @@ # XXX Should check input is sane... APIURL="https://network.satnogs.org/api" -DOWNDIR="download" +DOWNDIR="/srv/satnogs/download" OBSIDMIN="$1" OBSIDMAX="$2" OBSID=$OBSIDMIN diff --git a/wut-dl-sort b/wut-dl-sort index f0de132..fa36c3e 100755 --- a/wut-dl-sort +++ b/wut-dl-sort @@ -22,6 +22,8 @@ OBSIDMIN="$1" OBSIDMAX="$2" OBSID=$OBSIDMIN +cd /srv/satnogs + # Enable the following if you want to download waterfalls in this range: #echo "Downloading Waterfalls" #./wut-water-range $OBSIDMIN $OBSIDMAX diff --git a/wut-dl-sort-tx b/wut-dl-sort-tx index ca8ea26..a5c02a7 100755 --- a/wut-dl-sort-tx +++ b/wut-dl-sort-tx @@ -20,6 +20,8 @@ # # Possible vetted_status: bad, failed, good, null, unknown. +cd /srv/satnogs + OBSTX="$1" OBSIDMIN="$2" OBSIDMAX="$3" diff --git a/wut-dl-sort-txmode b/wut-dl-sort-txmode index 9385d1e..a58dcf5 100755 --- a/wut-dl-sort-txmode +++ b/wut-dl-sort-txmode @@ -23,6 +23,8 @@ # # Possible vetted_status: bad, failed, good, null, unknown. +cd /srv/satnogs + OBSENC="$1" OBSIDMIN="$2" OBSIDMAX="$3" diff --git a/wut-files b/wut-files index 3cbf69e..199d054 100755 --- a/wut-files +++ b/wut-files @@ -8,6 +8,8 @@ # Example: # wut-files +cd /srv/satnogs + echo DF=`df -h download/` echo "$DF" diff --git a/wut-files-data b/wut-files-data index eefdf10..1c23e48 100755 --- a/wut-files-data +++ b/wut-files-data @@ -8,6 +8,8 @@ # Example: # wut-files +cd /srv/satnogs + TRAIN=`find data/train -type f | wc -l` echo echo "Training Files: $TRAIN" diff --git a/wut-ml b/wut-ml index dc5d24e..15481e2 100755 --- a/wut-ml +++ b/wut-ml @@ -68,9 +68,9 @@ datagen = ImageDataGenerator( dtype='float32') print("datagen.flow") -train_it = datagen.flow_from_directory('data/train/', class_mode='binary') -val_it = datagen.flow_from_directory('data/val/', class_mode='binary') -test_it = datagen.flow_from_directory('data/test/', class_mode='binary') +train_it = datagen.flow_from_directory('/srv/satnogs/data/train/', class_mode='binary') +val_it = datagen.flow_from_directory('/srv/satnogs/data/val/', class_mode='binary') +test_it = datagen.flow_from_directory('/srv/satnogs/data/test/', class_mode='binary') print("train_it.next()") trainX, trainY = train_it.next() diff --git a/wut-ml-auto b/wut-ml-auto index a11d5da..0c58b87 100755 --- a/wut-ml-auto +++ b/wut-ml-auto @@ -68,9 +68,9 @@ datagen = ImageDataGenerator( dtype='float32') print("datagen.flow") -train_it = datagen.flow_from_directory('data/train/', class_mode='binary') -val_it = datagen.flow_from_directory('data/val/', class_mode='binary') -test_it = datagen.flow_from_directory('data/test/', class_mode='binary') +train_it = datagen.flow_from_directory('/srv/satnogs/data/train/', class_mode='binary') +val_it = datagen.flow_from_directory('/srv/satnogs/data/val/', class_mode='binary') +test_it = datagen.flow_from_directory('/srv/satnogs/data/test/', class_mode='binary') diff --git a/wut-ml-load b/wut-ml-load index 6bed01c..0e5e7a9 100755 --- a/wut-ml-load +++ b/wut-ml-load @@ -30,7 +30,7 @@ from tensorflow.python.keras.models import load_model from tensorflow.python.keras.preprocessing.image import load_img from tensorflow.python.keras.preprocessing.image import img_to_array -model = load_model('data/wut.h5') +model = load_model('/srv/satnogs/data/wut.h5') img_width=256 img_height=256 model = Sequential() diff --git a/wut-ml-save b/wut-ml-save index 88e8eaa..eedc1ce 100755 --- a/wut-ml-save +++ b/wut-ml-save @@ -30,9 +30,9 @@ from tensorflow.python.keras.preprocessing.image import load_img from tensorflow.python.keras.preprocessing.image import img_to_array datagen = ImageDataGenerator() -train_it = datagen.flow_from_directory('data/train/', class_mode='binary') -val_it = datagen.flow_from_directory('data/val/', class_mode='binary') -test_it = datagen.flow_from_directory('data/test/', class_mode='binary') +train_it = datagen.flow_from_directory('/srv/satnogs/data/train/', class_mode='binary') +val_it = datagen.flow_from_directory('/srv/satnogs/data/val/', class_mode='binary') +test_it = datagen.flow_from_directory('/srv/satnogs/data/test/', class_mode='binary') batchX, batchy = train_it.next() print('Batch shape=%s, min=%.3f, max=%.3f' % (batchX.shape, batchX.min(), batchX.max())) img_width=256 diff --git a/wut-obs b/wut-obs index a33e82b..40e6cf1 100755 --- a/wut-obs +++ b/wut-obs @@ -7,7 +7,7 @@ # Download Observation: JSON. Not waterfall, audio, or data files. APIURL="https://network.satnogs.org/api" -DOWNDIR="download" +DOWNDIR="/srv/satnogs/download" cd $DOWNDIR || exit diff --git a/wut-review-staging b/wut-review-staging index d430b7b..d0b5fd8 100755 --- a/wut-review-staging +++ b/wut-review-staging @@ -1,7 +1,7 @@ #!/bin/bash # wut-review-staging # Go through all the images in data/staging and review them. -cd data/staging || exit +cd /srv/satnogs/data/staging || exit for i in *.png do echo $i rm ../test/unvetted/*.png diff --git a/wut-rm-random b/wut-rm-random index 6553874..de4e3bd 100755 --- a/wut-rm-random +++ b/wut-rm-random @@ -22,7 +22,7 @@ KEEP=100 # this is so bad no one should ever run it again #exit 0 # XXX Delete data in this directory! XXX -cd data/test/unvetted/ || exit +cd /srv/satnogs/data/test/unvetted/ || exit TOTALFILES=`ls waterfall_*.png | wc -l` for wf in waterfall_*.png diff --git a/wut-water b/wut-water index 52f9b68..3d7dcd1 100755 --- a/wut-water +++ b/wut-water @@ -7,7 +7,7 @@ # Download Observation: JSON and waterfall. Not audio or data files. APIURL="https://network.satnogs.org/api" -DOWNDIR="download" +DOWNDIR="/srv/satnogs/download" cd $DOWNDIR || exit diff --git a/wut-water-range b/wut-water-range index a81ae3e..dc22b1a 100755 --- a/wut-water-range +++ b/wut-water-range @@ -21,7 +21,7 @@ # XXX Should check input is sane... APIURL="https://network.satnogs.org/api" -DOWNDIR="download" +DOWNDIR="/srv/satnogs/download" OBSIDMIN="$1" OBSIDMAX="$2" OBSID=$OBSIDMIN diff --git a/wut-worker-mas.py b/wut-worker-mas.py index d0b90a2..798f18d 100644 --- a/wut-worker-mas.py +++ b/wut-worker-mas.py @@ -52,8 +52,8 @@ NUM_WORKERS = 6 GLOBAL_BATCH_SIZE = 64 * NUM_WORKERS # XXX -POSITIVE_DIRECTORY = '/home/jebba/devel/spacecruft/satnogs-wut/data/pos' -pos_dir = '/home/jebba/devel/spacecruft/satnogs-wut/data/posdir' +POSITIVE_DIRECTORY = '/srv/satnogs/data/pos' +pos_dir = '/srv/satnogs/data/posdir' strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy( tf.distribute.experimental.CollectiveCommunication.RING) @@ -92,7 +92,7 @@ def process_image(image_bytes, label): AUTOTUNE = tf.data.experimental.AUTOTUNE NUM_TOTAL_IMAGES=100 -data_root = "/home/jebba/devel/spacecruft/satnogs-wut/data" +data_root = "/srv/satnogs/data" profile_dir = os.path.join(data_root, "profiles") dataset = tf.data.Dataset.list_files(data_root) dataset = dataset.shuffle(NUM_TOTAL_IMAGES) @@ -174,8 +174,8 @@ def handle_batching(): yield concat(batch) batch.reset() -train_dir = os.path.join('data/', 'train') -val_dir = os.path.join('data/', 'val') +train_dir = os.path.join('/srv/satnogs/data/', 'train') +val_dir = os.path.join('/srv/satnogs/data/', 'val') train_good_dir = os.path.join(train_dir, 'good') train_bad_dir = os.path.join(train_dir, 'bad') val_good_dir = os.path.join(val_dir, 'good')