V0.4.0 dev

.gitignore

@@ -21,6 +21,7 @@ __pycache__/
 /wandb/
 *test*.json
 dataset_config.*.json
+processed.*.json
 loader.*.json
 *.csv
 *.err
@@ -29,9 +30,11 @@ loader.*.json
 *.npy
 *.out
 tmp.*
+*.swp
 *test*
 *.png
 
+ENVS
 !ENVS.example
 ENVS.*
 

ENVS.example

@@ -61,6 +61,12 @@ DEMO_PIPELINE_VAL_END="2022-2-14"
 DEMO_PIPELINE_TEST_START="2022-2-15"
 DEMO_PIPELINE_TEST_END="2022-2-28"
 
+##
+# Training & Prediction ensemble run seeds
+#
+DEMO_PIPELINE_ENSEMBLE_TRAIN_SEEDS="42,46"
+DEMO_PIPELINE_ENSEMBLE_PREDICT_SEEDS="42,46"
+
 ##
 # The prefix to use for training date ranges
 #
@@ -83,6 +89,9 @@ VAL_END_NAME="${PREFIX}_VAL_END"
 TEST_START_NAME="${PREFIX}_TEST_START"
 TEST_END_NAME="${PREFIX}_TEST_END"
 
+ENSEMBLE_TRAIN_SEEDS_NAME="${PREFIX}_ENSEMBLE_TRAIN_SEEDS"
+ENSEMBLE_PREDICT_SEEDS_NAME="${PREFIX}_ENSEMBLE_PREDICT_SEEDS"
+
 # What are we exporting
 
 export TRAIN_START=${!TRAIN_START_NAME}
@@ -92,3 +101,5 @@ export VAL_END=${!VAL_END_NAME}
 export TEST_START=${!TEST_START_NAME}
 export TEST_END=${!TEST_END_NAME}
 
+export ENSEMBLE_TRAIN_SEEDS=${!ENSEMBLE_TRAIN_SEEDS_NAME}
+export ENSEMBLE_PREDICT_SEEDS=${!ENSEMBLE_PREDICT_SEEDS_NAME}

ensemble/predict.tmpl.yaml

@@ -14,6 +14,7 @@ ensemble:
       - ../../../processed
       - ../../../results
     mem:          224gb
+    cluster:      pvc
 
   pre_process:          []
   post_process:         []
@@ -24,7 +25,6 @@ ensemble:
     - icenet_predict.sh.j2
     email:        [email protected]
     job_file:     icenet_predict.sh
-    cluster:      short
     nodes:        1
     ntasks:       8
     length:       00:30:00
@@ -38,11 +38,21 @@ ensemble:
         - name:   execute
           args: 
               cmd:  /usr/bin/ln -s ../../data
+        - name:   execute
+          args:
+              cmd:  /usr/bin/ln -s ../../processed
+        - name:   execute
+          args:
+              cmd:  /usr/bin/ln -s ../../processed_data
+        - name:   execute
+          args:
+              cmd:  /usr/bin/ln -s ../../ref.osisaf.north.nc
+        - name:   execute
+          args:
+              cmd:  /usr/bin/ln -s ../../ref.osisaf.south.nc
       pre_run:    []
       runs:
-        - seed:   42
-        - seed:   46
-        - seed:   45
+        - seed:   SEEDS
       post_run:     []
       post_batch:
         - name:   execute

ensemble/template/dawn.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+module purge
+module load default-dawn
+module load intelpython-conda

ensemble/template/icenet_predict.sh.j2

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -l
 #SBATCH --output={{ run.dir }}/predict.%j.%N.{{ run.seed }}.out
 #SBATCH --error={{ run.dir }}/predict.%j.%N.{{ run.seed }}.err
 #SBATCH --chdir={{ run.dir }}

ensemble/template/icenet_train.sh.j2

@@ -1,4 +1,5 @@
-#!/bin/bash
+#!/bin/bash -l
+{% if run.cluster != "test" %}
 #SBATCH --output={{ run.dir }}/train.%j.%N.{{ run.seed }}.out
 #SBATCH --error={{ run.dir }}/train.%j.%N.{{ run.seed }}.err
 #SBATCH --chdir={{ run.dir }}
@@ -15,6 +16,7 @@
 #SBATCH --cpus-per-task={{ run.ntasks }}
 #SBATCH --mem={{ run.mem }}
 {% if run.nodelist %}#SBATCH --nodelist={{ run.nodelist }}{% endif %}
+{% endif %}
 
 cd {{ run.dir }}
 
@@ -36,8 +38,24 @@ echo "START `date +%F\ %T`"
 source $PREP_SCRIPT
 conda activate $ICENET_CONDA
 
-# TODO: run.arg_filter_factor comes from ENVS now
-COMMAND="icenet_train -v {{ run.arg_dataset }} {{ run.name }} {{ run.seed }} $TRAIN_STATIC_ARGS -b {{ run.arg_batch }} -e {{ run.arg_epochs }} -m -qs {{ run.arg_queue }} -w {{ run.ntasks }} -s {{ run.arg_strategy }} {% if run.arg_preload %} -p results/networks/{{ run.name }}/{{ run.name }}.network_{{ run.arg_preload }}.{{ run.seed }}.h5 {% endif %}{% if run.arg_filter_factor %} -n {{ run.arg_filter_factor }}{% endif %}"
+PRELOAD=""
+FINAL_WEIGHTS="results/networks/{{ run.name }}/{{ run.name }}.network_{{ run.preload }}.{{ run.seed }}.h5"
+CHECKPOINT_WEIGHTS="`ls results/networks/{{ run.name }}/checkpoint.{{ run.name }}.network_{{ run.preload }}.{{ run.seed }}.*.keras 2>/dev/null`"
+
+# TODO: do we have keras / h5 weight multi-handling in place in library?
+if [ -f $FINAL_WEIGHTS ]; then
+    echo "Preloading from previously trained network $FINAL_WEIGHTS"
+    PRELOAD="-p $FINAL_WEIGHTS"
+elif [ ! -z "$CHECKPOINT_WEIGHTS" ]; then
+    CHECKPOINT_FILE=`echo "$CHECKPOINT_WEIGHTS" | sort | head -n 1`
+    echo "Preloading from checkpoint file $CHECKPOINT_FILE"
+    PRELOAD="-p $CHECKPOINT_FILE"
+fi
+
+COMMAND="icenet_train_tensorflow -v \
+    $TRAIN_STATIC_ARGS \
+    -b {{ run.batch }} -e {{ run.epochs }} -n $FILTER_FACTOR -s {{ run.strategy }} \
+    $PRELOAD {{ run.dataset }} {{ run.name }} {{ run.seed }} "
 
 echo "Running $COMMAND"
 eval $COMMAND

ensemble/train.tmpl.yaml

@@ -1,22 +1,27 @@
 ---
 ensemble:
   vars:
-    arg_batch:          4
-    arg_dataset:        DATASET
-    arg_epochs:         100
-    arg_filter_factor:  1
-    arg_queue:          2
-    arg_strategy:       default
+    batch:          4
+    cluster:        dummy
+    dataset:        DATASET
+    email:          [email protected]
+    epochs:         100
+    filter_factor:  1
+    gpus:           1
+    length:         "1-00:00:00"
+    mem:            128gb
+    nodes:          1
+    ntasks:         2
+    preload:        DATASET
+    strategy:       default
     symlinks:
       - ../../../data
       - ../../../ENVS*
-      - ../../../loader.LOADER.json
+      - ../../../LOADER
       - ../../../dataset_config.DATASET.json
       - ../../../network_datasets
       - ../../../processed
       - ../../../results
-    gpus:         1
-    mem:          128gb
 
   pre_process:
     - name:   execute
@@ -29,13 +34,8 @@ ensemble:
     templatedir:  ../template
     templates:
     - icenet_train.sh.j2
-    email:        [email protected]
     job_file:     icenet_train.sh
-    cluster:      gpu
-    nodes:        1
-    ntasks:       NTASKS
-    length:       4-00:00:00
-    maxruns:      5
+    maxruns:      MAXJOBS
     maxjobs:      MAXJOBS
 
   batches:
@@ -44,16 +44,7 @@ ensemble:
       pre_batch:  []
       pre_run:    []
       runs:
-        - seed:   42
-        - seed:   46
-        - seed:   45
-        - seed:   17
-        - seed:   24
-        - seed:   84
-        - seed:   83
-        - seed:   16
-        - seed:   5
-        - seed:   3
+        - seed:   SEEDS
       post_run:   []
       post_batch:
         - name:   execute

environment.dawn.yml

@@ -0,0 +1,13 @@
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - cartopy
+  - eccodes
+  - ffmpeg
+  - hdf5
+  - netcdf4
+  - openh264
+  - xarray
+  pip:
+    - model-ensembler

environment.yml

@@ -1,15 +1,17 @@
 channels:
   - conda-forge
-  - defaults
 dependencies:
   - cartopy
-  - cudatoolkit=11.2
-  - cudnn=8.1.0
+  - cudatoolkit
+  - cudnn
   - eccodes
   - ffmpeg
   - hdf5
   - ipykernel
   - netcdf4
   - openh264
-  - python=3.8
+  - python=3.9
+  - pip
   - xarray
+  - pip:
+    - model-ensembler