Add project code argument and extra scripts

cleanup.sh

@@ -0,0 +1,3 @@
+rm *.in
+rm *.out
+rm *.sbatch

process_vast_epoch.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+epoch=$1
+in_dir=$SCRATCHDIR/data/vast/epoch_$1
+out_dir=$SCRATCHDIR/stokesv_processing/finished/epoch_$1
+
+echo python run_selavy.py $in_dir --out-dir=$out_dir

run_selavy.py

@@ -51,6 +51,11 @@ def parse_args():
                         default='21',
                         help='Number of tasks per node for each slurm job'
                         )
+    parser.add_argument('--project-code',
+                        type=str,
+                        default='ja3',
+                        help='Number of tasks per node for each slurm job'
+                        ) 
     parser.add_argument('--submit-jobs',
                         action='store_true',
                         help='Submit the jobs'
@@ -85,6 +90,7 @@ def generate_files(filename, jobname, args, invert=False, outdir="."):
                               ntasks=args.ntasks,
                               ntasks_per_node=args.ntasks_per_node,
                               memory=args.mem_request,
+                              project_code=args.project_code
                               )
     return
 

selavybatch.sh

@@ -0,0 +1 @@
+sbatch selavy.nimage.v.VAST_0530-68.SB50408.cont.taylor.0.restored.conv.sbatch

setonix_selavy.py

@@ -158,7 +158,8 @@ def _write_sbatch(job_name,
                   walltime,
                   ntasks,
                   ntasks_per_node,
-                  memory
+                  memory,
+                  project_code
                   ):
     '''write sbatch files'''
     logger = logging.getLogger('makeparset.sbatch')
@@ -181,10 +182,11 @@ def _write_sbatch(job_name,
 #SBATCH --ntasks={ntasks}
 #SBATCH --ntasks-per-node={ntasks_per_node}
 #SBATCH --mem={memory}
+#SBATCH --account={project_code}
 
-module use /software/projects/ja3/modulefiles
-module load singularity/3.8.6
-module load askapsoft/1.10.0.a
+module use /software/projects/askaprt/modulefiles
+module load singularity/4.1.0-mpi
+module load askapsoft/1.15.0
 
 srun selavy -c {parset_name}        
 ''')
@@ -198,6 +200,7 @@ def writebatch(job_name,
                ntasks = '21',
                ntasks_per_node = '21',
                memory = '110G',
+               project_code='ja3'
                ):
     '''write the final .sh file for submission'''
     logger = logging.getLogger('writebatch')
@@ -215,6 +218,7 @@ def writebatch(job_name,
                                     ntasks,
                                     ntasks_per_node,
                                     memory,
+                                    project_code
                                     )
         sbatch_names.append(sbatch_name)
         logger.info(f"Written {sbatch_name}")

stage_data.sh

@@ -1,6 +1,8 @@
 indir=$1
 outdir=$2
 
+mkdir $outdir
+
 mv $indir/selavy-*.xml $outdir/
 mv $indir/noiseMap.image.*.fits $outdir/
 mv $indir/meanMap.image*.fits $outdir/

stage_vast_epoch.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+epoch=$1
+
+finished_dir=$SCRATCHDIR/stokesv_processing/finished/epoch_$1
+staging_dir=$SCRATCHDIR/stokesv_processing/staging/epoch_$1
+
+./stage_data.sh $finished_dir $staging_dir

unstage_data.sh

@@ -0,0 +1,6 @@
+indir=$1
+outdir=$2
+
+mv $indir/selavy-*.xml $outdir/
+mv $indir/noiseMap.image.*.fits $outdir/
+mv $indir/meanMap.image*.fits $outdir/

unstage_vast_epoch.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+epoch=$1
+
+finished_dir=$SCRATCHDIR/stokesv_processing/finished/epoch_$1
+staging_dir=$SCRATCHDIR/stokesv_processing/staging/epoch_$1
+
+mv $staging_dir/* $finished_dir