ENH Add ability to limit the number of simultaneously active jobs

bin/submitjob

@@ -14,6 +14,7 @@ RESERVE=""
 NODES="1"
 MEM="4"  # 4 GB by default
 MEMALL="$MEM"
+ACTIVE="100"
 QUEUE=""
 TARGET=""
 FEATURES=""
@@ -63,6 +64,7 @@ usage() {
     echo >&2 "  * -N --nodes     = how many nodes to spread across (LSF/SLURM). Defaults to ${NODES}"
     echo >&2 "    -m --mem       = how much RAM per job (in GB). Defaults to ${MEM}G/job"
     echo >&2 "                     for less than 1G/job use fractionals such as 0.1 (100M/job)"
+    echo >&2 "    -a --active    = limit number of simultaneously active jobs. Defaults to ${ACTIVE}"
     echo >&2 "    -q --queue     = which queue to use. Uses cluster's default if unspecified"
     echo >&2 "  * -t --target    = restricts to running on the given host(s) [comma separated] (SGE)"
     echo >&2 "  * -F --features  = request specific resources to constrain your jobs (SLURM)"
@@ -97,7 +99,7 @@ usage() {
     echo >&2 ""
     echo >&2 "  In addition to standard SGE/LSF/SLURM environment variables, the following"
     echo >&2 "  can be used in the jobs file referencing the values used."
-    echo >&2 "    \${Q_NAME} \${Q_CORES} \${Q_MEM} \${Q_MEMALL} \${Q_QUEUE} \${Q_LOGFILE} \${Q_WAITFOR}"
+    echo >&2 "    \${Q_NAME} \${Q_CORES} \${Q_MEM} \${Q_MEMALL} \${Q_ACTIVE} \${Q_QUEUE} \${Q_LOGFILE} \${Q_WAITFOR}"
     echo >&2 ""
     echo >&2 ""
 }
@@ -189,6 +191,7 @@ cat << EOF | ${LOCALCMD}
 #$ -j yes
 #$ -N ${NAME}
 #$ -S /bin/bash
+#$ -tc ${ACTIVE}
 ${BANG_BEGIN}
 ${BANG_QUEUE}
 ${BANG_EMAIL}
@@ -200,6 +203,7 @@ export Q_LOGFILE="${LOGFILE}"
 export Q_CORES="${CORES}"
 export Q_MEM="${MEM}"
 export Q_MEMALL="${MEMALL}"
+export Q_ACTIVE="${ACTIVE}"
 export Q_QUEUE="${QUEUE}"
 export Q_WAITFOR="${WAITFOR}"
 
@@ -225,7 +229,7 @@ slurm_submit() {
 
 cat << EOF | ${LOCALCMD}
 #!/usr/bin/env bash
-#SBATCH --array=${JOB_COUNT}
+#SBATCH --array=${JOB_COUNT}%${ACTIVE}
 #SBATCH --cpus-per-task=${CORES}
 #SBATCH -N ${NODES}
 #SBATCH --mem-per-cpu=${MEM}M
@@ -243,6 +247,7 @@ export Q_LOGFILE="${LOGFILE}"
 export Q_CORES="${CORES}"
 export Q_MEM="${MEM}"
 export Q_MEMALL="${MEMALL}"
+export Q_ACTIVE="${ACTIVE}"
 export Q_QUEUE="${QUEUE}"
 export Q_WAITFOR="${WAITFOR}"
 
@@ -274,7 +279,7 @@ cat << EOF | ${LOCALCMD}
 #BSUB -n ${CORES}
 #BSUB -M ${MEM}
 #BSUB -o ${LOGFILE}
-#BSUB -J ${NAME}[$JOB_COUNT]
+#BSUB -J ${NAME}[${JOB_COUNT}]%${ACTIVE}
 #BSUB -R "select[(mem>=${MEM})] rusage[mem=${MEM}] span[hosts=${NODES}]"
 ${BANG_BEGIN}
 ${BANG_QUEUE}
@@ -286,6 +291,7 @@ export Q_LOGFILE="${LOGFILE}"
 export Q_CORES="${CORES}"
 export Q_MEM="${MEM}"
 export Q_MEMALL="${MEMALL}"
+export Q_ACTIVE="${ACTIVE}"
 export Q_QUEUE="${QUEUE}"
 export Q_WAITFOR="${WAITFOR}"
 
@@ -306,7 +312,7 @@ EOF
 
 }
 
-ARG_PARSE="getopt -o s:n:c:N:m:q:t:F:l:e:E:w:b:d:fh -l system:,name:,cores:,nodes:,mem:,queue:,target:,features:,logfile:,email:,emailwhen:,waitfor:,begin:,debug:,fatal,help -n $0 --"
+ARG_PARSE="getopt -o s:n:c:N:m:q:t:F:l:e:E:w:b:d:fh -l system:,name:,cores:,nodes:,mem:,active:,queue:,target:,features:,logfile:,email:,emailwhen:,waitfor:,begin:,debug:,fatal,help -n $0 --"
 
 # We process arguments twice to handle any argument parsing error:
 ARG_ERROR=$($ARG_PARSE "$@" 2>&1 1>/dev/null)
@@ -353,6 +359,11 @@ while true; do
             MEM="$1"
             shift
             ;;
+        -a|--active)
+            shift
+            ACTIVE="$1"
+            shift
+            ;;
         -q|--queue)
             shift
             QUEUE="$1"