DefaultSerialQueue "msg-sgehead1 cpu-serial"
DefaultClusterQueue "msg-sgehead1 cpu-cluster"
DefaultThreadedQueue "msg-sgehead1 cpu-threaded"
DefaultGPUQueue "msg-sgehead1 gpu-threaded"

# If running directly on gpu, make the "gpu1 8800 GT" queue the default
# destination for GPU jobs.
IFHOST { "gpu1.ucsf.edu" }
DefaultGPUQueue "gpu1 8800 GT"
ENDIF

# Collect all the MSG multiprocessor hosts.

IFHOST { "lbox1.ucsf.edu" "lbox2.ucsf.edu" "venus.ucsf.edu" }
HOST
ProcessControl "simple"
DefaultProcessorCount 8
StatusCommand "w"
ENDIF

IFHOST { "blue.ucsf.edu" "ganymede.ucsf.edu" "neptune.ucsf.edu" "pluto.ucsf.edu" }
HOST
ProcessControl "simple"
DefaultProcessorCount 2
StatusCommand "w"
ENDIF

# Define a generic host.
HOST
ProcessControl "simple"
DefaultProcessorCount 1
StatusCommand "w"

# Collect all the MSG compute queues.

QUEUE
Name "msg-sgehead1 cpu-serial"
Description "Submits a serial job to one node of agard.q which has 10 16-core 2.6 GHz Xeon E5-2670 systems each with 132 GB of RAM."
AcceptsSerial true
AcceptsCluster false
AcceptsThreaded false
AcceptsGPU false
# SGE will set NSLOTS so don't pass a processor count to the applications.
SetProcessorCountForApp false
ProcessControl "simple"
DefaultProcessorCount 1
DefaultMPIRunCommand "/software/openmpi-1.6/bin/mpiexec"
SubmissionCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qsub -pe cpu 1 -l qname=agard.q -l mem_free=8G -S /bin/sh"
StatusCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qstat"
# Skip the unneeded call to ssh on hosts that are submit hosts.
IFHOST { "sgehead.msg.ucsf.edu" "node-9-01.c.msg.ucsf.edu" "node-9-02.c.msg.ucsf.edu" "node-9-03.c.msg.ucsf.edu" "node-9-04.c.msg.ucsf.edu" "node-9-05.c.msg.ucsf.edu" "node-9-06.c.msg.ucsf.edu" "node-9-07.c.msg.ucsf.edu" "node-9-08.c.msg.ucsf.edu" "node-9-09.c.msg.ucsf.edu" "node-9-10.c.msg.ucsf.edu" "nvgpu-1-1.c.msg.ucsf.edu" "nvgpu-1-2.c.msg.ucsf.edu" "nvgpu-1-3.c.msg.ucsf.edu" "nvgpu-1-4.c.msg.ucsf.edu" "nvgpu-1-5.c.msg.ucsf.edu" "nvgpu-1-6.c.msg.ucsf.edu" "nvgpu-1-7.c.msg.ucsf.edu" "nvgpu-1-8.c.msg.ucsf.edu" "nvgpu-2-1.c.msg.ucsf.edu" "nvgpu-2-2.c.msg.ucsf.edu" "nvgpu-2-3.c.msg.ucsf.edu" "nvgpu-2-4.c.msg.ucsf.edu" "nvgpu-2-5.c.msg.ucsf.edu" "nvgpu-2-6.c.msg.ucsf.edu" "nvgpu-2-7.c.msg.ucsf.edu" "nvgpu-2-8.c.msg.ucsf.edu" }
SubmissionCommand "/opt/sge/bin/lx24-amd64/qsub -pe cpu 1 -l qname=agard.q -l mem_free=8G -S /bin/sh"
StatusCommand "/opt/sge/bin/lx24-amd64/qstat"
ENDIF

QUEUE
Name "msg-sgehead1 cpu-threaded"
Description "Submits a parallel job to one node of agard.q which has 10 16-core 2.6 GHz Xeon E5-2670 systems each with 132 GB of RAM."
AcceptsSerial false
AcceptsCluster true
AcceptsThreaded true
AcceptsGPU false
# SGE will set NSLOTS so don't pass a processor count to the applications.
SetProcessorCountForApp false
ProcessControl "simple"
DefaultProcessorCount 8
DefaultMPIRunCommand "/software/openmpi-1.6/bin/mpiexec"
SubmissionCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qsub -pe cpu 1-%p -l qname=agard.q -l mem_free=50G -S /bin/sh"
StatusCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qstat"
# Skip the unneeded call to ssh on hosts that are submit hosts.
IFHOST { "sgehead.msg.ucsf.edu" "node-9-01.c.msg.ucsf.edu" "node-9-02.c.msg.ucsf.edu" "node-9-03.c.msg.ucsf.edu" "node-9-04.c.msg.ucsf.edu" "node-9-05.c.msg.ucsf.edu" "node-9-06.c.msg.ucsf.edu" "node-9-07.c.msg.ucsf.edu" "node-9-08.c.msg.ucsf.edu" "node-9-09.c.msg.ucsf.edu" "node-9-10.c.msg.ucsf.edu" "nvgpu-1-1.c.msg.ucsf.edu" "nvgpu-1-2.c.msg.ucsf.edu" "nvgpu-1-3.c.msg.ucsf.edu" "nvgpu-1-4.c.msg.ucsf.edu" "nvgpu-1-5.c.msg.ucsf.edu" "nvgpu-1-6.c.msg.ucsf.edu" "nvgpu-1-7.c.msg.ucsf.edu" "nvgpu-1-8.c.msg.ucsf.edu" "nvgpu-2-1.c.msg.ucsf.edu" "nvgpu-2-2.c.msg.ucsf.edu" "nvgpu-2-3.c.msg.ucsf.edu" "nvgpu-2-4.c.msg.ucsf.edu" "nvgpu-2-5.c.msg.ucsf.edu" "nvgpu-2-6.c.msg.ucsf.edu" "nvgpu-2-7.c.msg.ucsf.edu" "nvgpu-2-8.c.msg.ucsf.edu" }
SubmissionCommand "/opt/sge/bin/lx24-amd64/qsub -pe cpu 1-%p -l qname=agard.q -l mem_free=50G -S /bin/sh"
StatusCommand "/opt/sge/bin/lx24-amd64/qstat"
ENDIF

QUEUE
Name "msg-sgehead1 cpu-cluster"
Description "Submits a parallel job to one or more nodes of agard.q which has 10 16-core 2.6 GHz Xeon E5-2670 systems each with 132 GB of RAM."
AcceptsSerial false
AcceptsCluster true
AcceptsThreaded false
AcceptsGPU false
# SGE will set NSLOTS so don't pass a processor count to the applications.
SetProcessorCountForApp false
ProcessControl "openmpi"
DefaultProcessorCount 20
DefaultMPIRunCommand "/software/openmpi-1.6/bin/mpiexec"
SubmissionCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qsub -pe openmpi 1-%p -l qname=agard.q -l mem_free=80G -S /bin/sh"
StatusCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qstat"
# Skip the unneeded call to ssh on hosts that are submit hosts.
IFHOST { "sgehead.msg.ucsf.edu" "node-9-01.c.msg.ucsf.edu" "node-9-02.c.msg.ucsf.edu" "node-9-03.c.msg.ucsf.edu" "node-9-04.c.msg.ucsf.edu" "node-9-05.c.msg.ucsf.edu" "node-9-06.c.msg.ucsf.edu" "node-9-07.c.msg.ucsf.edu" "node-9-08.c.msg.ucsf.edu" "node-9-09.c.msg.ucsf.edu" "node-9-10.c.msg.ucsf.edu" "nvgpu-1-1.c.msg.ucsf.edu" "nvgpu-1-2.c.msg.ucsf.edu" "nvgpu-1-3.c.msg.ucsf.edu" "nvgpu-1-4.c.msg.ucsf.edu" "nvgpu-1-5.c.msg.ucsf.edu" "nvgpu-1-6.c.msg.ucsf.edu" "nvgpu-1-7.c.msg.ucsf.edu" "nvgpu-1-8.c.msg.ucsf.edu" "nvgpu-2-1.c.msg.ucsf.edu" "nvgpu-2-2.c.msg.ucsf.edu" "nvgpu-2-3.c.msg.ucsf.edu" "nvgpu-2-4.c.msg.ucsf.edu" "nvgpu-2-5.c.msg.ucsf.edu" "nvgpu-2-6.c.msg.ucsf.edu" "nvgpu-2-7.c.msg.ucsf.edu" "nvgpu-2-8.c.msg.ucsf.edu" }
SubmissionCommand "/opt/sge/bin/lx24-amd64/qsub -pe openmpi 1-%p -l qname=agard.q -l mem_free=80G -S /bin/sh"
StatusCommand "/opt/sge/bin/lx24-amd64/qstat"
ENDIF

QUEUE
Name "msg-sgehead1 gpu-threaded"
Description "Submits a GPU job to one node of nvgpu.q which has 16 12-core 2.67 GHz Xeon X5650 systems each with 46 GB of RAM and 8 GTX 580."
AcceptsSerial false
AcceptsCluster false
AcceptsThreaded false
AcceptsGPU true
# SGE will set NSLOTS so don't pass a processor count to the applications.
SetProcessorCountForApp false
ProcessControl "simple"
# The GPU reconstruction software will use all the GPUs (unless some are marked
# as busy) so allocate all of the CPUs so that the queueing system thinks the
# system is full.
DefaultProcessorCount 12
DefaultMPIRunCommand "/software/openmpi-1.6/bin/mpiexec"
SubmissionCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qsub -pe cpu %p -l qname=nvgpu.q -l mem_free=40G -S /bin/sh"
StatusCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qstat"
# Skip the unneeded call to ssh on hosts that are submit hosts.
IFHOST { "sgehead.msg.ucsf.edu" "node-9-01.c.msg.ucsf.edu" "node-9-02.c.msg.ucsf.edu" "node-9-03.c.msg.ucsf.edu" "node-9-04.c.msg.ucsf.edu" "node-9-05.c.msg.ucsf.edu" "node-9-06.c.msg.ucsf.edu" "node-9-07.c.msg.ucsf.edu" "node-9-08.c.msg.ucsf.edu" "node-9-09.c.msg.ucsf.edu" "node-9-10.c.msg.ucsf.edu" "nvgpu-1-1.c.msg.ucsf.edu" "nvgpu-1-2.c.msg.ucsf.edu" "nvgpu-1-3.c.msg.ucsf.edu" "nvgpu-1-4.c.msg.ucsf.edu" "nvgpu-1-5.c.msg.ucsf.edu" "nvgpu-1-6.c.msg.ucsf.edu" "nvgpu-1-7.c.msg.ucsf.edu" "nvgpu-1-8.c.msg.ucsf.edu" "nvgpu-2-1.c.msg.ucsf.edu" "nvgpu-2-2.c.msg.ucsf.edu" "nvgpu-2-3.c.msg.ucsf.edu" "nvgpu-2-4.c.msg.ucsf.edu" "nvgpu-2-5.c.msg.ucsf.edu" "nvgpu-2-6.c.msg.ucsf.edu" "nvgpu-2-7.c.msg.ucsf.edu" "nvgpu-2-8.c.msg.ucsf.edu" }
SubmissionCommand "/opt/sge/bin/lx24-amd64/qsub -pe cpu %p -l qname=nvgpu.q -l mem_free=40G -S /bin/sh"
StatusCommand "/opt/sge/bin/lx24-amd64/qstat"
ENDIF

QUEUE
Name "msg-sgehead1 gpu-cluster"
Description "Submits a GPU job to one or more nodes of nvgpu.q which has 16 12-core 2.67 GHz Xeon X5650 systems each with 46 GB of RAM and 8 GTX 580."
AcceptsSerial false
AcceptsCluster false
AcceptsThreaded false
AcceptsGPU true
# SGE will set NSLOTS so don't pass a processor count to the applications.
SetProcessorCountForApp false
ProcessControl "openmpi"
# The GPU reconstruction software will use all the GPUs (unless some are marked
# as busy) so best to use something that is a multiple of the total number of
# processors on each system.
DefaultProcessorCount 24
DefaultMPIRunCommand "/software/openmpi-1.6/bin/mpiexec"
SubmissionCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qsub -pe openmpi %p -l qname=nvgpu.q -l mem_free=40G -S /bin/sh"
StatusCommand "ssh -nx sgehead.msg.ucsf.edu /opt/sge/bin/lx24-amd64/qstat"
# Skip the unneeded call to ssh on hosts that are submit hosts.
IFHOST { "sgehead.msg.ucsf.edu" "node-9-01.c.msg.ucsf.edu" "node-9-02.c.msg.ucsf.edu" "node-9-03.c.msg.ucsf.edu" "node-9-04.c.msg.ucsf.edu" "node-9-05.c.msg.ucsf.edu" "node-9-06.c.msg.ucsf.edu" "node-9-07.c.msg.ucsf.edu" "node-9-08.c.msg.ucsf.edu" "node-9-09.c.msg.ucsf.edu" "node-9-10.c.msg.ucsf.edu" "nvgpu-1-1.c.msg.ucsf.edu" "nvgpu-1-2.c.msg.ucsf.edu" "nvgpu-1-3.c.msg.ucsf.edu" "nvgpu-1-4.c.msg.ucsf.edu" "nvgpu-1-5.c.msg.ucsf.edu" "nvgpu-1-6.c.msg.ucsf.edu" "nvgpu-1-7.c.msg.ucsf.edu" "nvgpu-1-8.c.msg.ucsf.edu" "nvgpu-2-1.c.msg.ucsf.edu" "nvgpu-2-2.c.msg.ucsf.edu" "nvgpu-2-3.c.msg.ucsf.edu" "nvgpu-2-4.c.msg.ucsf.edu" "nvgpu-2-5.c.msg.ucsf.edu" "nvgpu-2-6.c.msg.ucsf.edu" "nvgpu-2-7.c.msg.ucsf.edu" "nvgpu-2-8.c.msg.ucsf.edu" }
SubmissionCommand "/opt/sge/bin/lx24-amd64/qsub -pe openmpi %p -l qname=nvgpu.q -l mem_free=40G -S /bin/sh"
StatusCommand "/opt/sge/bin/lx24-amd64/qstat"
ENDIF

QUEUE
Name "gpu1 GTX 580"
Description "Submits a job to 2 systems.  Both have an Intel Core2 6600
(2.4 GHz), 2 GB RAM, and a Nvidia GTX 580 (1536 MB).  Please use
this queue only for jobs that can be accelerated by the GPUs."
AcceptsSerial false
AcceptsCluster false
AcceptsThreaded false
AcceptsGPU true
ProcessControl "simple"
GrecsrvRunning true
GrecsrvHosts { "c0-0" }
SubmissionCommand "ssh -nx gpu1.ucsf.edu /opt/gridengine/bin/lx26-amd64/qsub -S /bin/sh -q nv580.q"
StatusCommand "ssh -x gpu1.ucsf.edu /opt/gridengine/bin/lx26-amd64/qstat -f"
# Skip the unneeded call to ssh.
IFHOST { "gpu1.ucsf.edu" }
SubmissionCommand "/opt/gridengine/bin/lx26-amd64/qsub -S /bin/sh -q nv580.q"
StatusCommand "/opt/gridengine/bin/lx26-amd64/qstat -f"
ENDIF

QUEUE
Name "gpu1 8800 GT"
Description "Submits a job to 2 systems.  Both have an Intel P4 (3.2 GHz),
2 GB RAM, and a Nvidia 8800 GT (512 MB).  Please use this queue
only for jobs that can be accelerated by the GPUs."
AcceptsSerial false
AcceptsCluster false
AcceptsThreaded false
AcceptsGPU true
ProcessControl "simple"
GrecsrvRunning true
GrecsrvHosts { "c0-4" "c0-5" }
SubmissionCommand "ssh -nx gpu1.ucsf.edu /opt/gridengine/bin/lx26-amd64/qsub -S /bin/sh -q gt.q"
StatusCommand "ssh -x gpu1.ucsf.edu /opt/gridengine/bin/lx26-amd64/qstat -f"
# Skip the unneeded call to ssh.
IFHOST { "gpu1.ucsf.edu" }
SubmissionCommand "/opt/gridengine/bin/lx26-amd64/qsub -S /bin/sh -q gt.q"
StatusCommand "/opt/gridengine/bin/lx26-amd64/qstat -f"
ENDIF

QUEUE
Name "gpu1 GTX 480"
Description "Submits a job to one system with an Intel Core2 6600
(2.4 GHz), 2 GB RAM, and a Nvidia GTX 480 (1536 MB).  Please use
this queue only for jobs that can be accelerated by the GPUs."
AcceptsSerial false
AcceptsCluster false
AcceptsThreaded false
AcceptsGPU true
ProcessControl "simple"
GrecsrvRunning true
GrecsrvHosts { "c0-1" }
SubmissionCommand "ssh -nx gpu1.ucsf.edu /opt/gridengine/bin/lx26-amd64/qsub -S /bin/sh -q nv480.q"
StatusCommand "ssh -x gpu1.ucsf.edu /opt/gridengine/bin/lx26-amd64/qstat -f"
# Skip the unneeded call to ssh.
IFHOST { "gpu1.ucsf.edu" }
SubmissionCommand "/opt/gridengine/bin/lx26-amd64/qsub -S /bin/sh -q nv480.q"
StatusCommand "/opt/gridengine/bin/lx26-amd64/qstat -f"
ENDIF