用qsub提交连续和独立的作业有多快？

#!/bin/bash #PBS -N analyse #PBS -o analyse.out #PBS -q compute #PBS -j oe #PBS -l nodes=1:ppn=1 #PBS -l walltime=5:00:00 cd $PBS_O_WORKDIR ./x_analyse.py data_$PBS_ARRAYID.pkl

[batman@gotham tmp]$tracejob 1184430.mgt1 /var/spool/torque/server_priv/accounting/20121213: Permission denied /var/spool/torque/mom_logs/20121213: No such file or directory /var/spool/torque/sched_logs/20121213: No such file or directory Job: 1184430.mgt1 12/13/2012 13:53:13 S enqueuing into compute, state 1 hop 1 12/13/2012 13:53:13 S Job Queued at request of batman@mgt1, owner = batman@mgt1, job name = analysis_1, queue = compute 12/13/2012 13:53:13 S Job Run at request of root@mgt1 12/13/2012 13:53:13 S Not sending email: User does not want mail of this type. 12/13/2012 13:54:48 S Not sending email: User does not want mail of this type. 12/13/2012 13:54:48 S Exit_status=135 resources_used.cput=00:00:00 resources_used.mem=15596kb resources_used.vmem=150200kb resources_used.walltime=00:01:35 12/13/2012 13:54:53 S Post job file processing error 12/13/2012 13:54:53 S Email 'o' to batman@mgt1 failed: Child process '/usr/lib/sendmail -f adm batman@mgt1' returned 67 (errno 10:No child processes) [batman@gotham tmp]$tracejob 1184431.mgt1 /var/spool/torque/server_priv/accounting/20121213: Permission denied /var/spool/torque/mom_logs/20121213: No such file or directory /var/spool/torque/sched_logs/20121213: No such file or directory Job: 1184431.mgt1 12/13/2012 13:53:13 S enqueuing into compute, state 1 hop 1 12/13/2012 13:53:13 S Job Queued at request of batman@mgt1, owner = batman@mgt1, job name = analysis_2, queue = compute 12/13/2012 13:53:13 S Job Run at request of root@mgt1 12/13/2012 13:53:13 S Not sending email: User does not want mail of this type. 12/13/2012 13:53:31 S Not sending email: User does not want mail of this type. 12/13/2012 13:53:31 S Exit_status=0 resources_used.cput=00:00:16 resources_used.mem=19804kb resources_used.vmem=154364kb resources_used.walltime=00:00:18

[batman@gotham tmp]$qstat -f 1184673.mgt1 Job Id: 1184673.mgt1 Job_Name = analysis_7 Job_Owner = batman@mgt1 resources_used.cput = 00:00:16 resources_used.mem = 17572kb resources_used.vmem = 152020kb resources_used.walltime = 00:01:36 job_state = C queue = compute server = mgt1 Checkpoint = u ctime = Fri Dec 14 14:00:31 2012 Error_Path = mgt1:/gpfs1/batman/tmp/analysis_7.e1184673 exec_host = node26/0 Hold_Types = n Join_Path = oe Keep_Files = n Mail_Points = a mtime = Fri Dec 14 14:02:07 2012 Output_Path = mgt1.gotham.cis.XXXX.edu:/gpfs1/batman/tmp/analysis_7.out Priority = 0 qtime = Fri Dec 14 14:00:31 2012 Rerunable = True Resource_List.nodect = 1 Resource_List.nodes = 1:ppn=1 Resource_List.walltime = 05:00:00 session_id = 9397 Variable_List = PBS_O_HOME=/gpfs1/batman,PBS_O_LANG=en_US.UTF-8, PBS_O_LOGNAME=batman, PBS_O_PATH=/gpfs1/batman/bin:/usr/mpi/gcc/openmpi-1.4/bin:/gpfs1/batman/workhere/instal ls/mygnuplot-4.4.4/bin/:/gpfs2/condor-7.4.4/bin:/gpfs2/condor-7.4.4/sb in:/usr/lib64/openmpi/1.4-gcc/bin:/usr/kerberos/bin:/usr/local/bin:/bi n:/usr/bin:/opt/moab/bin:/opt/moab/sbin:/opt/xcat/bin:/opt/xcat/sbin, PBS_O_MAIL=/var/spool/mail/batman,PBS_O_SHELL=/bin/bash, PBS_SERVER=mgt1,PBS_O_WORKDIR=/gpfs1/batman/tmp, PBS_O_QUEUE=compute,PBS_O_HOST=mgt1 sched_hint = Post job file processing error; job 1184673.mgt1 on host node 26/0Unknown resource type REJHOST=node26 MSG=invalid home directory ' /gpfs1/batman' specified, errno=116 (Stale NFS file handle) etime = Fri Dec 14 14:00:31 2012 exit_status = 135 submit_args = analysis_7.sub start_time = Fri Dec 14 14:00:31 2012 Walltime.Remaining = 1790 start_count = 1 fault_tolerant = False comp_time = Fri Dec 14 14:02:07 2012

[batman@gotham tmp]$qstat -f 1184687.mgt1 Job Id: 1184687.mgt1 Job_Name = analysis_1 Job_Owner = batman@mgt1 resources_used.cput = 00:00:16 resources_used.mem = 19652kb resources_used.vmem = 162356kb resources_used.walltime = 00:02:38 job_state = C queue = compute server = mgt1 Checkpoint = u ctime = Fri Dec 14 14:40:46 2012 Error_Path = mgt1:/gpfs1/batman/tmp/analysis_1.e118468 7 exec_host = ionode2/0 Hold_Types = n Join_Path = oe Keep_Files = n Mail_Points = a mtime = Fri Dec 14 14:43:24 2012 Output_Path = mgt1.gotham.cis.XXXX.edu:/gpfs1/batman/tmp/analysis_1.out Priority = 0 qtime = Fri Dec 14 14:40:46 2012 Rerunable = True Resource_List.nodect = 1 Resource_List.nodes = 1:ppn=1 Resource_List.walltime = 05:00:00 session_id = 28039 Variable_List = PBS_O_HOME=/gpfs1/batman,PBS_O_LANG=en_US.UTF-8, PBS_O_LOGNAME=batman, PBS_O_PATH=/gpfs1/batman/bin:/usr/mpi/gcc/openmpi-1.4/bin:/gpfs1/batman/workhere/instal ls/mygnuplot-4.4.4/bin/:/gpfs2/condor-7.4.4/bin:/gpfs2/condor-7.4.4/sb in:/usr/lib64/openmpi/1.4-gcc/bin:/usr/kerberos/bin:/usr/local/bin:/bi n:/usr/bin:/opt/moab/bin:/opt/moab/sbin:/opt/xcat/bin:/opt/xcat/sbin, PBS_O_MAIL=/var/spool/mail/batman,PBS_O_SHELL=/bin/bash, PBS_SERVER=mgt1,PBS_O_WORKDIR=/gpfs1/batman/tmp, PBS_O_QUEUE=compute,PBS_O_HOST=mgt1 etime = Fri Dec 14 14:40:46 2012 exit_status = 0 submit_args = analysis_1.sub start_time = Fri Dec 14 14:40:47 2012 Walltime.Remaining = 1784 start_count = 1

1条回答

网友

1楼 · 发布于 2024-06-16 11:46:07

我在失败作业的tracejob输出中看到两个问题。在

首先是Exit_status=135。此退出状态不是Torque错误代码，而是脚本返回的退出状态x_analyse.py。Python没有使用sys.exit()函数的约定，135代码的源代码可能在脚本中使用的某个模块中。在

第二个问题是作业后文件处理失败。这可能表示节点配置错误。在

从现在开始我猜。由于一个成功的作业需要大约00:00:16，因此在延迟50秒的情况下，您的所有作业都会到达第一个可用节点。通过较小的延迟，您可以让更多的节点参与进来，并最终命中一个配置错误的节点，或者让两个脚本在单个节点上同时执行。我会修改提交脚本添加一行

  'echo $PBS_JOBID :: $PBS_O_HOST >> debug.log',

生成.sub文件的python脚本。这会将执行主机的名称添加到调试日志如果我正确理解了您的设置，它将驻留在一个公共文件系统中。在

然后，您（或Torque管理员）可能希望在故障节点上的MOMspool目录中查找未处理的输出文件，以获取进一步诊断所需的信息。在

相关问题更多 >

编程相关推荐

热门问题

热门文章