Link: API Status Page for Caltech-Dev
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586711 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586711 CXX_THREAD fork_parent@MountPointStatus.cc end
941586711 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586711 CXX_THREAD fork_parent@MountPointStatus.cc end
941586711 STARTUP execOverload overloaded exec
941586711 STARTUP closeListenSock no cid registered for service 'data'
941586711 STARTUP mpi::init unused data port 10021 closed
941586711 STARTUP mpi::init port 10021 (jobstate) opened on metaserver as sock3
941586711 STARTUP bgLoop Looping process watchlogs started
941586711 STARTUP openListenSock port 10019 (operator) opened on metaserver as sock4
941586711 STARTUP openListenSock port 10020 (emergency) opened on metaserver as sock5
941586711 STARTUP setResourceLimit vmemoryuse=unlimited; datasize=unlimited; core=unlimited; maxproc=65536; descriptors=1024; memorylocked=32768; filesize=unlimited; cputime=unlimited
941586711 STARTUP leakLogger inital size of mpi API: 13456 kB
941586711 STARTUP bgLoop Looping process etchosts started
941586711 STARTUP mpi Trying API
941586711 STARTUP mpi API yes
941586711 STARTUP mpi Trying LDAS_SYSTEM
941586711 STARTUP mpi LDAS_SYSTEM yes
941586711 IDLE exec exec /usr/sbin/lsof -b -p 5712
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586711 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586711 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586711 CXX_THREAD fork_parent@MountPointStatus.cc end
941586711 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586711 CXX_THREAD fork_parent@MountPointStatus.cc end
941586711 IDLE bgLoop Looping process statpagefile started
941586711 IDLE bgLoop Looping process killedjobreaper started
941586716 IDLE setFTPandHTTPinfo (::FTPURL 'ftp://131.215.115.248') (::FTPDIR '') (::HTTPURL 'http://131.215.115.248/ldas_outgoing/jobs') (::HTTPDIR '/ldas_outgoing/jobs') (::GRIDFTPURL 'gridftp:/export/grid/ldas') (::GRIDFTPDIR '/export/grid/ldas') (::LDAS_GATEWAY 'ldas-dev 131.215.115.248') (::LDAS_SYSTEM 'ldas-dev') (::RUNCODE 'LDAS-DEV')
941586716 STARTUP mpi::killAllMpirun cleaning up for user ldas
941586716 IDLE ::exec exec /usr/bin/ssh -n -l ldas beowulf /bin/rm -rf /tmp/lam-ldas@*
941586716 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586716 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586716 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586716 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586716 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586716 CXX_THREAD fork_parent@MountPointStatus.cc end
941586716 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586716 CXX_THREAD fork_parent@MountPointStatus.cc end
941586719 IDLE ::exec exec /usr/bin/ssh -n beowulf sudo pkill -9 -u search01,search02,search03,search04,search05,search06,search07,search08,search09,search10,search11,search12,search13,search14,search15,search16 wrapperAPI
941586719 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586719 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586719 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586719 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586719 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586719 CXX_THREAD fork_parent@MountPointStatus.cc end
941586719 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586719 CXX_THREAD fork_parent@MountPointStatus.cc end
941586722 IDLE ::exec exec /usr/bin/ssh -n beowulf sudo pkill -9 -u search01,search02,search03,search04,search05,search06,search07,search08,search09,search10,search11,search12,search13,search14,search15,search16 lamd
941586722 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586722 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586722 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586722 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586722 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586722 CXX_THREAD fork_parent@MountPointStatus.cc end
941586722 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586722 CXX_THREAD fork_parent@MountPointStatus.cc end
941586725 STARTUP mpi::killAllMpirun ran kill 10 times in 9.134 seconds
941586725 STARTUP mpi::prestartLamds running lamboot for user search01
941586725 IDLE ::exec exec scp -B conf.lam search01@metaserver:
941586725 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586725 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586725 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586725 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586725 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586725 CXX_THREAD fork_parent@MountPointStatus.cc end
941586725 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586725 CXX_THREAD fork_parent@MountPointStatus.cc end
941586726 IDLE ::exec exec scp -B conf.lam search01@metaserver:
941586726 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586726 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586726 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941586726 CXX_THREAD fork_prepare@MountPointStatus.cc end
941586726 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586726 CXX_THREAD fork_parent@MountPointStatus.cc end
941586726 CXX_THREAD fork_parent@MountPointStatus.cc begin
941586726 CXX_THREAD fork_parent@MountPointStatus.cc end
941586726 STARTUP mpi::prestartLamds Subject: LDAS-DEV mpiAPI lamboot error!; Body: lam::boot: lam::pushSchema: search01@metaserver: Permission denied (publickey,gssapi-with-mic,password). lost connection
941586726 STARTUP mpi::killAllMpirun {mpi::cleanupTmp: ldas@beowulf: ssh: connect to host beowulf port 22: No route to host} {ldas@beowulf:wrapperAPI: ssh: connect to host beowulf port 22: No route to host} {ldas@beowulf:lamd: ssh: connect to host beowulf port 22: No route to host}
941586726 IDLE setFTPandHTTPinfo (::FTPURL 'ftp://131.215.115.248') (::FTPDIR '') (::HTTPURL 'http://131.215.115.248/ldas_outgoing/jobs') (::HTTPDIR '/ldas_outgoing/jobs') (::GRIDFTPURL 'gridftp:/export/grid/ldas') (::GRIDFTPDIR '/export/grid/ldas') (::LDAS_GATEWAY 'ldas-dev 131.215.115.248') (::LDAS_SYSTEM 'ldas-dev') (::RUNCODE 'LDAS-DEV')
941586731 IDLE mpi::updateCmonNodelist updated ::beowulfNodes in cntlmonAPI to 'beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf'
941586737 IDLE mpi::updateCmonNodelist updated ::beowulfNodes in cntlmonAPI to 'beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf'
941587299 SHUTDOWN closeListenSock port 10019 (sock4) (operator) closed on metaserver
941587299 SHUTDOWN mpi::sHuTdOwN Subject: LDAS Caltech-Dev mpi shutdown at 941587298 ( 11/06/2009 04:01:23 PM PST ); Body: mpi shutting down NOW , nocore
941587299 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search01'
941587299 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search01 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587299 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587299 CXX_THREAD fork_parent@MountPointStatus.cc end
941587299 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587299 CXX_THREAD fork_parent@MountPointStatus.cc end
941587299 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587299 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search02'
941587299 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search02 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587299 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587299 CXX_THREAD fork_parent@MountPointStatus.cc end
941587299 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587299 CXX_THREAD fork_parent@MountPointStatus.cc end
941587299 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587299 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search03'
941587299 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search03 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587299 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587299 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587299 CXX_THREAD fork_parent@MountPointStatus.cc end
941587299 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587299 CXX_THREAD fork_parent@MountPointStatus.cc end
941587300 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587300 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search04'
941587300 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search04 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587300 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587300 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587300 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587300 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587300 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587300 CXX_THREAD fork_parent@MountPointStatus.cc end
941587300 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587300 CXX_THREAD fork_parent@MountPointStatus.cc end
941587301 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587301 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search05'
941587301 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search05 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587301 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587301 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587301 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587301 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587301 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587301 CXX_THREAD fork_parent@MountPointStatus.cc end
941587301 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587301 CXX_THREAD fork_parent@MountPointStatus.cc end
941587302 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587302 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search06'
941587302 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search06 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587302 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587302 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587302 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587302 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587302 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587302 CXX_THREAD fork_parent@MountPointStatus.cc end
941587302 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587302 CXX_THREAD fork_parent@MountPointStatus.cc end
941587303 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587303 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search07'
941587303 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search07 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587303 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587303 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587303 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587303 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587303 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587303 CXX_THREAD fork_parent@MountPointStatus.cc end
941587303 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587303 CXX_THREAD fork_parent@MountPointStatus.cc end
941587304 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587304 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search08'
941587304 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search08 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587304 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587304 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587304 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587304 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587304 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587304 CXX_THREAD fork_parent@MountPointStatus.cc end
941587304 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587304 CXX_THREAD fork_parent@MountPointStatus.cc end
941587306 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587306 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search09'
941587306 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search09 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587306 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587306 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587306 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587306 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587306 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587306 CXX_THREAD fork_parent@MountPointStatus.cc end
941587306 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587306 CXX_THREAD fork_parent@MountPointStatus.cc end
941587307 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587307 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search10'
941587307 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search10 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587307 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587307 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587307 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587307 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587307 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587307 CXX_THREAD fork_parent@MountPointStatus.cc end
941587307 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587307 CXX_THREAD fork_parent@MountPointStatus.cc end
941587308 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587308 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search11'
941587308 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search11 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587308 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587308 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587308 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587308 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587308 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587308 CXX_THREAD fork_parent@MountPointStatus.cc end
941587308 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587308 CXX_THREAD fork_parent@MountPointStatus.cc end
941587309 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587309 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search12'
941587309 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search12 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587309 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587309 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587309 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587309 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587309 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587309 CXX_THREAD fork_parent@MountPointStatus.cc end
941587309 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587309 CXX_THREAD fork_parent@MountPointStatus.cc end
941587310 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587310 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search13'
941587310 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search13 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587310 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587310 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587310 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587310 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587310 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587310 CXX_THREAD fork_parent@MountPointStatus.cc end
941587310 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587310 CXX_THREAD fork_parent@MountPointStatus.cc end
941587311 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587311 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search14'
941587311 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search14 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587311 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587311 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587311 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587311 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587311 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587311 CXX_THREAD fork_parent@MountPointStatus.cc end
941587311 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587311 CXX_THREAD fork_parent@MountPointStatus.cc end
941587314 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
941587314 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search15'
941587314 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search15 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
941587314 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587314 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587314 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587314 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587314 CXX_THREAD fork_parent@MountPointStatus.cc begin