diff --git a/buffer/client b/buffer/client index 7d2c37c..30cd075 100755 --- a/buffer/client +++ b/buffer/client @@ -5,59 +5,46 @@ # When Stonehenge runs to completion, it touches the the file filereceived, # filetryagain, or filefailed, depending on how it has exited. This triggers # this script to move on to the next file (or try again as appropriate). -# If Stonehenge completed successfully, the script prepares a macro for its -# further processing, and creates a file in the job queue. # This script is designed for use with the L2 Master Script. It can be run # independently by giving it the arguments listed below: +# 2019 update to remove L2 and disentangle from pushnlug + # Specifying where things are located L1=$1 LOG=$2 CHOPPER=$3 -JOBQUEUE=$4 -MAC=$5 -GRIDREADY=$6 SERVERDIR=~/BufferScripts/buffer -DATA=$7 -TESTDATA=$8 -FL=$9 -LS=${10} -REDISHOST=${11} -ZDAB=${12} -testFL=~/BufferScripts/buffer/testfl.txt -testLS=~/BufferScripts/buffer/testls.txt +DATA=$4 +FL=$5 +LS=$6 +REDISHOST=$7 # This function is a wrapper for Stonehenge, which will send the appropriate # signals to the server chopper() { -if [ $TEST -eq 0 ] -then - $CHOPPER -i /dev/stdin -o $NAME -r -c $CONFIGFILE -s $TRIES -b burst >> $LOG - STONEHENGERETURN=$? -else - $CHOPPER -i /dev/stdin -o $NAME -r -c $CONFIGFILE -s $TRIES -b burst >> $LOG - STONEHENGERETURN=$? -fi +$CHOPPER -i /dev/stdin -o $NAME -r -c $CONFIGFILE -s $TRIES -b burst >> $LOG +STONEHENGERETURN=$? if [ $STONEHENGERETURN -eq 0 ] then - echo SUCCESS - Stonehenge finished processing file $FILE successfully. - touch $SERVERDIR/filereceived + echo SUCCESS - Stonehenge finished processing file $FILE successfully. + touch $SERVERDIR/filereceived else - echo WARNING - Stonehenge exited on file $FILE. + echo WARNING - Stonehenge exited on file $FILE. if [ $TRIES -lt 2 ] then - echo INFO - Trying to read file $FILE again. + echo INFO - Trying to read file $FILE again. touch $SERVERDIR/filetryagain else - echo ERROR - Three failured trying to process file $FILE. Moving on. + echo ERROR - Three failured trying to process file $FILE. Moving on. touch $SERVERDIR/filefailed fi fi } -# This function updates the text files holding the lists of files available +# This function updates the text files holding the lists of files available # on the builder, and the list of files we have already processed. updatefilelists() { @@ -78,31 +65,6 @@ sort temp.txt > $FL rm temp.txt } -# This function checks for new testdata, if available, it echos 1, -# otherwise, it echos 0. -checkfortestdata() -{ -# First update testfile lists as in updatefilelists -ls $TESTDATA/SNOP*.zdab > temp.txt -sort temp.txt > $testLS -rm temp.txt -if [ ! -f $testFL ] -then - cp $testLS $testFL -fi -comm -12 $testLS $testFL > temp.txt -sort temp.txt > $testFL -rm temp.txt - -# Then check for new data -if [ $(comm -23 $testLS $testFL | wc -l) -lt 1 ] -then - echo 0 -else - echo 1 -fi -} - # MAIN - Control flow begins here echo SUCCESS - L2 Client starting TRIES=0 @@ -118,14 +80,8 @@ do # Check for test data only if there is no real data available COUNT=0 TEST=0 - while [ $(comm -23 $LS $FL | wc -l) -lt 1 ] - do - if [ $(checkfortestdata) -eq 1 ] - then - TEST=1 - FILE=$(ls -t $(comm -23 $testLS $testFL) | tail -n 1) - break - fi + while [ $(comm -23 $LS $FL | wc -l) -lt 1 ] + do sleep 1 let COUNT++ if [ $(( COUNT % 60 )) -eq 0 ] @@ -143,12 +99,12 @@ do # TODO - Check different cases for parsing filename: # SNOP, default #////////////////////////// - echo INFO - Receiving file $FILE + echo INFO - Receiving file $FILE RUN=$( echo $FILE | awk 'BEGIN {FS="_"}{print $2}' ) SUBRUN2=$( echo $FILE | awk 'BEGIN {FS="_"}{print $3}' ) SUBRUN=$( echo $SUBRUN2 | awk 'BEGIN {FS="."}{print $1}' ) BIGFILE=$( echo $FILE | awk 'BEGIN {FS="/"}{print $5}' ) - NAME=$RUN"_"$SUBRUN + NAME=$RUN"_"$SUBRUN if [ $TEST -eq 1 ] then NAME=test$NAME @@ -162,7 +118,7 @@ do OLDRUN=$RUN # CONFIGFILE=$(runtype $RUN) CONFIGFILE=~/stonehenge/default.cnfg - fi + fi # Receive file - try three times, report errors FILESTOPROCESS=$(comm -23 $LS $FL) @@ -180,12 +136,12 @@ do FILESTOPROCESS=$(comm -23 $LS $FL) if [ $FILESTOPROCESS -gt 1 ] then - WAITTIME=$($(date +%s) - $(date +%s -r /raid/data/l2/SNOP_$NAME.l2.zdab)) + WAITTIME=$($(date +%s) - $(date +%s -r /raid/data/l1/SNOP_$NAME.zdab)) if [ $WAITTIME -gt 60 ] then touch $SERVERDIR/fileended fi - fi +# fi sleep 1 done if [ -f $SERVERDIR/filereceived ] @@ -194,12 +150,7 @@ do rm $SERVERDIR/filereceived kill $PID echo INFO - Reached end of $FILE - if [ $TEST -eq 0 ] - then - echo $FILE >> $FL - else - echo $FILE >> $testFL - fi + echo $FILE >> $FL TRIES=0 DONE=1 elif [ -f $SERVERDIR/filetryagain ] @@ -213,12 +164,7 @@ do rm $SERVERDIR/filefailed kill $PID echo INFO - Server will not try to read file $FILE again. - if [ $TEST -eq 0 ] - then - echo $FILE >> $FL - else - echo $FILE >> $testFL - fi + echo $FILE >> $FL let TRIES++ elif [ -f $SERVERDIR/fileended ] then @@ -229,26 +175,9 @@ do echo DEBUG - Message file disappeared fi -# If file successfully received, prepare for conversion +# If file successfully received, end if [ $DONE -eq 1 ] then - cat > $MAC/$NAME.mac << EOF - -/rat/physics_list/OmitAll true -/rat/inzdab/load $ZDAB/$NAME.zdab - -/run/initialize - -/rat/proclast outroot -/rat/procset file "$NAME.l2.root" - -/rat/inzdab/read - -exit -EOF - - JOBFILE=$JOBQUEUE/$NAME - echo $SERVERDIR/job $MAC $NAME $DATA/$BIGFILE $GRIDREADY $REDISHOST >> $JOBFILE echo SUCCESS - Stonehenge done with run $NAME redis-cli -h $REDISHOST ZREM l2:process $BIGFILE redis-cli -h $REDISHOST ZADD l2:wait $(date +%s) $BIGFILE diff --git a/l2 b/l2 index 5df5c51..bd2d0f6 100755 --- a/l2 +++ b/l2 @@ -2,28 +2,25 @@ # This script can start and stop the three main components of the L2 system: # the client, convert, and manager scripts. It tracks the PGIDs of these -# processes in the file $PGID. When starting, it also ensures that the +# processes in the file $PGID. When starting, it also ensures that the # necessary files and directory structures are present, and, if not, constructs # them. # Do not delete this weird line # It sets the option which assigns new pgid's to backgrounded pipelines + +# 2019 update to remove L2 output and disentangle from pushnlug + set -m # Organization of locations. Set directory structure here: -BS=$HOME/BufferScripts # Path to the BufferScripts repository -FILESYSTEM=/raid # Buffer filesystem -L1=$FILESYSTEM/data/l1 # Directory for raw data -LOGDIR=$BS/log # Central location for L2 log files -ZDAB=$HOME/zdab # Directory for post-L2 zdab files -MAC=$BS/mac # Directory for conversion macros -GRIDREADY=$FILESYSTEM/data/l2 # Output directory for finished products -RATENV=/opt/env/latest.sh # Location of rat_env.sh script -DONTDELETE=$BS/dontdelete # directory for writing deletion locks -DATA=$FILESYSTEM/data/l1 # normal data directory -TESTDATA=$HOME/testbuilderdata # test data directory -CHOPPER=$HOME/stonehenge/stonehenge.sh # Location of Stonehenge program -REDISHOST=192.168.80.128 # Location of monitoring redis db +BS=$HOME/BufferScripts # Path to the BufferScripts repository +FILESYSTEM=/raid # Buffer filesystem +L1=$FILESYSTEM/data/l1 # Directory for raw data +LOGDIR=$BS/log # Central location for L2 log files +DATA=$FILESYSTEM/data/l1 # normal data directory +CHOPPER=$HOME/stonehenge/stonehenge.sh # Location of Stonehenge program +REDISHOST=192.168.80.128 # Location of monitoring redis db #---------------------------------------------------------- # Below this line we specify the relative paths to different components of # the L2 system. This should not need to be edited to reflect local installations. @@ -32,11 +29,6 @@ SERVERLS=$BS/buffer/ls.txt CLIENT=$BS/buffer/client STONELOG=$LOGDIR/stone.log CLIENTLOG=$LOGDIR/client.log -JOBQUEUE=$BS/buffer/jobs -CONVERT=$BS/buffer/convert -CONVERTLOG=$LOGDIR/convert.log -MANAGER=$BS/buffer/manager -MANAGERLOG=$LOGDIR/manager.log PGIDFILE=$BS/pgid.txt MONITOR=$BS/monitor.py #----------------------------------------------------------- @@ -79,14 +71,14 @@ parsepgid(){ exit 1 ;; esac - done < $PGIDFILE + done < $PGIDFILE fi } # This function updates the PGID file # Should only be called if parsepgid has already been called writepgid(){ - echo -e $clientpgid '\n'$convertpgid '\n'$managerpgid > $PGIDFILE + echo -e $clientpgid '\n'$convertpgid '\n'$managerpgid > $PGIDFILE } # This function starts individual scripts @@ -97,35 +89,14 @@ startprocess(){ "client") if [ "$clientpgid" -eq 0 ] then - $CLIENT $L1 $STONELOG $CHOPPER $JOBQUEUE $MAC \ - $GRIDREADY $DATA $TESTDATA $SERVERFL $SERVERLS $REDISHOST $ZDAB \ + $CLIENT $L1 $STONELOG $CHOPPER \ + $DATA $SERVERFL $SERVERLS $REDISHOST \ 2>> $CLIENTLOG | python $MONITOR L2-client >> $CLIENTLOG 2>&1 & clientpgid=$(jobs -lp | tail -n 1) else echo Client process was already running. fi ;; - "convert") - if [ "$convertpgid" -eq 0 ] - then - $CONVERT $JOBQUEUE $CONVERTLOG $MAC $RATENV $REDISHOST \ - $ZDAB 2>> $CONVERTLOG | \ - python $MONITOR L2-convert >> $CONVERTLOG 2>&1 & - convertpgid=$(jobs -lp | tail -n 1) - else - echo Convert process was already running. - fi - ;; - "manager") - if [ "$managerpgid" -eq 0 ] - then - $MANAGER $FILESYSTEM $L1 $DONTDELETE $MAC 2>> $MANAGERLOG | \ - python $MONITOR L1-delete >> $MANAGERLOG 2>&1 & - managerpgid=$(jobs -lp | tail -n 1) - else - echo Manager process was already running. - fi - ;; *) echo -e 'Invalid process name passed\n' echo -e 'Valid options are client, convert, manager\n' @@ -149,24 +120,6 @@ stopprocess(){ echo Client process was not running. fi ;; - "convert") - if [ $convertpgid -ne 0 ] - then - kill -- -$convertpgid - convertpgid=0 - else - echo Convert process was not running. - fi - ;; - "manager") - if [ $managerpgid -ne 0 ] - then - kill -- -$managerpgid - managerpgid=0 - else - echo Manager process was not running. - fi - ;; *) echo -e 'Invalid process name passed.\n' echo -e 'Valid options are client, convert, and manager.\n' @@ -176,56 +129,6 @@ stopprocess(){ writepgid } -# This function sets up the L2 scripts to start -start_all(){ - parsepgid - if [[ "$clientpgid" -eq 0 && "$convertpgid" -eq 0 && "$managerpgid" -eq 0 ]] - then - echo Preparing file structure... - if [ ! -d $LOGDIR ] - then - mkdir $LOGDIR - fi - if [ ! -d $ZDAB ] - then - mkdir $ZDAB - fi - if [ ! -d $MAC ] - then - mkdir $MAC - fi - if [ ! -d $L1 ] - then - mkdir $L1 - fi - if [ ! -d $JOBQUEUE ] - then - mkdir $JOBQUEUE - fi - if [ ! -d $DONTDELETE ] - then - echo No DONTDELETE folder! Aborting! - exit 0 - fi - echo File structure prepared. - echo Starting level 2 scripts... - startprocess client - startprocess convert - startprocess manager - else - echo Some processes were already running. Please stop all scripts before \ - using the -a option. If you do not want to stop running processes, \ - you can start the processes one at a time using the -s option. - fi -} - -stop_all(){ - echo Killing L2 processes - stopprocess client - stopprocess convert - stopprocess manager -} - # MAIN if [[ $# -eq 0 ]] then diff --git a/pushnlug.sh b/pushnlug.sh index 43318b6..90646ea 100755 --- a/pushnlug.sh +++ b/pushnlug.sh @@ -3,26 +3,21 @@ # This script handles pushing files to the nfs server on nlug. # It watches several directories and scp's new files as they # become available. The code is based on the L2 client. +# 2019 update to remove L2 and disentangle from L2 client # Specifying where things are located-------------- ## directories holding the data to ship burstdir=/raid/data/burst l1dir=/raid/data/l1 -l2dir=/raid/data/l2 -testdir=/raid/data/test -zdabdir=/raid/data/zdab -l2donedir=/home/trigger/l2done +l1donedir=/home/trigger/l1done + ## filelist files LSl1=/home/trigger/BufferScripts/nlug/lsl1.txt -LSl2=/home/trigger/BufferScripts/nlug/lsl2.txt -LSburst=/home/trigger/BufferScripts/nlug/lsburst.txt -LStest=/home/trigger/BufferScripts/nlug/lstest.txt FLl1=/home/trigger/BufferScripts/nlug/fll1.txt -FLl2=/home/trigger/BufferScripts/nlug/fll2.txt -FLburst=/home/trigger/BufferScripts/nlug/flburst.txt -FLtest=/home/trigger/BufferScripts/nlug/fltest.txt + ## nlug stuff NLUG=192.168.80.138 + ## builder directory BUILDERDATA=/raid/data/l1 #-------------------------------------------------- @@ -42,61 +37,26 @@ do done sort nlugtemp.txt > $LSl1 rm nlugtemp.txt -## L2 -find $l2dir | grep .zdab > nlugtemp.txt -sort nlugtemp.txt > $LSl2 -rm nlugtemp.txt -## Bursts -find $burstdir | grep .zdab > nlugtemp.txt -sort nlugtemp.txt > $LSburst -rm nlugtemp.txt -## Test -find $testdir | grep .zdab > nlugtemp.txt -sort nlugtemp.txt > $LStest -rm nlugtemp.txt # If there is no $FL file, assume we want to ship everything available if [ ! -f $FLl1 ] then cp $LSl1 $FLl1 fi -if [ ! -f $FLl2 ] -then - cp $LSl2 $FLl2 -fi -if [ ! -f $FLburst ] -then - cp $LSburst $FLburst -fi -if [ ! -f $FLtest ] -then - cp $LStest $FLtest -fi # Drop files from $FL no longer in $LS comm -12 $LSl1 $FLl1 > nlugtemp.txt sort nlugtemp.txt > $FLl1 rm nlugtemp.txt -comm -12 $LSl2 $FLl2 > nlugtemp.txt -sort nlugtemp.txt > $FLl2 -rm nlugtemp.txt - -comm -12 $LSburst $FLburst > nlugtemp.txt -sort nlugtemp.txt > $FLburst -rm nlugtemp.txt - -comm -12 $LStest $FLtest > nlugtemp.txt -sort nlugtemp.txt > $FLtest -rm nlugtemp.txt } # This function checks whether all the subfiles of a run have been transferred -# and if so, pushes a file named by the runnumber to nlug. This is for the +# and if so, pushes a file named by the runnumber to nlug. This is for the # use of the nearline master program. fullrun(){ # First identify last file shipped - LASTFILE=$(cat $FLl2 | tail -n 1) + LASTFILE=$(cat $FLl1 | tail -n 1) LASTRUN=$( echo $LASTFILE | awk 'BEGIN {FS="_"}{print $2}' ) LASTSUBRUN=$( echo $LASTFILE | awk 'BEGIN {FS="_"}{print $3}' ) LASTSUBRUN=$( echo $LASTSUBRUN | cut -d '.' -f1) @@ -105,27 +65,29 @@ fullrun(){ BUILDERFILE=$(ls $BUILDERDATA | grep closed | grep $LASTRUN | grep $GLASTSUBRUN) if [ $(cat $BUILDERDATA/'SNOP_'$LASTRUN'_'$LASTSUBRUN'.zdab_closed' | grep END_RUN | wc -l) -ne 0 ] then - # Builder is done + # Builder is done # Check whether other subfiles have been shipped let "LASTSUBRUN +=1" - if [ $(cat $FLl2 | grep $LASTRUN | wc -l) -eq $LASTSUBRUN ] + if [ $(cat $FLl1 | grep $LASTRUN | wc -l) -eq $LASTSUBRUN ] then # All files shipped - tell nlug - ssh $NLUG touch $l2donedir/$LASTRUN + ssh $NLUG touch $l1donedir/$LASTRUN else # Remember this for future - echo $LASTSUBRUN > $l2donedir/$LASTRUN + echo $LASTSUBRUN > $l1donedir/$LASTRUN fi else # Not last file, check whether this completes a finished run anyway - if [ -f $l2donedir/$LASTRUN ] + if [ -f $l1donedir/$LASTRUN ] then - SUBFILES=$(echo $l2donedir/$LASTRUN) - if [ $(cat FLl2 | grep $LASTRUN | wc -l) -eq $SUBFILES -1 ] + LASTSUBRUN=$( cat $l1donedir/$LASTRUN ) + LASTSUBRUN=$((10#$LASTSUBRUN)) + let "LASTSUBRUN +=1" + if [ $(cat FLl1 | grep $LASTRUN | wc -l) -eq $LASTSUBRUN ] then - ssh $NLUG touch $l2donedir/$LASTRUN + ssh $NLUG touch $l1donedir/$LASTRUN fi - fi + fi fi } @@ -137,48 +99,20 @@ do if [ $(comm -23 $LSl1 $FLl1 | wc -l) -gt 0 ] then FILE=$(ls -t $(comm -23 $LSl1 $FLl1) | tail -n 1) - scp $FILE $NLUG:$FILE + scp $FILE $NLUG:$FILE echo $FILE >> $FLl1 sort $FLl1 > nlugtemp.txt mv nlugtemp.txt $FLl1 ZCOUNT=$(ssh nlug "ls /raid/data/zdab" | wc -l) - if [ $ZCOUNT -eq 10 ] + if [ $ZCOUNT -eq 200 ] then - ZFILE=$(ssh nlug "ls -tr /raid/data/zdab" | tail -n 1) + ZFILE=$(ssh nlug "ls -t /raid/data/zdab" | tail -n 1) ssh nlug "rm /raid/data/zdab/$ZFILE" fi ZFILE=$(ssh nlug 'ls -tr /raid/data/l1' | tail -n 1) ssh nlug "cp /raid/data/l1/$ZFILE /raid/data/zdab/$ZFILE" fi - # L2 - if [ $(comm -23 $LSl2 $FLl2 | wc -l) -gt 0 ] - then - FILE=$(ls -t $(comm -23 $LSl2 $FLl2) | tail -n 1) - scp $FILE $NLUG:$FILE - echo $FILE >> $FLl2 - sort $FLl2 > nlugtemp.txt - mv nlugtemp.txt $FLl2 - fullrun - fi - # bursts - if [ $(comm -23 $LSburst $FLburst | wc -l) -gt 0 ] - then - FILE=$(ls -t $(comm -23 $LSburst $FLburst) | tail -n 1) - scp $FILE $NLUG:$FILE - echo $FILE >> $FLburst - sort $FLburst > nlugtemp.txt - mv nlugtemp.txt $FLburst - fi - # test - if [ $(comm -23 $LStest $FLtest | wc -l) -gt 0 ] - then - FILE=$(ls -t $(comm -23 $LStest $FLtest) | tail -n 1) - scp $FILE $NLUG:$FILE - echo $FILE >> $FLtest - sort $FLtest > nlugtemp.txt - mv nlugtemp.txt $FLtest - fi # Don't sleep for too long to reduce latency in getting burst # files to nlug - sleep 1 + sleep 0.1 done