1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101

#!/bin/bash

# qtail.sh - run just like qsub, but:
#   (1) It also runs tail on the output and error files, so it "feels" like an
#       interactive session
#   (2) If you kill the tail function, it kills the qsub job.  This is useful
#       for debugging.  If it is not desired, either delete the last while
#       loop, or add an option to not perform that loop
# Author: Andrew M. Chap
# Last edited August 2018

# Trap function so we kill the qsub job when we kill our qtail function
function finish {
    echo deleting jobID $jobID
    qdel $jobID
}

# start main function
function qtail {

    # First argument must be the qsub file
    qsubFile=$1

    # Store the rest of the arguments (including options)
    otherArgs="${@:2}"

    # Get the name of the file before the extension via parameter expansion
    qsubName=${qsubFile%%.*}
    echo qsubName is $qsubName
    
    # Get the name of the error and output files
    for errout in err out
    do

        flag="-${errout:0:1}" # flag for err is "-e" and out is "-o" 

        # First, look in the passed arguments for -e or -o options
        file=$(echo $otherArgs | sed -n -e "s|^.*$flag ||p") # get everything after flag
        file=${file/%\ */} # keep first word only

        # If we didn't find any -e or -o options in arguments, search the qsub file
        # for these arguments
        if [ -z "$file" ]
        then
            file=$(sed -n -e "s|^.*$flag ||p" $qsubFile)
        fi

        # If we still didn't find them, then they were unspecified, so we will name
        # them with the same name as the job name
        if [ -z "$file" ]
        then
            file=$qsubName.$errout
        fi

        # Assign to error file or output file
        if [ $errout == err ]
        then
            errFile=$file
        elif [ $errout == out ]
        then
            outFile=$file
        fi        

    done

    # if error or output files exist, remove them
    rm $outFile $errFile 2> /dev/null

    # create error and output files
    touch $outFile $errFile

    jobOutput=$(qsub $qsubFile $otherArgs)
    # ID number of the job is the first number output by the qsub command
    jobID=$(echo $jobOutput | sed -e "s|\([0-9]\+\).*|\1|")
    echo jobID is $jobID
    
    # setup exit trap, so that if we kill this function it kills the qsub job too
    trap finish SIGINT EXIT SIGTERM INT
        
    # Launch tail function as a subshell but redirect output back to stdout
    exec 3>&1 # creates 3 as alias for 1

    $(tail -f $outFile -f $errFile >&3) & #>&3 &  # tail both the out and error files

    # get PID of tail so we can terminate it when qsub job is complete
    TAIL_PID=$! 
    
    # check every second the status of the job and kill TAIL_PID if the job is complete
    stillRunning=True
    while [[ "$stillRunning" == 'True' ]]
    do
        if qstat $jobID | grep ' C '
        then
            echo job is complete
            kill -9 $TAIL_PID
            stillRunning=False
            echo qsub complete
        fi
        sleep 1
    done
}