#! /bin/sh
#
# NOTE: run this command with the -help option for explanation.
#

# can be changed from "echo" to ":" later... I don't trust the command
# enough to it yet though.
verbose=echo

# for picky builds (gcc compiler)
#conf_extraargs="\"--with-TARGET-CFLAGS=-W -Wall\""

# will suppress testing
#supress_test=yes

started_date=`date`

report_error()
{
    host=`hostname`
    date=`date`
    mailmsg="$HOME/tmp-mailmsg.$$"
    touch ${mailmsg} 1>/dev/null 2>/dev/null
    cat 1>>${mailmsg} 2>/dev/null <<EOF
Subject: nightly build output
To: ${mailto}

${date} :  ${host}  [pid $$, started ${started_date}] 

$2

EOF
    tail -100 $1 1>>${mailmsg} 2>/dev/null


    if [ -z "${architecture}" ]; then
	architecture=dummy
    fi
    case "${nomail}:${architecture}" in
	true:*)
	    cat ${mailmsg}
	    ;;
	*:linux | *:aix)
	    mail -s "nightly build output" ${mailto} < ${mailmsg} 1>/dev/null 2>/dev/null
	    ;;
	*)
	    mail ${mailto} < ${mailmsg} 1>/dev/null 2>/dev/null
	    ;;
    esac	    
    rm -f ${mailmsg} 1>/dev/null 2>/dev/null
    exit 1
}


usage()
{
    cat <<EOF

Syntax: globus-test-suite  <architecture> [setup opts] [location opts] [action opts]

    This script will download, configure, build, install and test Globus
    and MPICH-G. I have tried to make as much as possible optional, and
    then provide a couple of convenience alias options (-full*).

    Default values for various options below are listed in brackets [ ].
    Dependencies between options (one option might use the directory defined
    in another option) are listed in curly brackets { }.

Required Argument:

    <architecture>      One of aix,irix,linux,nt,solaris2,solaris7,...

Optionally Required:

    -mdsorg    <dn>    MDS organizational DN  [same organization as your proxy]
    -mdspasswd <pass>  MDS admin password    

    These variables are required when you trigger a -local-deploy action
    (see below).

Basic Options:

    -dryrun            output what would have been done
    -noemail           do not send mail: print to stderr instead
    -user      <user>  local user name       [output of "whoami"]
    -domain    <dom.>  domain name           [domain part of "hostname"]
    -mailto    <addr>  email address of user [<user>@<domain>]

Source options:

    The default settings point to tar files that are currently updated
    on a daily basis.

    -srchost   <srchost>  [pitcairn.mcs.anl.gov]
    -scp       <scp>      [scp]
    -globustar <file>     [/sandboxes/pitcairn.mcs.anl.gov/larsson/globus.tar]
    -mpichtar  <file>     [/sandboxes/pitcairn.mcs.anl.gov/larsson/mpich.tar ]

    If you have a different username at the host pointed to by <srchost>,
    set <srchost> to <remoteuser>@<remotehost>.

    Note that -scp globus-rcp is a viable and recommended alternative,
    as you don't have to provide a password (good for cronjobs, etc)

    NOTE: if <srchost> is set to "local", no copy to local disk is
	  performed: the tar files are untarred directly from their
	  respective locations.

Test Space options:

    This is the default layout of the test space:
    <anchor>
	<globus>
	<mpich>
	<base1>
	    <scratch>
	    <build>
	    <install>
	    <deploy>
	<base2>
	...

    NOTE: Unless full paths are used, <base>, <globus> and <mpich> are
	  relative to <anchor>, and all other directories are relative
	  to <base>.

    -anchor    <dir>   test space location   [/sandbox/<user>/nightly-build]
    -globus    <dir>   globus source tree    [globus]
    -mpich     <dir>   mpich source tree     [mpich]
    -base      <dir>   test case base dir    [.]

    -scratch   <dir>   temp config files etc [.]
    -build     <dir>   build dir             [BUILD]
    -install   <dir>   install dir           [install]
    -deploy    <dir>   deploy dir            [deploy]
    -tools     <dir>   tools dir for tests   [output of
					      <install>/bin/globus-tools-path]
    -flavor    <dir>   mpich "globusdir"     [output of 
                                        <install>/bin/globus-development-path]

    -reset-base        resets {scratch,build,install,deploy,tools,flavor} to
		       the default settings (relative to <base>)

    -ssl32     <dir>   32-bit SSLeay install [/soft/pub/packages/SSLeay-0.9.0]
    -ssl64     <dir>   64-bit SSLeay install [<ssl32>/../SSLeay-0.9.0-n64]

    -ldap32    <dir>   32-bit LDAP install [/soft/pub/packages/OpenLDAP-1.2.7]
    -ldap64    <dir>   64-bit LDAP install [<ldap32>/../OpenLDAP-1.2.7-n64]
    
    -lock     <file>   file containing anchor lock [action option aquires one]
    -nolock            override acquiring a lock before starting

Configure options:

    -inparallel        build all flavors in parallel    [sequential]
    -64                do a 64-bit compile/install      [32-bit is default]
    -nodebug           compile without debug            [with debug]
    -threads  <type>   use the <type> thread package    [no threads at all]
    -jmtype   <type>   flavor of job manager to ues     [fork]

    -contact <contact> gatekeeper contact for testing. If not given, one is
		       configured for a fork service and started {deploy,tools}

Action options:

    -clean             remove all contents {base}
    -setup-globus      acquire tar file and untar {src*,scp,globustar,globus}
    -setup-mpich       acquire tar file and untar {src*,scp,mpichtar,mpich}
    -configure         configure/make/make install {globus,build,install}
    -gatekeeper        start a gatekeeper print out contact {deploy,tools}
    -test              run tests {build,tools,contact}
    -globus-install    globus-install {globus,build,install}
    -flavor-tests      run -test for each build flavor {build,tools,contact}
    -local-deploy      do a globus-setup/globus-local deploy {install,deploy}
    -make-mpich        configure and make MPICH-G {mpich,flavor}
    -run-mpich         run test suite {mpich,contact}
    -lock-anchor       lock the anchor directory and exit {anchor}

Future action options:

    [-test-deploy]      ???  {deploy}

Special alias options:

    -mcscron           -srchost local -scp globus-rcp
		       will also define PATH to something normal (MCS only)

    -full-setup        -clean -setup-globus -setup-mpich
    -full-basic        -base basic -clean -reset-base -configure \\
		       -deploy install -test
    -full-threaded     -base threaded -clean -reset-base -threads pthreads \\
		       -configure -deploy ../basic/install -test
    -full-32bit        -base 32bit -clean -reset-base -build . \\
		       -globus-install -local-deploy -flavor-tests
    -full-mpich-32     -base 32bit -reset-base -make-mpich -run-mpich
    -full-64bit        -base 64bit -clean  -reset-base -build . -64 \\
		       -globus-install -local-deploy -flavor-tests
    -full-mpich-64     -base 64bit -reset-base -make-mpich -run-mpich

    -full              all of the -full-* commands (hence all the -reset-base
		       options...), in the order they are listed, with the
		       exception that 64-bit builds are done on IRIX only. 

EOF
}


stringify_arguments()
{
    # stringify $@ returns "$1" "$2" ...

    _arglist=
    for _arg in "$@" ; do
	_item=`echo ${_arg} | sed -e 's/"/""/g' -e 's/^.*$/"&"/g'`
	_arglist="${_arglist} ${_item}"
    done
    echo ${_arglist}
}


kill_my_globus_processes ()
{
    eval ${ps_command}  > /tmp/processlist.$$
    if grep globus /tmp/processlist.$$ > /dev/null ; then
	for p in `grep globus /tmp/processlist.$$ | awk '{print $1}'` ; do
	    if [ $p -ne $$ ]; then
		${verbose} killing $p
		kill -9 $p
	    fi
	done
    fi
    rm -f /tmp/processlist.$$
    return 1
}


wait_or_timeout ()
{
    # wait_or_timeout <pid> <timeout>
    #
    # replacement for wait <pid>, where the pid might hang because
    # of deadlock in Globus.
    #

    _pid=$1
    _timeout=$2
    _total=0
    _rc=1
    while [ ${_total} -lt ${_timeout} ]; do
	_alive=false
	${dryrun} rm -f ${scratch}/x
	${dryrun} eval ${ps_command} | awk '{print $1}' > ${scratch}/x
	for p in `cat ${scratch}/x` ; do
	    if [ $p != PID ]; then
		if [ $p -eq ${_pid} ]; then
		    _alive=true
		fi
	    fi
	done
	if [ ${_alive} = false ]; then
	    _rc=0
	    break
	fi
	${dryrun} sleep 10
	_total=`expr ${_total} + 10`
    done

    return ${_rc}
}


cleanup()
{
    for item in $locklist ; do
	if [ -f "$item" ]; then
	    rm -f "$item" 1>/dev/null 2>/dev/null
	fi
    done

    if [ ${killprocesses} = true ]; then
	${dryrun} kill_my_globus_processes
    fi
}

    
get_lock()
{
    lockprefix="$1"
    locklist="${locklist} ${lockprefix}.$$"
    ${dryrun} hostname > "${lockprefix}.$$"
    sed_rule="s%""${lockprefix}.""%%"

    while [ `ls -1rt ${lockprefix}* | sed -n -e "${sed_rule}" -e 1p` -ne $$ ]
    do
	${dryrun} sleep 300
    done
}


directory_name()
{
    # usaged:  directory_name <base> <ext>
    #
    # if <ext> is a relative path, echo back <base>/<ext>

    case "$2" in
    /*)
	echo "$2" | sed -e 's|//|/|g' -e 's|/$||g'
	;;
    *)
	echo "$1/$2" | sed -e 's|//|/|g' -e 's|/$||g'
	;;
    esac
}


setup_variables()
{
    [ -z "$arg_user"     ] && arg_user=`whoami`
    user=${arg_user}

    [ -z "$arg_domain"   ] && arg_domain=`hostname | \
	  awk -F. '{for (i=2;i<NF;++i) { printf "%s.", $i }; print $NF }'`
    domain=${arg_domain}

    [ -z "$arg_mailto"   ] && arg_mailto="${user}@${domain}"
    mailto=${arg_mailto}

    _sourcedir=/sandboxes/pitcairn.mcs.anl.gov/larsson

    [ -z "$arg_srchost"   ] && arg_srchost=pitcairn.mcs.anl.gov
    [ -z "$arg_globustar" ] && arg_globustar=${_sourcedir}/globus.tar
    [ -z "$arg_mpichtar"  ] && arg_mpichtar=${_sourcedir}/mpich.tar
    [ -z "$arg_scp"       ] && arg_scp=scp

    srchost=${arg_srchost}
    globustar=${arg_globustar}
    mpichtar=${arg_mpichtar}
    scp=${arg_scp}

    [ -z "$arg_jmtype"   ] && arg_jmtype="fork"
    jmtype=${arg_jmtype}

    [ -z "$arg_anchor"   ] && arg_anchor=/sandbox/${user}/nightly-build
    [ -z "$arg_globus"   ] && arg_globus=globus
    [ -z "$arg_mpich"    ] && arg_mpich=mpich
    [ -z "$arg_base"     ] && arg_base=
    [ -z "$arg_scratch"  ] && arg_scratch=
    [ -z "$arg_build"    ] && arg_build=BUILD
    [ -z "$arg_install"  ] && arg_install=install
    [ -z "$arg_deploy"   ] && arg_deploy=deploy

    anchor=`directory_name ${arg_anchor} ""`
    globus=`directory_name ${anchor} ${arg_globus}`
    mpich=`directory_name ${anchor} ${arg_mpich}`
    base=`directory_name ${anchor} ${arg_base}`
    scratch=`directory_name ${base} ${arg_scratch}`
    build=`directory_name ${base} ${arg_build}`
    install=`directory_name ${base} ${arg_install}`
    deploy=`directory_name ${base} ${arg_deploy}`

    [ -z "$arg_ssl32"    ] && arg_ssl32=/soft/pub/packages/SSLeay-0.9.0
    ssl32=`directory_name ${arg_ssl32} ""`

    [ -z "$arg_ssl64"    ] && arg_ssl64="../SSLeay-0.9.0-n64"
    ssl64=`directory_name ${arg_ssl32} ${arg_ssl64}`

    [ -z "$arg_ldap32"    ] && arg_ldap32=/soft/pub/packages/OpenLDAP-1.2.7
    ldap32=`directory_name ${arg_ldap32} ""`

    [ -z "$arg_ldap64"    ] && arg_ldap64="../OpenLDAP-1.2.7-n64"
    ldap64=`directory_name ${arg_ldap32} ${arg_ldap64}`

    case "${architecture}" in
	nt)
	    ps_command="ps xww"
	    conf_extraargs="--with-domainname=${domain} ${conf_extraargs}"
	    ;;
	aix|linux)
	    ps_command="ps xww"
	    ;;
	*)
	    ps_command="ps -u ${user}"
	    ;;
    esac
}


setup_mcs_paths()
{
    resoft=yes
    case "$architecture" in
    solaris*)
	PATH=/mcs/bin:/usr/local/bin:/bin:/usr/openwin/bin:/usr/sbin:/etc:/opt/SUNWspro/foo:/soft/gnu/bin:/soft/apps/bin:/opt/SUNWspro/bin:/usr/ccs/bin:/soft/com/packages/workshop-5.0/SUNWspro/bin:/usr/ucb:/soft/pub/bin:/soft/adm/bin:
	;;
    irix)
	PATH=/mcs/bin:/usr/local/bin:/usr/sbin:/usr/bsd:/sbin:/usr/bin:/etc:/usr/etc:/usr/bin/X11:/soft/com/bin:/soft/apps/bin:/soft/pub/bin:/soft/adm/bin:
	;;
    aix)
	PATH=/usr/bin:/etc/:/usr/sbin:/usr/ucb:/usr/bin/X11:/sbin:/usr/local/bin:/usr/lpp/ssp/bin:/usr/lpp/LoadL/nfs/bin:/home/larsson/bin:/soft/pub/bin:/soft/adm/bin:
	export PATH
	resoft=no
	;;
    linux)
	PATH=/homes/larsson/gnu/linux/bin:/soft/adm/bin:/soft/apps/packages/gcc-2.95.1/bin:/mcs/bin:/usr/local/bin:/bin:/usr/sbin:/etc:/usr/X11R6/bin:/usr/bin:/sbin:/soft/apps/bin:/homes/larsson/bin
	;;
    nt)
	resoft=no
	;;
    esac

    if [ "$resoft" = yes ]; then
	. $HOME/.software-cache.sh
	export PATH
    fi
}


do_gatekeeper() {

    ${verbose} "in do_gatekeeper..."
    ${verbose} "contact = $contact"
    ${verbose} "tools = $tools"
    ${verbose} "deploy =$deploy"
    ${verbose} "scratch =$scratch"

    if [ -n "$contact" ]; then
	${tools}/bin/globusrun -a -r "$contact" \
	    1>${scratch}/auth.log 2>&1
	xxx=$?
	if [ $xxx -ne 0 ]; then
	    report_error ${scratch}/auth.log "authentication problems to contact $contact, tools=$tools"
	fi
	return
    fi

    for file in x gridmap serv.conf gk.conf jm.conf gatekeeper.log ; do
	rm -f ${scratch}/${file}
    done

    ${tools}/bin/grid-proxy-info -exists -hours 4 \
	1>${scratch}/proxy.log 2>&1
    if [ $? -ne 0 ]; then
	report_error ${scratch}/proxy.log "no valid proxy"
    fi

    subject=`${tools}/bin/grid-cert-info -subject`

    cat > ${scratch}/gridmap <<EOF
"${subject}" ${user}
EOF

    cat > ${scratch}/serv.conf <<EOF
jobmanager stderr_log - globus-jobmanager globus-jobmanager -conf ${scratch}/jm.conf
EOF

    cat > ${scratch}/gk.conf <<EOF
-home ${deploy}
-e ${deploy}/libexec
-grid_services ${scratch}/serv.conf
-gridmap ${scratch}/gridmap
-d
-l ${scratch}/gatekeeper.log
EOF

    gram_dn=`echo "${subject}" | \
      awk -F/ '{ for (i=NF;i>2;--i) printf "%s, ", $i; printf "%s\n", $2 ; }'`

    host=`${deploy}/bin/globus-hostname`
    host_info=`${deploy}/libexec/grid-info-osguess`
    host_cputype=`echo $host_info | awk '{print $1}'`
    host_manufacturer=`echo $host_info | awk '{print $2}'`
    host_osname=`echo $host_info | awk '{print $3}'`
    host_osversion=`echo $host_info | awk '{print $4}'`

    cat > ${scratch}/jm.conf <<EOF
-type ${jmtype}
-home ${deploy}
-e ${deploy}/libexec
-rdn blabla
-globus-gatekeeper-subject "${subject}"
-globus-host-dn "hn=${host}, ${mdsorg}"
-globus-host-cputype ${host_cputype}
-globus-host-manufacturer ${host_manufacturer}
-globus-host-osname ${host_osname}
-globus-host-osversion ${host_osversion}
-save-logfile on_error
EOF

    eval ${ps_command} > ${scratch}/x
    total=0
    while grep globus-g ${scratch}/x > /dev/null ; do
	${verbose} "another gatekeeper is most likely running already..."
	${verbose} "my cleanup may interfer, so I'll wait."
	sleep 120
	total=`expr $total + 120`
	if [ $total -ge 7200 ]; then
	    report_error ${scratch}/x \
		"globus processes exists after 2 hours, deploydir=${deploy}"
	fi
	rm -f ${scratch}/x
	eval ${ps_command} > ${scratch}/x
    done
    rm -f ${scratch}/x
    rm -f ${deploy}/etc/globus-nologin
    rm -f ${deploy}/var/globus-nologin

    eval "(${deploy}/sbin/globus-gatekeeper -c ${scratch}/gk.conf) &"
    x=$?

    killprocesses=true

    if [ $x -ne 0 ]; then
	report_error /dev/null "gatekeeper problem, deploydir=${deploy}"
    fi

    contact=
    count=0
    while [ ${count} -lt 60 -a -z "${contact}" ]; do
	sleep 1
	contact="`grep GRAM ${scratch}/gatekeeper.log | sed 's/^.*:.*: //'`"
	count=`expr ${count} + 1`
    done

    if [ -z "${contact}" ]; then
	report_error ${scratch}/gatekeeper.log "gatekeeper startup problem"
    fi

    (
	${tools}/bin/globusrun -a -r "${contact}" 1>${scratch}/auth.log 2>&1
	if [ $? -ne 0 ]; then
	    report_error ${scratch}/auth.log "globusrun -a problem"
	fi

	${tools}/bin/globusrun -dryrun -r "${contact}" \
	    "&(executable=/bin/ls)" 1>${scratch}/dryrun.log 2>&1
	if [ $? -ne 0 ]; then
	    touch ${scratch}/failure
	    report_error ${scratch}/dryrun.log "globusrun -dryrun problem"
	fi

	${tools}/bin/globusrun -o -r "${contact}" "&(executable=/bin/ls)" \
	    1>${scratch}/withgass.log 2>&1
	if [ $? -ne 0 ]; then
	    touch ${scratch}/failure
	    report_error ${scratch}/withgass.log "globusrun -o problem"
	fi
    ) &

    pid=$!
    wait_or_timeout ${pid} 300
    if [ -f ${scratch}/failure ]; then
	exit 1
    fi
}


do_clean()
{
    ${verbose} "in do_clean..."
    ${verbose} "base = $base"
    if [ -d ${base} ]; then
	parent=`dirname ${base}`
	${dryrun} mv ${base} ${parent}/tmp-trash
	${dryrun} rm -rf ${parent}/tmp-trash &
    fi
    ${dryrun} mkdir -p ${base}
}


do_setup()
{
    ${verbose} "in do_setup..."
    ${verbose} "argument = $1"
    ${verbose} "globus = $globus"
    ${verbose} "globustar = $globustar"
    ${verbose} "mpich = $mpich"
    ${verbose} "mpichtar = $mpichtar"

    _software="$1"
    case "${_software}" in
	globus)
	    _destdir="$globus"
	    _srcfile="$globustar"
	    ;;
	mpich)
	    _destdir="$mpich"
	    _srcfile="$mpichtar"
	    ;;
	*)
	    report_error /dev/null "do_setup: strange software ${_software}"
	    ;;
    esac

    ${verbose} "software = ${_software}"
    ${verbose} "destdir  = ${_destdir}"
    ${verbose} "srchost  = ${srchost}"
    ${verbose} "srcfile  = ${_srcfile}"
    ${dryrun} cd `dirname "${_destdir}"`
    ${dryrun} rm -rf "${_software}"

    if [ "${srchost}" = local ]; then
	_tarfile="${_srcfile}"
	_remove=false
    else
	# don't redirect output as this command probably has some
	# user interaction...
	${dryrun} ${scp} "${srchost}:${_srcfile}" tmp.tar
	if [ $? -ne 0 ]; then
	    report_error /dev/null \
		"error transferring file ${_srcfile} from ${srchost}"
	fi
	_tarfile=tmp.tar
	_remove=true
    fi	

    ${dryrun} tar xf ${_tarfile} 1>${scratch}/tar.log 2>&1
    xxx=$?
    
    if [ ${_remove} = true -a -r ${_tarfile} ]; then
	${dryrun} rm -f ${_tarfile}
    fi
    if [ $xxx -ne 0 ]; then
	report_error ${scratch}/tar.log "tar failed for ${_software}" 
    fi
}


do_configure_globus()
{
    ${verbose} "in do_configure_globus..."
    ${verbose} "build=${build}"
    ${verbose} "globus=${globus}"
    ${verbose} "install=${install}"

    if [ -n "${conf_64bits}" ]; then
	sslpath="${ssl64}"
	ldappath="${ldap64}"
    else
	sslpath="${ssl32}"
	ldappath="${ldap32}"
    fi
    ${verbose} "sslpath=${sslpath}"
    ${verbose} "ldappath=${ldappath}"

    ${dryrun} rm -rf ${build}
    ${dryrun} mkdir -p ${build}
    ${dryrun} cd ${build}

    if [ -z "${conf_extraargs}" ]; then
	${dryrun} ${globus}/configure --prefix=${install} \
	    --with-ssl-path=${sslpath} --with-ldap-path=${ldappath} \
	    --with-security-tests ${conf_debug} ${conf_threads} \
	    ${conf_64bits} \
	    1>${scratch}/configure.log 2>&1
	xxx=$?
    else
	${dryrun} ${globus}/configure --prefix=${install} \
	    --with-ssl-path=${sslpath} --with-ldap-path=${ldappath} \
	    --with-security-tests ${conf_debug} ${conf_threads} \
	    ${conf_64bits} "${conf_extraargs}" \
	    1>${scratch}/configure.log 2>&1
	xxx=$?
    fi
    if [ 0 -ne $xxx ]; then
	report_error ${scratch}/configure.log \
	    "configure failed for ${conf_debug} ${conf_64bits} ${conf_threads}"
    fi

    ${dryrun} make 1>${scratch}/make.log 2>&1
    xxx=$?
    if [ 0 -ne $xxx ]; then
	report_error ${scratch}/make.log
	    "make failed for ${conf_debug} ${conf_64bits} ${conf_threads}"
    fi

    if [ -n "${conf_threads}" ]; then
	${dryrun} Miscellaneous/common/tests/globus_thread_test 1 2 10 \
	    1>${scratch}/threadtest.log  2>&1
	xxx=$?	    
	if [ 0 -ne $xxx ]; then
	    report_error ${scratch}/threadtest.log \
	"threads test failed for ${conf_debug} ${conf_64bits} ${conf_threads}"
	fi
    fi

    ${dryrun} make install 1>${scratch}/makeinstall.log 2>&1
    xxx=$?
    if [ 0 -ne $xxx ]; then
	report_error ${scratch}/makeinstall.log \
  "make install failed failed for ${conf_debug} ${conf_64bits} ${conf_threads}"
    fi
}


do_test()
{
    ${verbose} "in do_test..."
    ${verbose} "build   = $build"
    ${verbose} "tools   = $tools"
    ${verbose} "contact = $contact"

    if [ -n "${supress_test}" ]; then
	return 0
    fi

    rsl=
    ${dryrun} ${tools}/bin/grid-proxy-info -exists -hours 4 1>${scratch}/proxytest.log 2>&1
    if [ $? -ne 0 ]; then
	report_error ${scratch}/proxytest.log "no valid proxy"
    fi

    case "`echo ${build} | sed -e 's|/$||' -e 's|.*/||'`" in
	*mpi* | *mpl* | *inx*)
	    rsl="(jobtype=mpi)"
	    startup="mpirun -np 1"
	    ;;
    esac

    DIRS="  Communication/nexus/tests/miscellaneous \
	    FileAccess/gass/tests/cache  \
	    Miscellaneous/common/tests   \
	    Security/tests"

    globusrun=${tools}/bin/globusrun

    for dir in $DIRS ; do
	subdir=${build}/${dir}
	if [ ! -d ${subdir} ]; then
	    echo "WARNING: no directory ${subdir}"
	else
	    cd ${subdir}
	    echo "running tests in ${subdir}"
	    if [ ${dir} = FileAccess/gass/tests/cache ]; then
		${dryrun} ./run-tests "${startup}" &
		pid=$!
	    else		
		${dryrun} ./run-tests ${globusrun} "${contact}" "${rsl}" &
		pid=$!
	    fi
	    ${dryrun} wait_or_timeout ${pid} 900
	    xxx=$?
	    if [ ${xxx} -ne 0 ]; then
		kill -9 ${pid}
		echo '*****'
		echo '*****'
		echo '*****'
		echo '*****    WARNING : test in '${subdir}' most likely hung'
		echo '*****'
		echo '*****'
		echo '*****'
	    fi
	fi
    done
}


do_install()
{
    ${verbose} "in do_install..."
    ${verbose} "globus = $globus"
    ${verbose} "build = $build"
    ${verbose} "install = $install"

    if [ -n "${conf_64bits}" ]; then
	sslpath="${ssl64}"
	ldappath="${ldap64}"
    else
	sslpath="${ssl32}"
	ldappath="${ldap32}"
    fi

    if [ ! -d ${build} ]; then
	${dryrun} mkdir -p ${build}
    fi
    ${dryrun} cd ${build}
    ${dryrun} rm -rf ${build}/globus-build

    if [ -z "${conf_extraargs}" ]; then
	${dryrun} eval ${globus}/globus-install -prefix=${install} \
		-with-ssl-path=${sslpath} -with-ldap-path=${ldappath} \
		--with-umask=022 --builddirs-persist --development-tests \
		--with-shm ${conf_64bits} ${conf_debug} ${conf_inparallel} \
		-- --with-security-tests \
		1>${scratch}/build.log 2>&1
	xxx=$?
    else
	${dryrun} eval ${globus}/globus-install -prefix=${install} \
		-with-ssl-path=${sslpath} -with-ldap-path=${ldappath} \
		-with-umask=022 -builddirs-persist -development-tests \
		-with-shm ${conf_64bits} ${conf_debug} ${conf_inparallel} \
		-- --with-security-tests "\"${conf_extraargs}\"" \
		1>${scratch}/build.log 2>&1
	xxx=$?
    fi
    if [ 0 -ne $xxx ]; then
	report_error ${scratch}/build.log \
	    "globus-install ${conf_64bits} ${conf_debug} failed"
    fi
}


do_local_deploy()
{
    ${verbose} "in do_local_deploy..."
    ${verbose} "install = $install"
    ${verbose} "deploy  = $deploy"
    ${verbose} "extra_services = $extra_services"

    GLOBUS_INSTALL_PATH="${install}"
    export GLOBUS_INSTALL_PATH

    if [ -z "${mdsorg}" ]; then
	_toolsbin=`${install}/bin/globus-tools-path -bindir 2>/dev/null`
	if [ -n "${_toolsbin}" ]; then
	    mdsorg=`${_toolsbin}/grid-proxy-info -subject | \
		sed 's|/CN=.*$||' | \
		awk -F/ '{ for (i=NF;i>2;--i) printf "%s, ", $i;
			   printf "%s\n", $2 ; }'`
        else
	    report_error /dev/null \
		"local_deploy ${conf_64bits} ${conf_debug} failed (MDS org)"
	fi
    fi

    ${verbose} "mdsorg  = $mdsorg"
    ${verbose} "mdspass = $mdspasswd"

    #
    # step 1: run globus-setup
    # 
    cd ${install}
    cat >setup_input <<EOF
y
3
${mdsorg}
5
${mdspasswd}
${mdspasswd}
6
7
8
y
s
q
EOF

    ${dryrun} ./sbin/globus-setup <setup_input 1>${scratch}/setup.log 2>&1
    xxx=$?
    if [ 0 -ne $xxx ]; then
	report_error ${scratch}/setup.log \
	    "globus-setup failed for ${conf_64bits} ${conf_debug}"
    fi

    ${dryrun} rm -rf "${deploy}"
    ${dryrun} mkdir -p "${deploy}"

    ${dryrun}./sbin/globus-local-deploy -force "${deploy}" \
	1>${scratch}/deploy.log 2>&1
    xxx=$?
    if [ 0 -ne $xxx ] ; then
	report_error ${scratch}/deploy.log \
	    "globus-local-deploy failed for ${conf_64bits} ${conf_debug}"
    fi



}


do_configure_mpich()
{
    ${verbose} "in do_configure_mpich..."
    ${verbose} "mpich   = $mpich"
    ${verbose} "install = $install"

    GLOBUS_INSTALL_PATH=${install}
    export GLOBUS_INSTALL_PATH

    [ -z "$flavor" ] && flavor=`${install}/bin/globus-development-path`
    [ -z "$flavor" ] && report_error /dev/null "flavor error in ${install}"
    ${verbose} "flavor = $flavor"

    ${dryrun} cd "$mpich"

    ${dryrun} ./configure -device=globus -globusdir="${flavor}" -noromio \
	-no_short_longs 1>${scratch}/configure.log 2>&1

    if [ $? -ne 0 ]; then	
	report_error ${scratch}/configure.log "error configuring MPICH"
    fi

    ${dryrun} make 1>${scratch}/make.log 2>&1
    if [ $? -ne 0 ]; then	
	report_error ${scratch}/make.log "error making MPICH"
    fi
}


do_mpich_test_suite ()
{
    ${verbose} "in do_mpich_test_suite..."
    ${verbose} "mpich   = $mpich"
    ${verbose} "install = $install"
    ${verbose} "contact = $contact"

    GLOBUS_INSTALL_PATH=${install}
    export GLOBUS_INSTALL_PATH

    _dir=`ls -1d ${mpich}/build/*/globus/bin`
    ${dryrun} echo "\"${contact}\" 2" > ${_dir}/machines
    ${dryrun} cd "${mpich}/examples/test"

    ${dryrun} make default 1>${scratch}/maketest.log 2>&1
    if [ $? -ne 0 ] ; then
	report_error ${scratch}/maketest.log \
	    "MPICH failed to compile its test programs, dir=$mpich"
    fi

    ${dryrun} make testing_save 1>${scratch}/test.log 2>&1 &
    pid=$!
    wait_or_timeout $pid 18000   # 18K sec = 5 hours
    if [ $? -ne 0 ]; then
	kill -9 $pid	    
	report_error ${scratch}/test.log \
	    "MPICH test suite most likely hung, dir = $mpich"
    fi
    ${verbose} "MPICH test suite done"
}


#########################################################################


if [ $# -lt 1 ]; then
    echo "error: missing architecture. type -help for full usage"
    exit 1
fi

if [ "$1" = "-help" ] ; then
    usage
    exit 0
fi

all_args="$@"

case "$1" in
    -*)
	echo "architecture is missing"
	exit 1
	;;
    *)
	architecture="$1"
	;;
esac
shift

# default settings
#
dryrun=
nomail=false
user=
domain=
mailto=
mdsorg=
mdspasswd=
srchost=
srcdir=
scp=
globustar=
mpichtar=
anchor=
globus=
mpich=
base=
scratch=
build=
install=
deploy=
flavor=
tools=
ssl32=
ssl64=
ldap32=
ldap64=
contact=
jmtype=
conf_threads=
conf_64bits=
conf_debug="--enable-debug"
conf_inparallel="--disable-parallel-build"
got_anchor_lock=false
locklist=
killprocesses=false

trap cleanup 0 1 2 3 6 9 12 15

while [ -n "$1" ]; do
    case "$1" in
    -help)
	usage
	exit 0
	;;
    -dryrun)
	dryrun=echo
	dryrun_loop=yes
	if [ ${dryrun_loop} = yes ]; then	
	    sed -e 's|>\&|\\>\\\&|g' -e 's/>>/\\>\\>/g' \
		-e 's|1>|1\\>|g' -e 's/dryrun_loop=yes/dryrun_loop=no/' \
		$0 > .tmp$$
	    chmod +x .tmp$$
	    .tmp$$ ${all_args}
	    status=$?
	    rm -f .tmp$$
	    exit $status
	fi
	;;
    -noemail|-nomail)
	nomail=true
	;;
    -user|-domain|-mdsorg|-mdspasswd|-anchor|-ssl32|-ssl64|-ldap32|-ldap64| \
    -contact|-scratch|-base|-globus|-mpich|-build|-install|-deploy|-tools|  \
    -flavor|-srchost|-scp|-globustar|-mpichtar|-jmtype)
	var=`echo "$1" | sed 's/^-//'`
	eval "arg_${var}=\"$2\""
	eval "${var}=\"$2\""
	shift
	;;
    -reset-base)
	for var in scratch build install deploy tools flavor ; do
	    eval "arg_${var}="
	done
	;;
    -inparallel)
	conf_inparallel="--enable-parallel-build"
	;;
    -lock)
	locklist="${locklist} $2"
	got_anchor_lock=true
	shift
	;;
    -nolock)
	got_anchor_lock=true
	;;
    -64)
	conf_64bits="--enable-64bit"
	;;
    -nodebug)
	conf_debug=
	;;
    -threads)
	conf_threads="--with-threads=$2"
	shift
	;;
    -extraargs)
	conf_extraargs="$2"
	shift
	;;
    -full*)
	#
	case "$1" in
	    -full-setup)
		opts="-clean -setup-globus -setup-mpich"
		;;
	    -full-basic)
		opts="-base basic -clean -reset-base -configure \
		    -deploy install -test"
		;;
	    -full-threaded)
		opts="-base threaded -clean -reset-base -threads pthreads \
		    -configure -deploy ../basic/install -test"
		;;		    
	    -full-32bit)
		opts="-base 32bit -clean -reset-base -build . -globus-install \
		      -local-deploy -flavor-tests"
		;;
	    -full-mpich-32)
		opts="-base 32bit -reset-base -make-mpich -run-mpich"
		;;
	    -full-64bit)
		opts="-base 64bit -clean -reset-base -64 -build . \
		      -globus-install -local-deploy -flavor-tests"
		;;
	    -full-mpich-64)
		opts="-base 64bit -reset-base -make-mpich -run-mpich"
		;;
	    -full)
		opts="-full-setup -full-basic -full-threaded -full-32bit \
		    -full-mpich-32"
		if [ ${architecture} = irix ]; then
		    opts="${opts} -full-64bit -full-mpich-64"
		fi
		;;
	    *)
		echo "option $1 not yet supported or understood"
		exit 1
		;;
	esac
	# one argument will be shifted away at the end of the big while
	# loop... but we need to do it here in order get the rest of the
	# argument line, so replace $1 with a noop.
	shift
	eval set -- "noop ${opts} `stringify_arguments $@`"
	;;
    -mcscron)
	setup_mcs_paths
	shift
	eval set -- "noop -srchost local -scp globus-rcp \
		    `stringify_arguments $@`"
	;;
    -*)
	setup_variables
	if [ ${got_anchor_lock} = false ]; then
	    get_lock "${anchor}.queue"
	    got_anchor_lock=true
	fi
	case "$1" in
	-lock-anchor)
	    echo "remove lock file when you are done: ${anchor}.queue.$$"
	    locklist=
	    exit 0
	    ;;
	-clean)
	    do_clean
	    ;;
	-setup-globus)
	    do_setup "globus"
	    ;;
	-setup-mpich)
	    do_setup "mpich"
	    ;;
	-configure)
	    do_configure_globus
	    ;;
	-make-mpich)
	    ${dryrun} do_configure_mpich
	    ;;
	-test|-flavor-tests|-gatekeeper|-run-mpich)
	    GLOBUS_INSTALL_PATH="${install}"
	    export GLOBUS_INSTALL_PATH
	    [ -z "$tools" ] && tools=`${install}/bin/globus-tools-path`
	    [ -z "$tools" ] && report_error /dev/null "tools error in $install"
	    ${dryrun} do_gatekeeper
	    case "$1" in
	    -test)
		do_test
		;;
	    -flavor-tests)
		_build=${build}
		for _flavor in `ls -1 ${_build}/globus-build/development/ | \
				grep -v '.out.log'`
		do
		    build=${_build}/globus-build/development/${_flavor}
		    do_test
		done
		;;
	    -gatekeeper)
		killprocesses=false
		;;
	    -run-mpich)
		${dryrun} do_mpich_test_suite
		;;
	    esac
	    GLOBUS_INSTALL_PATH=
	    export GLOBUS_INSTALL_PATH
	    ;;
	-globus-install)
	    do_install
	    ;;
	-local-deploy)
	    ${dryrun} do_local_deploy
	    ;;
	*)
	    echo "option $1 not yet supported"
	    exit 1
	    ;;
	esac
	;;
    *)
	echo "don't know what \"$1\" is -- skipping"
	;;
    esac
    shift
done

exit 0

