#! /bin/csh
#
# Job script to build and execute the model on a 64-bit Linux desktop machine.
#
# User should set shell variables below (paths below are examples on the hao network)
#
#   modeldir:  Root directory to model source (may be an SVN working dir)
#   execdir:   Directory in which to build and execute (will be created if necessary)
#   tgcmdata:  Directory in which startup history and data files are accessed.
#              (If tgcmdata is not set, the model will use env var TGCMDATA)
#   input:     Namelist input file for the chosen model resolution
#   output:    Stdout file from model execution (will be created)
#   modelres:  Model resolution (5.0 or 2.5 degrees)
#   make:      Build file with platform-specific compile parameters (in scripts dir)
#   mpi:       TRUE/FALSE for MPI run (non-MPI runs are not supported in v2.0 and later)
#   debug:     If TRUE, build and execute a "debug" run (debug compiler flags are set)
#   exec:      If TRUE, execute the model (build only if exec is FALSE)
#   utildir:   Directory containing supporting scripts (default $modeldir/scripts)
#   runscript: LSF script with run commands (submitted with bsub from execdir)
#
# To switch to 2.5-deg resolution, set modelres below to 2.5,
# and change execdir, tgcmdata and namelist input if necessary.
#
set modeldir = tiegcm_trunk
set execdir  = /hao/aim/$user/tiegcm_trunk/tiegcm.exec
set tgcmdata = /hao/aim/tgcm/data/tiegcm2.0
set input    = $modeldir/scripts/tiegcm_res5.0_default.inp
set output   = tiegcm.out
set make     = Make.intel_hao64
set modelres = 5.0
set mpi      = TRUE  # must be TRUE for tiegcm2.0 and later
set nproc    = 4     # nproc=1 is permitted
set debug    = FALSE
set exec     = TRUE
set utildir  = $modeldir/scripts
# 
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#                        Shell Script for TIEGCM Linux job
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#
set mycwd = `pwd`
echo "" ; echo "${0}:"
echo "  Begin execution at `date`"
echo "  Current working directory: $mycwd"
echo "  System: `uname -a`"  
echo ""
#
# Verify directories and make_machine file (make execdir if necessary).
# Get absolute path for dirs that are accessed from the execdir.
#
if (! -d $modeldir) then
  echo ">>> Cannot find model directory $modeldir <<<"
  exit 1
endif
set model = $modeldir:t

if ($model == '') then
  echo "Please remove trailing slash from modeldir $modeldir"
  exit
endif

if (! -d $utildir) then
  echo ">>> Cannot find model scripts directory $utildir <<<" 
  exit 1
endif

set srcdir  = $modeldir/src
if (! -d $srcdir) then
  echo ">>> Cannot find model source directory $srcdir <<<"
  exit 1
endif
set srcdir = `perl $utildir/abspath $srcdir`

if (! -d $execdir) then
  echo "Making exec directory $execdir" 
  mkdir -p $execdir
endif
#
# Set data directory:
#   If tgcmdata is set above, use it to set (override) env var TGCMDATA
#   If tgcmdata not set above, use env var TGCMDATA
#   If neither are set, print a warning and set both to cwd.
#   Finally, print warning if the directory does not exist.
#
if ($?tgcmdata) then
  set tgcmdata = `perl $utildir/abspath $tgcmdata`
  setenv TGCMDATA $tgcmdata
  echo Set env var TGCMDATA = $TGCMDATA
else
  if ($?TGCMDATA) then
    set tgcmdata = $TGCMDATA
    echo Set tgcmdata = $TGCMDATA
  else
    echo WARNING: TGCMDATA is not set - using cwd
    set tgcmdata = .
    setenv TGCMDATA $tgcmdata
  endif
endif
if (! -d $tgcmdata) echo ">>> Cannot find data directory $tgcmdata"

if ($modelres != 5.0 && $modelres != 2.5) then
  echo ">>> Unknown model resolution $modelres <<<"
  exit 1
endif
#
# Copy make files to execdir if necessary:
#
if (-f $make) then # set by user above
  cp $make $execdir
  echo Copied $make to $execdir
else
  cp $utildir/$make $execdir
  echo Copied $utildir/$make to $execdir
endif
if (! -f $execdir/Makefile)  cp $utildir/Makefile $execdir
if (! -f $execdir/mkdepends) cp $utildir/mkdepends $execdir
#
# Namelist input file must exist:
#
if (! -f $input) then
  echo ">>> Cannot find namelist input file $input <<<"
  exit 1
endif
set input  = `perl $utildir/abspath $input`
set output = `perl $utildir/abspath $output`
set util   = `perl $utildir/abspath $utildir`
set mklogs = $util/mklogs
set rmbinchars = $util/rmbinchars # Nov, 2015: remove non-ascii chars from stdout files
#
# Report to stdout:
#
set svn_revision = `svnversion $modeldir` || set svn_revision = "[none]"
echo -n "  Model directory:   $modeldir" && echo " (SVN revision $svn_revision)"
echo "  Exec directory:    $execdir"
echo "  Source directory:  $srcdir"
echo "  Data directory:    $tgcmdata"
echo "  Make machine file: $make"
echo "  Namelist input:    $input"
echo "  Model resolution:  $modelres"
echo "  Debug flag:        $debug"
#
# If debug flag has changed from last gmake, clean execdir
# and reset debug file:
#
if (-f $execdir/debug) then
  set lastdebug = `cat $execdir/debug` 
  if ($lastdebug != $debug) then
    echo "Clean execdir $execdir because debug flag switched from $lastdebug to $debug"
    set mycwd = `pwd` ; cd $execdir ; gmake clean ; cd $mycwd
    echo $debug >! $execdir/debug
  endif
else
  echo $debug >! $execdir/debug
  echo "Created file debug with debug flag = $debug"
endif
#
# Copy defs header file to execdir, if necessary, according to 
# requested resolution. This should seamlessly switch between
# resolutions according to $modelres.
#
set defs = $srcdir/defs5.0
if ($modelres == 2.5) set defs = $srcdir/defs2.5
if (-f $execdir/defs.h) then
  cmp -s $execdir/defs.h $defs
  if ($status == 1) then # files differ -> switch resolutions
    echo "Switching defs.h for model resolution $modelres"
    cp $defs $execdir/defs.h
  else
    echo "defs.h already set for model resolution $modelres"
  endif 
else # defs.h does not exist in execdir -> copy appropriate defs file
  echo "Copying $defs to $execdir/defs.h for resolution $modelres"
  cp $defs $execdir/defs.h
endif
#
# cd to execdir and run make:
#
cd $execdir || echo ">>> Cannot cd to execdir $execdir" && exit 1
echo ""
echo "Begin building $model in `pwd`"
#
# Build Make.env file in exec dir, containing needed env vars for Makefile:
#
cat << EOF >! Make.env
MAKE_MACHINE  = $make
DIRS          = . $srcdir 
MPI           = $mpi
NPROC         = $nproc
EXECNAME      = $model
NAMELIST      = $input
OUTPUT        = $output
DEBUG         = $debug
SVN_REVISION  = $svn_revision
EOF
#
gmake machines.ini
gmake mpirun.command
set mpirun = `cat mpirun.command`
#
set compiler = 'unknown'
if ($make == Make.intel_hao64) then
  set compiler = intel
else if ($make == Make.pgi_hao64) then
  set compiler = pgi
else if ($make == Make.gfort_hao64) then
  set compiler = gfort
endif
echo Compiler: $compiler
#
# Build the model:
gmake -j4 all || echo ">>> Error return from gmake all" && exit 1
#
# Execute Linux job (MPI or non-MPI run):
#
if ($exec == "TRUE") then
  set model = ./$model
  echo "$model output will go to $output"
  if ($mpi == "TRUE") then
    echo Executing $mpirun with executable $model at `date`
#
# Execute mpirun with the model executable ($model), and a single argument
# providing the path to the namelist input file:
#
    $mpirun -machinefile machines.ini -np $nproc $model $input >&! $output

    echo "Linux MPI run of $model completed at `date`"
    $rmbinchars $output # remove any non-ascii chars in stdout file
#
# Do not call mklogs if built with PGI or gfortran:
# (only intel has ability to prefix task ids to stdout lines)
#
    if ($compiler == 'intel') then
      $mklogs $output     # break stdout into per-task log files
    else
      echo "Am not calling mklogs because model was not built with intel"
    endif
  else # MPI is FALSE
    echo "Executing $model for Linux non-MPI run."
    $model < $input >&! $output || \
      echo ">>> ${0} Execution of $model FAILED at `date`" && \
      echo "See output in $output"
    echo "Linux non-MPI run of $model completed at `date`"
  endif
else
  echo "I am NOT executing $model (exec was not set)"
endif
exit 0