#! /bin/bash
#
# Adapted version of xce_remap_inidata for Mistral
# Created by Guido Cioni (guido.cioni@mpimet.mpg.de)
#-----------------------------------------------------------------------------
#SBATCH --account=bm0682
#SBATCH --job-name=inidata.run
#SBATCH --partition=compute2
#SBATCH --nodes=1
#SBATCH --threads-per-core=2
#SBATCH --output=LOG.inidata.run.%j.o
#SBATCH --error=LOG.inidata.run.%j.o
#SBATCH --exclusive
#SBATCH --time=00:30:00
#=============================================================================
set -x

#---------------------------------------------------------------------------
# openmp environment variables
# ----------------------------
export OMP_NUM_THREADS=6 # 6 on compute2
export ICON_THREADS=6    # 6 on compute2
export OMP_SCHEDULE=dynamic,1
export OMP_DYNAMIC="false"
export OMP_STACKSIZE=4096M
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.3-gcc62/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/netcdf/netcdf_c-4.4.1.1-gcc48/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/hdf5/hdf5-1.8.14-gcc48/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/grib_api/grib_api-1.15.0-gcc48/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/eccodes/eccodes-2.5.0-gcc48/lib/
#-----------------------------------------------------------------------------
# MPI variables
# ----------------------------
mpi_root=/sw/rhel6-x64/intel/impi/5.0.3.048/lib64
no_of_nodes=${SLURM_JOB_NUM_NODES:-1}
mpi_procs_pernode=6 # 6 or 12 (hyperthreading) on compute2
mpi_total_procs=$((no_of_nodes * mpi_procs_pernode))
START="srun --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks-per-node=${mpi_procs_pernode}"

# load modules
module purge
module load intel gcc 
module list

ulimit -s unlimited

# SETTINGS: DIRECTORIES AND INPUT/OUTPUT FILE NAMES --------------------------

# directory containing DWD icon tools binaries
ICONTOOLS_DIR=/work/mh0731/m300382/dwd_icon_tools_git/icontools

# file name of input grid
INGRID=/work/mh0731/m300382/mesovortices/data/ifs_oper_T1279_2014022400.grb

# file name of limited-area (output) grid
LOCALGRID=/work/mh0731/m300382/mesovortices/grid_NA_2492m.nc                      

# directory containing data files which shall be mapped to limited-area grid
DATADIR=/work/mh0731/m300382/mesovortices/data/
DATAFILELIST=$(find ${DATADIR}/ifs_oper_T1279_2014022400.grb.nc)

# output directory for extracted boundary data
OUTDIR=/work/mh0731/m300382/mesovortices/

mkdir -p $OUTDIR
#-----------------------------------------------------------------------------

BINARY_ICONSUB=iconsub_mpi
BINARY_REMAP=iconremap_mpi
AUXGRID=auxgrid
#-----------------------------------------------------------------------------
# Remap inital data onto local (limited-area) grid
#-----------------------------------------------------------------------------

cd ${OUTDIR}


set +x
cat >> NAMELIST_ICONREMAP_FIELDS << EOF_2A
&input_field_nml
 inputname      = "u"         
 outputname     = "U"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "v"         
 outputname     = "V"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "w"         
 outputname     = "W"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "t"         
 outputname     = "T"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "q"         
 outputname     = "QV"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "clwc"         
 outputname     = "QC"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "ciwc"         
 outputname     = "QI"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "crwc"         
 outputname     = "QR"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "cswc"         
 outputname     = "QS"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "z_2"         
 outputname     = "GEOP_SFC"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "z"         
 outputname     = "GEOP_ML"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "tsn"         
 outputname     = "T_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "sd"         
 outputname     = "W_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "rsn"         
 outputname     = "RHO_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "asn"         
 outputname     = "ALB_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "skt"         
 outputname     = "SKT"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "sst"         
 outputname     = "SST"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "stl1"         
 outputname     = "STL1"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "stl2"         
 outputname     = "STL2"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "stl3"         
 outputname     = "STL3"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "stl4"         
 outputname     = "STL4"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "swvl1"         
 outputname     = "SMIL1"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "swvl2"         
 outputname     = "SMIL2"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "swvl3"         
 outputname     = "SMIL3"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "swvl4"         
 outputname     = "SMIL4"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "ci"         
 outputname     = "CI"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "src"         
 outputname     = "W_I"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "SR"         
 outputname     = "Z0"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "lsm"         
 outputname     = "LSM"          
 intp_method    = 3
/
&input_field_nml 
 inputname      = "lnsp"
 outputname     = "LNPS" 
 intp_method    = 3     
/
EOF_2A

set -x
cat NAMELIST_ICONREMAP_FIELDS

#-----------------------------------------------------------------------------
# loop over file list:

echo ${DATAFILELIST}
for datafilename in ${DATAFILELIST} ; do

datafile="${datafilename##*/}"  # get filename without path
outdatafile=${datafile%.*}      # get filename without suffix

cat > NAMELIST_ICONREMAP << EOF_2B
&remap_nml
 in_grid_filename  = '${INGRID}'
 in_filename       = '${DATADIR}/${datafile}'
 in_type           = 1
 out_grid_filename = '${LOCALGRID}'
 out_filename      = '${OUTDIR}/${outdatafile}.nc'
 out_type          = 2
 out_filetype      = 5
 l_have3dbuffer    = .false.
 ncstorage_file    = "ncstorage.tmp"
/
EOF_2B

${START}  ${ICONTOOLS_DIR}/${BINARY_REMAP} \
            --remap_nml NAMELIST_ICONREMAP                                  \
            --input_field_nml NAMELIST_ICONREMAP_FIELDS 2>&1

done

#-----------------------------------------------------------------------------
# clean-up

rm -f ncstorage.tmp*
rm -f nml.log  NAMELIST_SUB NAMELIST_ICONREMAP NAMELIST_ICONREMAP_FIELDS

#-----------------------------------------------------------------------------
exit
#-----------------------------------------------------------------------------