#! /bin/bash
#
# Adapted version of xce_remap_inidata for Mistral
# Created by Guido Cioni (guido.cioni@mpimet.mpg.de)
#-----------------------------------------------------------------------------
#SBATCH --account=bm0682
#SBATCH --job-name=inidata.run
#SBATCH --partition=compute2
#SBATCH --nodes=1
#SBATCH --threads-per-core=2
#SBATCH --output=LOG.inidata.run.%j.o
#SBATCH --error=LOG.inidata.run.%j.o
#SBATCH --exclusive
#SBATCH --time=00:30:00
#=============================================================================
set -x

#---------------------------------------------------------------------------
# openmp environment variables
# ----------------------------
export OMP_NUM_THREADS=6 # 6 on compute2
export ICON_THREADS=6    # 6 on compute2
export OMP_SCHEDULE=dynamic,1
export OMP_DYNAMIC="false"
export OMP_STACKSIZE=4096M
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/netcdf/netcdf_fortran-4.4.3-gcc62/lib/
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/netcdf/netcdf_c-4.4.1.1-gcc48/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/hdf5/hdf5-1.8.14-gcc48/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/sw/rhel6-x64/eccodes/eccodes-2.5.0-gcc48/lib/
#-----------------------------------------------------------------------------
# MPI variables
# ----------------------------
mpi_root=/sw/rhel6-x64/intel/impi/5.0.3.048/lib64
no_of_nodes=${SLURM_JOB_NUM_NODES:-1}
mpi_procs_pernode=6 # 6 or 12 (hyperthreading) on compute2
mpi_total_procs=$((no_of_nodes * mpi_procs_pernode))
START="srun --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks-per-node=${mpi_procs_pernode}"

# load modules
module purge
module load intel gcc 
module list

ulimit -s unlimited

# SETTINGS: DIRECTORIES AND INPUT/OUTPUT FILE NAMES --------------------------

# directory containing DWD icon tools binaries
ICONTOOLS_DIR=/work/mh0731/m300382/dwd_icon_tools_git/icontools

# file name of input grid
INGRID=/work/mh0731/m300382/icon_grid_0027_R03B08_N02.nc

# file name of limited-area (output) grid
LOCALGRID=/work/mh0731/m300382/test_nwp_new/italy_grid_nested_1231m.nc                      

# directory containing data files which shall be mapped to limited-area grid
DATADIR=/work/mh0731/m300382/test_nwp/data
DATAFILELIST=$(find ${DATADIR}/iefff00000000)

# output directory for extracted boundary data
OUTDIR=/work/mh0731/m300382/test_nwp_new/

mkdir -p $OUTDIR
#-----------------------------------------------------------------------------

BINARY_ICONSUB=iconsub_mpi
BINARY_REMAP=iconremap_mpi
AUXGRID=auxgrid
#-----------------------------------------------------------------------------
# Remap inital data onto local (limited-area) grid
#-----------------------------------------------------------------------------

cd ${OUTDIR}


set +x
cat >> NAMELIST_ICONREMAP_FIELDS << EOF_2A
&input_field_nml
 inputname      = "u"         
 outputname     = "U"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "v"         
 outputname     = "V"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "wz"         
 outputname     = "W"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "t"         
 outputname     = "T"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "q"         
 outputname     = "QV"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "clwmr"         
 outputname     = "QC"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "QI"         
 outputname     = "QI"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "rwmr"         
 outputname     = "QR"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "snmr"         
 outputname     = "QS"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "pres"         
 outputname     = "P"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "W_SO"         
 outputname     = "W_SO"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "T_SO"         
 outputname     = "T_SO"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "sd"         
 outputname     = "W_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "rsn"         
 outputname     = "RHO_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "T_SNOW"         
 outputname     = "T_SNOW"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "QV_S"         
 outputname     = "QV_S"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "T_G"         
 outputname     = "T_G"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "FRESHSNW"         
 outputname     = "freshsnow"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "cnwat"         
 outputname     = "W_I"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "icetk"         
 outputname     = "H_ICE"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "ist"         
 outputname     = "T_ICE"          
 intp_method    = 3
/
&input_field_nml
 inputname      = "HHL"         
 outputname     = "HHL"          
 intp_method    = 3
/
EOF_2A

set -x
cat NAMELIST_ICONREMAP_FIELDS

#-----------------------------------------------------------------------------
# loop over file list:

echo ${DATAFILELIST}
for datafilename in ${DATAFILELIST} ; do

datafile="${datafilename##*/}"  # get filename without path
outdatafile=${datafile%.*}      # get filename without suffix

cat > NAMELIST_ICONREMAP << EOF_2B
&remap_nml
 in_grid_filename  = '${INGRID}'
 in_filename       = '${DATADIR}/${datafile}'
 in_type           = 2
 out_grid_filename = '${LOCALGRID}'
 out_filename      = '${OUTDIR}/${outdatafile}.nc'
 out_type          = 2
 out_filetype      = 4
 l_have3dbuffer    = .false.
 ncstorage_file    = "ncstorage.tmp"
/
EOF_2B

${START}  ${ICONTOOLS_DIR}/${BINARY_REMAP} \
            --remap_nml NAMELIST_ICONREMAP                                  \
            --input_field_nml NAMELIST_ICONREMAP_FIELDS 2>&1

done

#-----------------------------------------------------------------------------
# clean-up

rm -f ncstorage.tmp*
rm -f nml.log  NAMELIST_SUB NAMELIST_ICONREMAP NAMELIST_ICONREMAP_FIELDS

#-----------------------------------------------------------------------------
exit
#-----------------------------------------------------------------------------