#!/bin/bash
# ===== SLURM settings ========================================================
#SBATCH --account=bm0682
#SBATCH --job-name=inidata.run
#SBATCH --partition=compute2
#SBATCH --nodes=1
#SBATCH --threads-per-core=2
#SBATCH --output=LOG.inidata.run.%j.o
#SBATCH --error=LOG.inidata.run.%j.o
#SBATCH --exclusive
#SBATCH --time=00:30:00
#=============================================================================
set -x
#---------------------------------------------------------------------------
# openmp environment variables
# ----------------------------
export OMP_NUM_THREADS=6
export ICON_THREADS=6
export OMP_SCHEDULE=dynamic,1
export OMP_DYNAMIC="false"
export OMP_STACKSIZE=4096M
#-----------------------------------------------------------------------------
# MPI variables
# ----------------------------
mpi_root=/sw/rhel6-x64/intel/impi/5.0.3.048/lib64
no_of_nodes=${SLURM_JOB_NUM_NODES:-1}
mpi_procs_pernode=6
mpi_total_procs=$((no_of_nodes * mpi_procs_pernode))
START="srun --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --ntasks-per-node=${mpi_procs_pernode}"

# load modules
module purge
# module load intelmpi/2018.0.128 intel/18.0.0 gcc/7.1.0
module load intel gcc 
module list

ulimit -s unlimited

# ===== load other modules ====================================================
module load nco
module load cdo
#=============================================================================

# ===== directory and file name settings ======================================
# all names of intermediate, temporary files contain ${SLURM_JOB_ID} to avoid
# accidentally overwriting the intermediate files of another job
BINDIR=/work/mh0731/m300382/dwd_icon_tools_git/icontools
SCRATCHDIR=/scratch/m/m300382
OUTDIR=/work/mh0731/m300382/tornado_dolo
# INPUT FILE
INFILE_COSMO=/work/mh0731/m300382/tornado_dolo/pamore/smi/laf2015070806.nc

# OUTPUT FILE
#   /work/bm0834/k203095/pool/GRIDS/HDCP_FINAL_4/hdcp2_cosmodom_R0625m.nc
#   /work/bm0834/k203095/pool/GRIDS/HDCP_FINAL_4/hdcp2_cosmodom_nest_R0312m.nc
#   /work/bm0834/k203095/pool/GRIDS/HDCP_FINAL_4/hdcp2_cosmodom_nest_R0156m.nc
OUTGRIDFILE=/work/mh0731/m300382/tornado_dolo/tornado_parent0622m.nc
OUTFILE=${OUTDIR}/init_tornado_parent622m_2015070806.nc

# ====== check files ==========================================================
for file in ${INFILE_COSMO} ${OUTGRIDFILE}; do
    if [ ! -f ${file} ]; then
	echo "${file} not found. Exiting..."
	exit 1
    fi
done

i=0
while [ -f ${OUTFILE} ]; do
    if [ $i -eq 0 ]; then
	OUTFILE=${OUTFILE}_00
    fi
    i=$((i+1))
    echo "${OUTFILE} exists. Saving results to ${OUTFILE}_$(printf '%02d' $i)"
    OUTFILE=${OUTFILE%??}$(printf '%02d' $i)
done

# ====== make directories =====================================================
mkdir -p ${SCRATCHDIR}
mkdir -p ${OUTDIR}

# ====== interpolation methods, grid types, and file types ====================
# set these individually for each variable in the namelists below
CONS=2 #conservative
RBF=3  #radial basis funcion
NN=4   #nearest neighbour
intp_method=3

GLOBREG=1
ICONTRI=2
LOCREG=3

GRIB=2
NC2=4
NC4=5

# ====== remap constant HHL field =============================================
echo "Processing ${INFILE_COSMO}."
cat > ${SCRATCHDIR}/remap_init_HHL_${SLURM_JOB_ID}.nml << EOF
! REMAPPING NAMELIST FILE
!
&remap_nml
 in_grid_filename   = "${INFILE_COSMO}"
 in_filename        = "${INFILE_COSMO}"
 in_type            = ${LOCREG}
 !
 out_grid_filename  = "${OUTGRIDFILE}"
 out_filename       = "${SCRATCHDIR}/lffd_remap_HHL_${SLURM_JOB_ID}.nc"
 out_type           = ${ICONTRI}
 !
 out_filetype      = ${NC4} !use NetCDF Classic because ncrename does not work with NetCDF 4
/
! DEFINITION FOR INPUT DATA FIELD
!
&input_field_nml
 inputname      = "HHL"
 outputname     = "HHL"
 intp_method    = ${RBF}
/
EOF

${srun} ${BINDIR}/iconremap_mpi --remap_nml ${SCRATCHDIR}/remap_init_HHL_${SLURM_JOB_ID}.nml

# Rename variable and dimensions according to ICON definition
#ncrename -d height,lev_2 -v height,lev_2 "${SCRATCHDIR}/lffd_remap_HHL_${SLURM_JOB_ID}.nc"
#ncrename -d height,lev_3 -v height,lev_3 "${SCRATCHDIR}/lffd_remap_HHL_${SLURM_JOB_ID}.nc"

# Add to outputfile
#ncks -A -v HHL ${SCRATCHDIR}/lffd_remap_HHL_${SLURM_JOB_ID}.nc ${OUTFILE}
# ====== remap COSMO-DE fields ================================================
# Exclude U and V wind components because of different coordinates,
# which are a problem for the dwdtools
UVARS=U,AUMFL_S,srlon,slonu,slatu
VVARS=V,AVMFL_S,srlat,slonv,slatv
ncks -O -x -v ${UVARS},${VVARS} ${INFILE_COSMO} ${SCRATCHDIR}/lffd_xUV_${SLURM_JOB_ID}.nc

echo "Processing ${INFILE_COSMO}."
echo "Processing ${SCRATCHDIR}/lffd_xUV_${SLURM_JOB_ID}.nc."
cat > ${SCRATCHDIR}/remap_init_${SLURM_JOB_ID}.nml << EOF
! REMAPPING NAMELIST FILE
!
&remap_nml
 in_grid_filename   = "${INFILE_COSMO}"
 in_filename        = "${SCRATCHDIR}/lffd_xUV_${SLURM_JOB_ID}.nc"
 in_type            = ${LOCREG}
 !
 out_grid_filename  = "${OUTGRIDFILE}"
 out_filename       = "${SCRATCHDIR}/lffd_remap_xUV_${SLURM_JOB_ID}.nc"
 out_type           = ${ICONTRI}
 !
 out_filetype      = ${NC4}
/
! DEFINITION FOR INPUT DATA FIELD
!
&input_field_nml
 inputname      = "T"
 outputname     = "T"
 intp_method    = ${intp_method}
/
&input_field_nml
 inputname      = "QC"         
 outputname     = "QC"
 intp_method    = ${intp_method}
/
&input_field_nml
 inputname      = "W"
 outputname     = "W"
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "QR"
 outputname     = "QR" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "QS"
 outputname     = "QS" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "QV"
 outputname     = "QV" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "QI"
 outputname     = "QI" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "PS"
 outputname     = "PS" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "P"
 outputname     = "P" 
 intp_method    = ${intp_method}
/
&input_field_nml ! ist FIS
 inputname      = "FIS"
 outputname     = "GEOSP" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! Plant Canopy Surface Water
 inputname      = "W_I"         
 outputname     = "w_i"          
 intp_method    = ${intp_method}
/
&input_field_nml  ! Snow depth water equivalent
 inputname      = "W_SNOW"
 outputname     = "w_snow"
 intp_method    = ${intp_method}
/
&input_field_nml  ! horiz. wind comp. v
 inputname      = "T_G"         
 outputname     = "t_g"
 intp_method    = ${intp_method}
/
&input_field_nml  ! vertical velocity
 inputname      = "T_SNOW" 
 outputname     = "t_snow"
 intp_method    = ${intp_method}
/
&input_field_nml  ! surface pressure
 inputname      = "T_ICE"
 outputname     = "t_ice" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "QV_S"
 outputname     = "qv_s" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "H_SNOW"
 outputname     = "h_snow" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "H_ICE"
 outputname     = "h_ice" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "FRESHSNW"
 outputname     = "freshsnow" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! geopotential
 inputname      = "Z0"
 outputname     = "gz0" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! soil ice content (multilayers)
 inputname      = "W_SO_ICE"
 outputname     = "w_so_ice" 
 intp_method    = ${intp_method}
/
&input_field_nml  ! Column-integrated Soil Moisture (multilayers)
 inputname      = "SMI"
 outputname     = "smi"   
 intp_method    = 4
/
&input_field_nml  ! Soil Temperature (multilayers)
 inputname      = "T_SO"
 outputname     = "t_so" 
 intp_method    = 4
/
&input_field_nml  ! geopotential
 inputname      = "RHO_SNOW"
 outputname     = "rho_snow" 
 intp_method    = ${intp_method}
/
EOF

${srun} ${BINDIR}/iconremap_mpi --remap_nml ${SCRATCHDIR}/remap_init_${SLURM_JOB_ID}.nml

### copy .nc4 to .nc2
#cdo -f nc2 copy ${SCRATCHDIR}/lffd_remap_xUV_${SLURM_JOB_ID}.nc4  ${SCRATCHDIR}/lffd_remap_xUV_${SLURM_JOB_ID}.nc
#Error (cdf_enddef) : NetCDF: One or more variable sizes violate format constraints
#Error (cdf_close) : NetCDF: HDF error

# Rename variable and dimensions according to ICON definition
#ncrename -d height,lev -d height_2,lev_2 -v height,lev -v height_2,lev_2 ${SCRATCHDIR}/lffd_remap_xUV_${SLURM_JOB_ID}.nc

# Add to outputfile
#ncks -A -v T,W,P,PS,GEOSP,QV,QC,QI,QR,QS,w_i,w_snow,t_g,t_snow,t_ice,qv_s,h_snow,h_ice,freshsnow,gz0,w_so_ice,smi,t_so,rho_snow ${SCRATCHDIR}/lffd_remap_xUV_${SLURM_JOB_ID}.nc ${OUTFILE}
##ncks -A -v height,height_2 ${INFILE_COSMO} ${OUTFILE}

# ====== remap extracted wind fields from COSMO file ==========================
# remapping the U and V wind fields with the other fields does not work because
# the DWD ICON Tools cannot handle multiple horizontal grids in one file
for var in U V; do
    ncks -O -v ${var} ${INFILE_COSMO} ${SCRATCHDIR}/lffd_${var}_${SLURM_JOB_ID}.nc

    echo "Processing ${SCRATCHDIR}/lffd_${var}_${SLURM_JOB_ID}.nc."
cat > ${SCRATCHDIR}/remap_init_${SLURM_JOB_ID}.nml << EOF
! REMAPPING NAMELIST FILE
!
&remap_nml
 in_grid_filename   = "${INFILE_COSMO}"
 in_filename        = "${SCRATCHDIR}/lffd_${var}_${SLURM_JOB_ID}.nc"
 in_type            = ${LOCREG}
 !
 out_grid_filename  = "${OUTGRIDFILE}"
 out_filename       = "${SCRATCHDIR}/lffd_remap_${var}_${SLURM_JOB_ID}.nc"
 out_type           = ${ICONTRI}
 !
 out_filetype      = ${NC4}
/
! DEFINITION FOR INPUT DATA FIELD
&input_field_nml
 inputname      = "${var}"
 outputname     = "${var}"
 intp_method    = ${RBF}
/
EOF

    ${srun} ${BINDIR}/iconremap_mpi --remap_nml ${SCRATCHDIR}/remap_init_${SLURM_JOB_ID}.nml

    # Rename variable and dimensions according to ICON definition
    # ncrename -d height,lev -v height,lev ${SCRATCHDIR}/lffd_remap_${var}_${SLURM_JOB_ID}.nc

    # Add to outputfile
    #ncks -A -v ${var} ${SCRATCHDIR}/lffd_remap_${var}_${SLURM_JOB_ID}.nc ${OUTFILE}
done

# ====== Merge ==========================
cdo -P 16 -O merge ${SCRATCHDIR}/lffd_remap_xUV_${SLURM_JOB_ID}.nc  ${SCRATCHDIR}/lffd_remap_[UV]_${SLURM_JOB_ID}.nc  ${SCRATCHDIR}/lffd_remap_HHL_${SLURM_JOB_ID}.nc        ${OUTFILE}
cdo sinfov ${OUTFILE}

# Average over bnds dimension, to delete this unused dimension
ncwa -O -a bnds -d bnds,1,1 ${OUTFILE} ${OUTFILE}
cdo sinfov ${OUTFILE}
pwd