#!/bin/ksh #============================================================================= # ===================================== # mistral batch job parameters #----------------------------------------------------------------------------- #SBATCH --account=bm0682 #SBATCH --job-name=icon_nwp #SBATCH --partition=compute2 #SBATCH --workdir=/work/mh0731/m300382/test_nwp_new #SBATCH --nodes=64 #SBATCH --threads-per-core=2 #SBATCH --output=/work/mh0731/m300382/test_nwp_new/LOG_icon.run.%j.o #SBATCH --error=/work/mh0731/m300382/test_nwp_new/LOG_icon.run.%j.o #SBATCH --exclusive #SBATCH --time=00:15:00 #============================================================================= # # ICON run script. Created by ./config/make_target_runscript # target machine is bullx # target use_compiler is intel # with mpi=yes # with openmp=no # memory_model=large # submit with sbatch -N ${SLURM_JOB_NUM_NODES:-1} # #============================================================================= set -x #. ./add_run_routines #----------------------------------------------------------------------------- # target parameters # ---------------------------- site="dkrz.de" target="bullx" compiler="intel" loadmodule="intel/16.0 ncl/6.2.1-gccsys cdo/default svn/1.8.13 hpcx/1.9.7 openmpi/2.0.2p1_hpcx-intel14" with_mpi="yes" with_openmp="no" job_name="icon_nwp" submit="sbatch -N ${SLURM_JOB_NUM_NODES:-1}" #----------------------------------------------------------------------------- # openmp environment variables # ---------------------------- export OMP_NUM_THREADS=1 export ICON_THREADS=1 export OMP_SCHEDULE=dynamic,1 export OMP_DYNAMIC="false" export OMP_STACKSIZE=1024M #----------------------------------------------------------------------------- # MPI variables # ---------------------------- mpi_root=/sw/rhel6-x64/mpi/openmpi-2.0.2p1_hpcx-intel14 no_of_nodes=${SLURM_JOB_NUM_NODES:=1} mpi_procs_pernode=$((${SLURM_JOB_CPUS_PER_NODE%%\(*} / 2)) ((mpi_total_procs=no_of_nodes * mpi_procs_pernode)) START="srun --cpu-freq=HighM1 --kill-on-bad-exit=1 --nodes=${SLURM_JOB_NUM_NODES:-1} --cpu_bind=verbose,cores --distribution=block:block --ntasks=$((no_of_nodes * mpi_procs_pernode)) --ntasks-per-node=${mpi_procs_pernode} --cpus-per-task=$((2 * OMP_NUM_THREADS)) --propagate=STACK,CORE" #----------------------------------------------------------------------------- # load ../setting if exists if [ -a ../setting ] then echo "Load Setting" . ../setting fi #----------------------------------------------------------------------------- bindir="${basedir}/build/x86_64-unknown-linux-gnu/bin" # binaries BUILDDIR=build/x86_64-unknown-linux-gnu #----------------------------------------------------------------------------- #============================================================================= # load profile if [ -a /etc/profile ] ; then . /etc/profile #============================================================================= #============================================================================= # load modules module purge module load "$loadmodule" module list #============================================================================= fi #============================================================================= export LD_LIBRARY_PATH=/sw/rhel6-x64/netcdf/netcdf_c-4.4.0-parallel-openmpi2-intel14/lib:$LD_LIBRARY_PATH #============================================================================= nproma=16 cdo="cdo" cdo_diff="cdo diffn" icon_data_rootFolder="/pool/data/ICON" icon_data_poolFolder="/pool/data/ICON" icon_data_buildbotFolder="/pool/data/ICON/buildbot_data" icon_data_buildbotFolder_aes="/pool/data/ICON/buildbot_data/aes" icon_data_buildbotFolder_oes="/pool/data/ICON/buildbot_data/oes" export EXPNAME=icon_nwp export KMP_AFFINITY="verbose,granularity=core,compact,1,1" export KMP_LIBRARY="turnaround" export KMP_KMP_SETTINGS="1" export OMP_WAIT_POLICY="active" export OMPI_MCA_pml="cm" export OMPI_MCA_mtl="mxm" export OMPI_MCA_coll="^fca" export MXM_RDMA_PORTS="mlx5_0:1" export HCOLL_MAIN_IB="mlx5_0:1" export HCOLL_ML_DISABLE_BARRIER="1" export HCOLL_ML_DISABLE_IBARRIER="1" export HCOLL_ML_DISABLE_BCAST="1" export HCOLL_ENABLE_MCAST_ALL="1" export HCOLL_ENABLE_MCAST="1" export OMPI_MCA_coll_sync_barrier_after_alltoallv="1" export OMPI_MCA_coll_sync_barrier_after_alltoallw="1" export MXM_HANDLE_ERRORS="bt" export UCX_HANDLE_ERRORS="bt" export MALLOC_TRIM_THRESHOLD_="-1" ulimit -s 2097152 ulimit -c 0 # this can not be done in use_mpi_startrun since it depends on the # environment at time of script execution #case " $loadmodule " in # *\ mxm\ *) # START+=" --export=$(env | sed '/()=/d;/=/{;s/=.*//;b;};d' | tr '\n' ',')LD_PRELOAD=${LD_PRELOAD+$LD_PRELOAD:}${MXM_HOME}/lib/libmxm.so" # ;; #esac #-------------------------------------------------------------------------------------------------- #-------------------------------------------------------------------------------------------------- ICON_BASE_PATH=/work/mh0731/m300382/icon-git MODEL=${ICON_BASE_PATH}/build/x86_64-unknown-linux-gnu/bin/icon RUNSCRIPTDIR=${ICON_BASE_PATH}/run ##################################### EXPDIR=/work/mh0731/m300382/test_nwp_new/exp/test # the directory for the experiment will be created, if not already there if [ ! -d $EXPDIR ]; then mkdir -p $EXPDIR fi # cd $EXPDIR # #-------------------------------------------------------------------------------------------------- ###### the grid parameters ### Start from 1250 m domain GRIDFOLDER="/work/mh0731/m300382/test_nwp_new" grid_1=italy_grid_nested_1231m grid_2=italy_grid_nested_616m grid_3=italy_grid_nested_308m ln -sf ${GRIDFOLDER}/${grid_1}.nc ${EXPDIR}/ ln -sf ${GRIDFOLDER}/${grid_2}.nc ${EXPDIR}/ ln -sf ${GRIDFOLDER}/${grid_3}.nc ${EXPDIR}/ atmo_dyn_grids="${grid_1}.nc ${grid_2}.nc ${grid_3}.nc" dynamics_parent_grid_id=" 0, 1, 2 " ###### EXT DATA (default filename of external parameter input file, default:”extpar_”) EXTPARFOLDER=${GRIDFOLDER} ln -sf ${EXTPARFOLDER}/extpar_${grid_1}.nc ${EXPDIR}/ ln -sf ${EXTPARFOLDER}/extpar_${grid_2}.nc ${EXPDIR}/ ln -sf ${EXTPARFOLDER}/extpar_${grid_3}.nc ${EXPDIR}/ ln -sf ${ICON_BASE_PATH}/data/ECHAM6_CldOptProps.nc rrtm_cldopt.nc ln -sf ${ICON_BASE_PATH}/data/rrtmg_lw.nc . ln -sf ${ICON_BASE_PATH}/data/dmin_wetgrowth_lookup.dat . #-------------------------------------------------------------------------------------------------- #-------------------------------------------------------------------------------------------------- start_date="2018-10-29T00:00:00Z" end_date="2018-10-30T00:00:00Z" ###### INITIAL DATA #INIFOLDER=${GRIDFOLDER} # ln -sf ${INIFOLDER}/iefff00000000.nc ${EXPDIR}/ifs2icon_R2B11_DOM01.nc # ln -sf ${INIFOLDER}/iefff00000000.nc ${EXPDIR}/dwdFG_R2B11_DOM01.nc #-------------------------------------------------------------------------------------------------- ###### LATBC PATH # LATBCFOLDER=${GRIDFOLDER} # ln -sf ${LATBCFOLDER}/lateral_boundary.grid.nc ${EXPDIR}/lateral_boundary.grid.nc #-------------------------------------------------------------------------------------------------- #-------------------------------------------------------------------------------------------------- #-------------------------------------------------------------------------------------------------- # the namelist filename atmo_namelist=NAMELIST_${EXPNAME} # # #----------------------------------------------------------------------------- # model timing ### Start from 1250 m domain dtime=10 nhours_restart=180 dt_restart=`expr ${nhours_restart} \* 3600 ` nhours_checkpoint=12 dt_checkpoint=`expr ${nhours_checkpoint} \* 3600 ` # #----------------------------------------------------------------------------- # model parameters model_equations=3 # equation system # 1=hydrost. atm. T # 1=hydrost. atm. theta dp # 3=non-hydrost. atm., # 0=shallow water model # -1=hydrost. ocean #----------------------------------------------------------------------------- restart=.FALSE. #----------------------------------------------------------------------------- # create ICON master namelist # ------------------------ cat > icon_master.namelist << EOF &master_nml lrestart = ${restart} / &time_nml ini_datetime_string = "$start_date" end_datetime_string = "$end_date" dt_restart = $dt_restart / &master_model_nml model_type = 1 model_name = "ATMO" model_namelist_filename = "$atmo_namelist" model_min_rank = 1 model_max_rank = 65536 model_inc_rank = 1 / EOF #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- # write ICON namelist parameters # ------------------------ # # ------------------------ # reconstrcuct the grid parameters in namelist form dynamics_grid_filename="" for gridfile in ${atmo_dyn_grids}; do dynamics_grid_filename="${dynamics_grid_filename} '${gridfile}'" done dynamics_parent_grid_id="${dynamics_parent_grid_id}," # ------------------------ # cat > ${atmo_namelist} << EOF # ¶llel_nml nproma = 16 p_test_run = .FALSE. l_test_openmp = .FALSE. l_log_checks = .FALSE. num_io_procs = 2 num_prefetch_proc = 1 / &grid_nml dynamics_grid_filename = ${dynamics_grid_filename} dynamics_parent_grid_id = ${dynamics_parent_grid_id} l_limited_area = .TRUE. lfeedback = .FALSE. ! Daniel war TRUE start_time = 0. , 60. , 120. / &initicon_nml init_mode = 4 !4=cosmo, 2=ifs, 3=combined lread_ana = .FALSE. ifs2icon_filename = '${GRIDFOLDER}/iefff00000000.nc' dwdfg_filename = '${GRIDFOLDER}/iefff00000000.nc' / &io_nml itype_pres_msl = 5 ! IFS method with bug fix for self-consistency between SLP and geopotential itype_rh = 1 ! RH w.r.t. water restart_file_type = 5 ! 4: netcdf2, 5: netcdf4 dt_checkpoint = ${dt_checkpoint} restart_write_mode = "joint procs multifile" / &limarea_nml itype_latbc = 1 dtime_latbc = 3600 latbc_path = '${GRIDFOLDER}' latbc_filename = 'iefff__lbc.nc' latbc_boundary_grid = '${GRIDFOLDER}/lateral_boundary.grid.nc' ! Grid file defining the lateral boundary init_latbc_from_fg = .TRUE. !Otherwise with this version of ICON you get the Internal Error in mtime / &run_nml num_lev = 90 lvert_nest = .FALSE. dtime = ${dtime} ! timestep in seconds ldynamics = .TRUE. ! dynamics ltransport = .TRUE. iforcing = 3 ! NWP forcing ltestcase = .FALSE. ! FALSE: run with real data msg_level = 5 ! default: 5, much more: 20 ltimer = .TRUE. timers_level = 10 activate_sync_timers = .FALSE. output = 'nml' check_uuid_gracefully = .TRUE. / &nwp_phy_nml inwp_gscp = 2 ! 1: default, 2: graupel=NARVAL, 4: for two-moment=HDCP2 inwp_convection = 0 inwp_radiation = 1 inwp_cldcover = 1, 5, 5 ! 0: no cld, 1: new diagnostic, 3: COSMO, 5: grid scale inwp_turb = 1, 5, 5 ! 1/10: Raschendorfer, 2: GME, 3: EDMF-DUALM (ntracer+1,ntiles=8) inwp_satad = 1 != NARVAL inwp_sso = 0 != NARVAL inwp_gwd = 0 != NARVAL inwp_surface = 1 != NARVAL itype_z0 = 2 != NARVAL !1: default, 2: turn off SSO part of z0 dt_rad = 720. latm_above_top = .FALSE. != NARVAL ! maybe should be set to true? (=take into account atmosphere above model top for radiation computation) efdt_min_raylfric = 7200. != NARVAL lrtm_filename = 'rrtmg_lw.nc' cldopt_filename = 'rrtm_cldopt.nc' / &turbdiff_nml !!! Daniel Dynmod setup tkhmin = 0.75 !=MBOld=NARVAL tkmmin = 0.75 !=MBOld=NARVAL tkmmin_strat = 4 ! NA in MBOld pat_len = 750. !=MBOld=NARVAL c_diff = 0.2 !=MBOld=NARVAL rat_sea = 7.0 !=8.0 in MBOld ltkesso = .TRUE.!=MBOld=NARVAL frcsmot = 0.2 !=MBOld! these 2 switches together apply vertical smoothing of the TKE source terms imode_frcsmot = 2 !=MBOld! in the tropics (only), which reduces the moist bias in the tropical lower troposphere ! use horizontal shear production terms with 1/SQRT(Ri) scaling to prevent unwanted side effects: itype_sher = 3 !=MBOld=NARVAL ltkeshs = .TRUE.!=MBOld=NARVAL a_hshr = 2.0 !=MBOld=NARVAL alpha0 = 0.0123!=MBOld=NARVAL alpha0_max = 0.0335!=MBOld=NARVAL icldm_turb = 1 !=NA in MBOld !Mode of water cloud representation in turbulence / &lnd_nml ntiles = 1 nlev_snow = 3 lmulti_snow = .FALSE. itype_heatcond = 2 idiag_snowfrac = 20 lsnowtile = .FALSE. !! later on .TRUE. if GRIB encoding issues are solved lseaice = .TRUE. llake = .TRUE. itype_lndtbl = 3 ! minimizes moist/cold bias in lower tropical troposphere itype_root = 2 / &radiation_nml irad_o3 = 7 irad_aero = 6 albedo_type = 2 ! 1: default, 2: MODIS / &nonhydrostatic_nml iadv_rhotheta = 2 ivctype = 2 itime_scheme = 4 exner_expol = 0.333 vwind_offctr = 0.25 damp_height = 25000. rayleigh_coeff = 0.25 lhdiff_rcf = .TRUE. divdamp_order = 24 ! (does not allow checkpointing/restarting earlier than 2.5 hours of integration) divdamp_type = 32 divdamp_fac = 0.004 l_open_ubc = .FALSE. igradp_method = 2 l_zdiffu_t = .TRUE. thslp_zdiffu = 0.02 thhgtd_zdiffu = 125. htop_moist_proc = 22500. hbot_qvsubstep = 19000. / &sleve_nml min_lay_thckn = 20. max_lay_thckn = 400. ! maximum layer thickness below htop_thcknlimit htop_thcknlimit = 14000. ! this implies that the upcoming COSMO-EU nest will have 60 levels top_height = 30000. !=NARVAL stretch_fac = 0.9 decay_scale_1 = 4000. decay_scale_2 = 2500. decay_exp = 1.2 flat_height = 16000. / &dynamics_nml iequations = 3 idiv_method = 1 divavg_cntrwgt = 0.50 lcoriolis = .TRUE. / &transport_nml ivadv_tracer = 3,3,3,3,3 itype_hlimit = 3,4,4,4,4 ihadv_tracer = 52,2,2,2,2 / &diffusion_nml lhdiff_vn = .TRUE. ! diffusion on the horizontal wind field lhdiff_temp = .TRUE. ! diffusion on the temperature field lhdiff_w = .TRUE. ! diffusion on the vertical wind field hdiff_order = 5 ! order of nabla operator for diffusion itype_vn_diffu = 1 ! reconstruction method used for Smagorinsky diffusion itype_t_diffu = 2 ! discretization of temperature diffusion hdiff_efdt_ratio = 24.0 ! ratio of e-folding time to time step hdiff_smag_fac = 0.025 ! scaling factor for Smagorinsky diffusion lsmag_3d = .TRUE. ! drieg: test / &interpol_nml nudge_zone_width = 4 ! Daniel=4, Matt=8 ; Total width (in units of cell rows) for lateral boundary nudging zone. ! If < 0 the patch boundary_depth_index is used. lsq_high_ord = 3 ! Daniel=3, Matt=2 rbf_vec_scale_c = 0.09, 0.03, 0.010 rbf_vec_scale_v = 0.21, 0.07, 0.025 rbf_vec_scale_e = 0.45, 0.15, 0.050 / &gridref_nml grf_intmethod_e = 5 ! Daniel=6, Matt=5 grf_intmethod_ct = 2 ! Daniel=2, Matt=1 grf_tracfbk = 2 ! Daniel=2, Matt=1 l_mass_consvcorr = .TRUE. ! Daniel=TRUE, Matt=NA(default=FALSE) / &extpar_nml itopo = 1 fac_smooth_topo = 0.1 ! Daniel=einkommentiert , Matt=NA hgtdiff_max_smooth_topo = 750. ! Daniel=einkommentiert , Matt=NA n_iter_smooth_topo = 1,1,2 heightdiff_threshold = 1000., 1000., 750. / &les_nml ! Daniel=einkommentiert , Matt=NA smag_constant = 0.3 isrfc_type = 1 ldiag_les_out = .FALSE. les_metric = .TRUE. / &output_nml filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2 dom = -1 ! write all domains mode = 1 ! 1: forecast output_time_unit = 1 ! 1: seconds output_bounds = 0., 280800., 1800. steps_per_file = 24 include_last = .TRUE. output_filename = '${EXPNAME}-2D' ml_varlist = 'tqv_dia','tqc_dia','tqi_dia','t_2m','rh_2m','sp_10m','u_10m','v_10m','shfl_s','lhfl_s','tot_prec','clct','pres_msl','cape_ml','cin_ml' filename_format = "_DOM__" output_grid = .TRUE. / &output_nml filetype = 5 ! output format: 2=GRIB2, 4=NETCDFv2 dom = -1 ! write all domains mode = 1 ! 1: forecast output_time_unit = 1 ! 1: seconds output_bounds = 0., 280800., 1800. steps_per_file = 24 include_last = .TRUE. output_filename = '${EXPNAME}-3D' pl_varlist = 'u','v','w','rh','temp','clc','geopot','qv','qc','qr','qi','div' p_levels = 100000, 97500, 95000, 92500, 90000, 87500, 85000, 80000, 75000, 70000, 65000, 60000, 50000, 40000, 35000, 30000, 25000, 20000, 15000, 10000, 5000 output_grid = .TRUE. / EOF #----------------------------------------------------------------------------- # start experiment # Take the executable that is in the folder cp -p $MODEL ./icon.exe $START icon.exe finish_status=`cat finish.status` echo $finish_status echo "============================" echo "Script run successfully: $finish_status" echo "============================" #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- # check if we have to restart, ie resubmit #if [ $finish_status = "RESTART" ] ; then # if [ $finish_status = "OK" ] ; then # touch ${restartSemaphoreFilename} # create the file "isRestartRun.sem" exists. in the next iteration the script will restart # echo "restart next experiment..." # this_script="${RUNSCRIPTDIR}/${job_name}" # echo 'this_script: ' "$this_script" # cd ${RUNSCRIPTDIR} # ${submit} $this_script # fi #----------------------------------------------------------------------------- exit #-----------------------------------------------------------------------------