# Invocation command line:
# /root/aocc/SPEChpc2021/bin/harness/runhpc -c aocc-hpc2021.cfg --reportable -n 2 -T base,peak --define model=omp --threads=16 --ranks=24 -D tiny
# output_root was not used for this run
############################################################################
######################################################################
# Example configuration file for AOMP LLVM/Clang compiler.
#
# Defines: "model" => "mpi", "omp", "omp_target", omp_host_target"
#              default "mpi"
#          "label" => ext base label,
#              default "clang"
#          "gputype" => "host" , "x86", "gfx900", "gfx906", "gfx908"
#              default "host"
# Example runhpc commands
#
# MPI-only Command:
# runhpc -c amdgpu_clang.cfg -I -l -n 1 -T base --define model=mpi --threads=1 --ranks=16 -i test 628
#
# MPI+OpenMP Command:
# runhpc -c amdgpu_clang.cfg -I -l -n 1 -T base --define model=omp --threads=16--ranks=1 -i test 628
#
# MPI+OpenMP target offload Command:
# runhpc -c amdgpu_clang.cfg -I -l -n 1 -T base --define model=omp_target --define gputype=gfx908 --threads=1 --ranks=4 -i test 628
#
# MPI+OpenMP target offload to host Command:
# runhpc -c amdgpu_clang.cfg -I -l -n 1 -T base --define model=omp_host_target --define gputype=x86 --threads=16 --ranks=1 -i test 628
#
#######################################################################

%ifndef %{label}         # IF label is not set use clang
%   define label clang
%endif

%ifndef %{model}       # IF model is not set use mpi
%   define model mpi
%endif

%ifndef %{gputype}
%   define gputype host
%endif
######################################################################
# The header section of the config file.  Must appear
# before any instances of "section markers" (see below)
#
# ext = how the binaries you generated will be identified
# tune = specify "base" or "peak" or "all"

label         = %{label}_%{model}_%{gputype}
tune          = base
output_format = text
use_submit_for_speed = 1

makeflags = -j 16

#strict_rundir_verify=0

#include: desc_amdgpu.inc
#  ----- Begin inclusion of 'desc_amdgpu.inc'
############################################################################

# Tester Information
license_num     = 6569
showtimer = 0
test_sponsor    = Supermicro
tester          = Supermicro

######################################################
# SUT Section
######################################################
# General SUT info
system_vendor      = Supermicro
system_name        = A+ Server 2125HS-TNR (AMD EPYC 9654)
hw_avail           = Nov-2022
sw_avail           = Nov-2022

# Computation node info
# [Node_Description: Hardware]
node_compute_syslbl = A+ Server 2125HS-TNR
node_compute_order = 1
node_compute_count = 1
node_compute_purpose = compute
node_compute_hw_vendor = Supermicro
node_compute_hw_model =  A+ Server 2125HS-TNR
node_compute_hw_cpu_name =  AMD EPYC 9654
node_compute_hw_ncpuorder = 1-2 chips
node_compute_hw_nchips = 2
node_compute_hw_ncores = 192
node_compute_hw_ncoresperchip = 96
node_compute_hw_nthreadspercore = 2
node_compute_hw_cpu_char = Max. Boost Clock upto 3.7GHz
node_compute_hw_cpu_mhz = 2400
node_compute_hw_pcache = 32 KB I + 32 KB D on chip per core
node_compute_hw_scache = 1 MB I+D on chip per core
node_compute_hw_tcache    = 384 MB I+D on chip per chip
node_compute_hw_ocache = None
node_compute_hw_memory = 1536 GB (24 x 64 GB 2Rx4 PC5-4800B-R)
node_compute_hw_disk =  4 x 1.92 TB NVMe PCIe Gen4.0
node_compute_hw_other = None

#[Node_Description: Accelerator]

#[Node_Description: Software]
node_compute_hw_adapter_fs_model   = Mellanox ConnectX-5 Ex
node_compute_hw_adapter_fs_count = 1
node_compute_hw_adapter_fs_slot_type = PCI-Express 4.0 x16
node_compute_hw_adapter_fs_data_rate = 100 Gbits/s
node_compute_hw_adapter_fs_ports_used = 1
node_compute_hw_adapter_fs_interconnect = Nvidia Mellanox ConnectX-5 Ex
node_compute_hw_adapter_fs_driver = None
node_compute_hw_adapter_fs_firmware = None
node_compute_sw_os000 = SUSE Linux Enterprise Server 15 SP4
node_compute_sw_os001 = Kernel 5.14.21-150400.24.21-default
node_compute_sw_localfile = ext4
node_compute_sw_sharedfile = NFS share
node_compute_sw_state = Multi-user, run level 3
node_compute_sw_other = None

#[Fileserver]

#[Interconnect]

#[Software]
sw_compiler000   = AMD Optimizing C/C++ and Fortran Compilers (AOCC)
sw_compiler001 = Version 4.0.0 Build 389 for Linux
sw_mpi_library = OpenMPI Version 4.1.1
sw_mpi_other = None
system_class = Homogenous
sw_other = None

#[General notes]
notes_000 = MPI startup command:
notes_005 =   mpirun command was used to start MPI jobs.

#######################################################################
# End of SUT section
# If this config file were to be applied to several SUTs, edits would
# be needed only ABOVE this point.
######################################################################



# ---- End inclusion of '/home/hpcuser/SPEChpc2021/config/desc_amdgpu.inc'
flagsurl000=http://www.spec.org/hpc2021/flags/amd2021_flags.2022-11-10.xml


default:
CC           = mpicc
CXX          = mpicxx
FC           = mpif90

CC_VERSION_OPTION  = --version
CXX_VERSION_OPTION = --version
FC_VERSION_OPTION  = --version

#preENV_OMP_PROC_BIND=true
MPIRUN_OPTS = --bind-to none #socket # core
submit = mpirun ${MPIRUN_OPTS} -np $ranks $command

#######################################################################

default=base=default:
OPTIMIZE      = -O3 -ffast-math -flto -march=znver4
COPTIMIZE     =
CXXOPTIMIZE   =
PORTABILITY = -I${AOMP}/include
PORTABILITY_LIBS = -lm
FPPPORTABILITY +=  -DSPEC_USE_MPIFH -I${MPI}/include/

%if %{model} eq 'mpi'
  pmodel=MPI
  MPIRUN_OPTS += --mca topo basic
  submit = mpirun ${MPIRUN_OPTS} -np $ranks $command
%endif

%if %{model} eq 'omp'
  pmodel=OMP
  OPTIMIZE += -fopenmp
  MPIRUN_OPTS = --bind-to  core:overload-allowed
  MPIRUN_OPTS += --map-by ppr:1:numa:pe=8
  MPIRUN_OPTS += --mca topo basic
  submit = mpirun  --allow-run-as-root ${MPIRUN_OPTS} -np $ranks $command
%endif

%if %{model} eq 'omp_target'
  pmodel=TGT
  OPTIMIZE += -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=%{gputype}
  513.soma_t,613.soma_s:
    PORTABILITY += -DSPEC_NO_VAR_ARRAY_REDUCE
%endif

%if %{model} eq 'omp_host_target'
  pmodel=TGT
  OPTIMIZE += -fopenmp -fopenmp-targets=x86_64-pc-linux-gnu -Xopenmp-target=x86_64-pc-linux-gnu -mcpu=%{gputype}
  521.miniswp_t,621.miniswp_s:
    PORTABILITY += -DSPEC_USE_HOST_THREADS
%endif

# HIP is not a supported hpc2021 model, just for experimentation.
%if %{model} eq 'hip'
  OPTIMIZE += -DSPEC_HIP -DSPEC_CUDA --amdgpu-target=%{gputype}
%endif

# No peak flags set, so make peak use the same flags as base
default=peak=default:
basepeak=1