ETK in Mare Nostrum 4

EinsteinToolkit in MareNostrum 4
Alejandro Cruz Osorio, Institute for Theoretical Physics, Frankfurt

Modules necessary for intel compiler
papi,fftw,cmake,hwloc,openblas,gsl,autotools,ohpc

1) intel/2017.4
2) impi/2017.4
3) mkl/2017.4
4) bsc/1.0
5) szip/2.1.1

unset MPI

Errors that can appear
1.- /gpfs/projects/uv03/uv03281/Cactus> /usr/bin/git --git-dir='/gpfs/projects/uv03/uv03281/Cactus/configs/sim/configjar.git/.git' commit -m 'build-sim-login1.bsc.mn-uv03281-2019.07.31-10.58.44-45280'

Fix:

cd /gpfs/projects/uv03/uv03281/Cactus/configs/sim/configjar.git/.git

git config core.preloadIndex false

2.- /usr/bin/git --git-dir='/gpfs/projects/uv03/uv03281/Cactus/configs/sim/configjar.git/.git' gc

Fix:

git config --global pack.windowMemory "100m"

git config --global pack.packSizeLimit "100m"

git config --global pack.threads "1"

3. -/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc:438:30: error: 'pt_loc_x' was not declared in this scope normalised_xx = x[i3D] - pt_loc_x[PT_ss_idx];
/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc:439:30: error: 'pt_loc_y' was not declared in this scope normalised_yy = y[i3D] - pt_loc_y[PT_ss_idx];
/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc:440:30: error: 'pt_loc_z' was not declared in this scope normalised_zz = z[i3D] - pt_loc_z[PT_ss_idx];
/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc: In function 'void Disk_Analysis_calculate_angles_shell(cGH*)':
/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc:1562:16: error: 'qlm_coordspinx' was not declared in this scope BH_spin[0] = qlm_coordspinx[QLM_ss_idx];
/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc:1563:16: error: 'qlm_coordspiny' was not declared in this scope BH_spin[1] = qlm_coordspiny[QLM_ss_idx];
/gpfs/projects/uv03/uv03281/Cactus/configs/sim/build/disk_analysis/disk_analysis.cc:1564:16: error: 'qlm_coordspinz' was not declared in this scope BH_spin[2] = qlm_coordspinz[QLM_ss_idx];

Fix: Add a line code 'public:' in

Cactus/arrangements/EinsteinAnalysis/QuasiLocalMeasures/interface.ccl

Cactus/arrangements/EinsteinAnalysis/PunctureTracker/interface.ccl

Optionlist (recompile after change)
Cactus/simfactory/mdb/optionlists/generic.cfg

# generic

# Whenever this version string changes, the application is configured
# and rebuilt from scratch
VERSION = 2018-12-13

CPP = cpp
CC = gcc
CXX = g++

FPP = cpp
F90 = gfortran

CPPFLAGS =
FPPFLAGS = -traditional

CFLAGS = -g -std=gnu99
CXXFLAGS = -g -std=gnu++11
F90FLAGS = -g -fcray-pointer -ffixed-line-length-none

LDFLAGS = -rdynamic

DEBUG = no
CPP_DEBUG_FLAGS =
C_DEBUG_FLAGS =
CXX_DEBUG_FLAGS =

OPTIMISE = yes
CPP_OPTIMISE_FLAGS =
C_OPTIMISE_FLAGS = -O2
CXX_OPTIMISE_FLAGS = -O2
F90_OPTIMISE_FLAGS = -O2

PROFILE = no
CPP_PROFILE_FLAGS =
C_PROFILE_FLAGS = -pg
CXX_PROFILE_FLAGS = -pg
F90_PROFILE_FLAGS = -pg

WARN = yes
CPP_WARN_FLAGS = -Wall
C_WARN_FLAGS = -Wall
CXX_WARN_FLAGS = -Wall
F90_WARN_FLAGS = -Wall

OPENMP = yes
CPP_OPENMP_FLAGS = -fopenmp
FPP_OPENMP_FLAGS = -D_OPENMP
C_OPENMP_FLAGS = -fopenmp
CXX_OPENMP_FLAGS = -fopenmp
F90_OPENMP_FLAGS = -fopenmp

VECTORISE = yes
VECTORISE_ALIGNED_ARRAYS = no
VECTORISE_INLINE = yes

RunScript (recompile after change)
Cactus/simfactory/mdb/runscripts/generic.run

echo "Preparing:"
set -x # Output commands
set -e # Abort on errors

cd @RUNDIR@-active
echo "Checking:"
pwd
hostname
date

echo "Environment:"
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export GMON_OUT_PREFIX=gmon.out
export OMP_NUM_THREADS=@NUM_THREADS@
env | sort > SIMFACTORY/ENVIRONMENT

echo " Requested ressources ########################### "
echo " Requested NODES: @NODES@ "
echo " Requested CORES per NODE: @PPN@ "
echo " Requested CORES: @PROCS_REQUESTED@ "
echo " Requested MPI processes: @NUM_PROCS@ "
echo " Requested THREADS: @PROCS@ "
echo " Allocation per node ############################ "
echo " MPI processes per node: @NODE_PROCS@ "
echo " Threads per MPI process: @NUM_THREADS@ "
echo " CORES per NODE: @PPN_USED@ "
echo " Threads per core: @NUM_SMT@ "

echo "Starting:"
export CACTUS_STARTTIME=$(date +%s)

echo @EXECUTABLE@
time mpirun -np @NUM_PROCS@ @EXECUTABLE@ -L 3 @PARFILE@

echo "Stopping:"
date
echo "Done."

SubmitScript (recompile after change)
Cactus/simfactory/mdb/submitscripts/generic.sub

#!/bin/sh
#SBATCH --time=@WALLTIME@
#SBATCH --qos=class_a
#SBATCH --constraint=highmem
#SBATCH -N @NODES@ -n @NUM_PROCS@
#SBATCH --ntasks-per-node=@NODE_PROCS@
#SBATCH --cpus-per-task=@NUM_THREADS@
#SBATCH @("@CHAINED_JOB_ID@" != "" ? "-d afterok:@CHAINED_JOB_ID@" : "")@
#SBATCH --job-name=@SHORT_SIMULATION_NAME@
#SBATCH --mail-type=ALL
#SBATCH --output=@RUNDIR@/@SIMULATION_NAME@.out
#SBATCH --error=@RUNDIR@/@SIMULATION_NAME@.err

cd @SOURCEDIR@
#@SIMFACTORY@ run @SIMULATION_NAME@ --restart-id=@RESTART_ID@ @FROM_RESTART_COMMAND@
@SIMFACTORY@ run @SIMULATION_NAME@ --basedir=/gpfs/projects/uv03/uv03281/simulations --machine=login1.bsc.mn --restart -id=@RESTART_ID@ @FROM_RESTART_COMMAND@