diff --git a/.gitignore b/.gitignore index ae349ba..72c4448 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,10 @@ _upstream/ .deps/ build/ +# vendored upstream source directories that happen to be named build/ (Level 2 byte-identical copies); +# a clean clone cannot build hipBone / miniWeather without them +!level2/hipbone/occa/scripts/build/ +!level2/miniweather/cpp/build/ cmake-build-*/ results/ diff --git a/level2/branson/validate.sh b/level2/branson/validate.sh index 33fa4f6..5e78553 100755 --- a/level2/branson/validate.sh +++ b/level2/branson/validate.sh @@ -67,8 +67,22 @@ command -v mpirun >/dev/null 2>&1 || fail "mpirun not on PATH (source $R/hpcperf command -v python3 >/dev/null 2>&1 || fail "python3 not on PATH" # ---------------------------------------------------------------- (A) ctest +# Upstream's 2-rank unit tests (test_imc_state_2pe, test_photon_2pe) are launched by ctest with a bare +# `mpirun -np 2`. Inside a Slurm allocation made with one task slot per node (SLURM_TASKS_PER_NODE=1) PRRTE +# refuses that launch ("not enough slots"); the project launcher relaxes the same limit per launch after its +# GPU checks (hpcperf_mpi_launch.sh, --map-by :OVERSUBSCRIBE). The same relaxation is applied here, for the +# ctest step only, and only when the allocation actually has fewer slots than the tests need (2 CPU ranks on +# one node; no GPU sharing is involved). +# The relaxation is a command-local environment of the ctest invocation only (never exported, so the caller's +# environment and the later 1-rank GPU run (B) are untouched). +SLOTS="${SLURM_TASKS_PER_NODE:-}"; SLOTS="${SLOTS%%[^0-9]*}" +CTEST_ENV=() +if [ -n "${SLURM_JOB_ID:-}" ] && [ -n "$SLOTS" ] && [ "$SLOTS" -lt 2 ]; then + CTEST_ENV=(env PRTE_MCA_rmaps_default_mapping_policy=":OVERSUBSCRIBE") + echo "== note: Slurm allocation has $SLOTS task slot(s)/node; PRRTE mapping relaxed for the 2-rank unit tests (ctest command only)" +fi echo "== (A) upstream ctest in $BUILD (test_input_1pe excluded, see header)" -if ! ctest --test-dir "$BUILD" -E test_input_1pe --output-on-failure > "$BUILD/validate_ctest.log" 2>&1; then +if ! "${CTEST_ENV[@]}" ctest --test-dir "$BUILD" -E test_input_1pe --output-on-failure > "$BUILD/validate_ctest.log" 2>&1; then tail -30 "$BUILD/validate_ctest.log" fail "ctest failed (log: $BUILD/validate_ctest.log)" fi diff --git a/level2/hipbone/occa/scripts/build/Make.fortran b/level2/hipbone/occa/scripts/build/Make.fortran new file mode 100644 index 0000000..0b5f19c --- /dev/null +++ b/level2/hipbone/occa/scripts/build/Make.fortran @@ -0,0 +1,31 @@ +#---[ Paths/Flags ]------------------------------- +# OCCA (global) library path +fPaths = -L$(OCCA_DIR)/lib + +# Extra (user-supplied) include and library paths +fPaths += $(foreach path, $(subst :, ,$(OCCA_INCLUDE_PATH)), -I$(path)) +fPaths += $(foreach path, $(subst :, ,$(OCCA_LIBRARY_PATH)), -L$(path)) + +# OCCA (global) Fortran module path - add as include path to allow for +# specifying a local project module path) +fPaths += -I$(subst $(PROJ_DIR),$(OCCA_DIR),$(modPath)) + +fLinkerFlags += $(fCppFlag) -locca -locca_fortran + +fSoNameFlag = $(subst libocca,libocca_fortran,$(soNameFlag)) +#================================================= + + +#---[ Source/Object files ]----------------------- +# ---[ libocca_fortran ]------------- +# Fortran source files +fSources = $(shell find $(OCCA_DIR)/src/fortran -type f -iname '*.f90') +fSrcToObject = $(subst $(OCCA_DIR)/src,$(objPath),$(1:.f90=.o)) +fObjects = $(call fSrcToObject,$(fSources)) +# =================================== + +# ---[ Fortran tests ]--------------- +fTestSources = $(realpath $(shell find $(PROJ_DIR)/tests/src -type f -name '*.f90')) +fTests = $(subst $(testPath)/src,$(testPath)/bin,$(fTestSources:.f90=)) +# =================================== +#================================================= diff --git a/level2/hipbone/occa/scripts/build/Make.fortran_rules b/level2/hipbone/occa/scripts/build/Make.fortran_rules new file mode 100644 index 0000000..1bb39cf --- /dev/null +++ b/level2/hipbone/occa/scripts/build/Make.fortran_rules @@ -0,0 +1,28 @@ +#---[ Fortran rules ]----------------------------- +# Fortran module dependencies +fObjPath = $(objPath)/fortran +$(fObjPath)/occa_types_m.o: $(fObjPath)/occa_typedefs_m.o +$(fObjPath)/occa_base_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_dtype_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_device_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_memory_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_kernel_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_kernelBuilder_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_stream_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_uva_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_scope_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_json_m.o: $(fObjPath)/occa_types_m.o +$(fObjPath)/occa_m.o: $(fObjPath)/fc_string_m.o \ + $(fObjPath)/occa_typedefs_m.o \ + $(fObjPath)/occa_types_m.o \ + $(fObjPath)/occa_base_m.o \ + $(fObjPath)/occa_dtype_m.o \ + $(fObjPath)/occa_device_m.o \ + $(fObjPath)/occa_memory_m.o \ + $(fObjPath)/occa_kernel_m.o \ + $(fObjPath)/occa_kernelBuilder_m.o \ + $(fObjPath)/occa_stream_m.o \ + $(fObjPath)/occa_uva_m.o \ + $(fObjPath)/occa_scope_m.o \ + $(fObjPath)/occa_json_m.o +# ================================================ diff --git a/level2/hipbone/occa/scripts/build/Makefile b/level2/hipbone/occa/scripts/build/Makefile new file mode 100644 index 0000000..a07d1dc --- /dev/null +++ b/level2/hipbone/occa/scripts/build/Makefile @@ -0,0 +1,513 @@ +rmSlash = $(patsubst %/,%,$(1)) + +PROJ_DIR := $(call rmSlash,$(PROJ_DIR)) + +ifdef OCCA_DIR + OCCA_DIR := $(call rmSlash,${OCCA_DIR}) +endif + +ifeq ($(strip $(OCCA_DIR)),) + OCCA_DIR := $(call rmSlash,$(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../..)) +endif + +binPath = $(PROJ_DIR)/bin +libPath = $(PROJ_DIR)/lib +modPath = $(PROJ_DIR)/lib/fortran/modules +objPath = $(PROJ_DIR)/obj +srcPath = $(PROJ_DIR)/src +incPath = $(PROJ_DIR)/include +testPath = $(PROJ_DIR)/tests + + +#---[ Default Variables ]------------------------- +debugEnabled = 0 +checkEnabled = 1 + +ifeq ($(OCCA_DEVELOPER),1) + ifeq ($(DEBUG),0) + debugEnabled = 0 + else + debugEnabled = 1 + endif +else + ifeq ($(DEBUG),1) + debugEnabled = 1 + else + debugEnabled = 0 + endif +endif + +flags = + +# CXX : C++ Compiler +# CXXFLAGS : C++ Compiler Flags + +# OCCA_INCLUDE_PATH : Extra include paths +# OCCA_LIBRARY_PATH : Extra library paths + +OCCA_COVERAGE ?= 0 +#================================================= + + +#---[ OS Detection ]------------------------------ +OCCA_LINUX_OS = 1 +OCCA_MACOS_OS = 2 +OCCA_WINDOWS_OS = 4 +OCCA_WINUX_OS = 5 # (OCCA_LINUX_OS | OCCA_WINDOWS_OS) + +usingLinux = 0 +usingMacOS = 0 +usingWinux = 0 +usingWindows = 0 + +UNAME := $(shell uname) + +ifeq ($(UNAME),Linux) + usingLinux = 1 +else ifeq ($(UNAME),Darwin) + usingMacOS = 1 +else ifneq (,$(findstring CYGWIN,$(UNAME))) + usingLinux = 1 + usingWinux = 1 +else ifneq (,$(findstring MINGW,$(UNAME))) + usingWinux = 1 + usingWindows = 1 +else + usingWindows = 1 +endif +#================================================= + + +#---[ Variables ]--------------------------------- +ifndef CXX + ifdef OCCA_CXX + CXX = ${OCCA_CXX} + else + CXX = g++ + endif +endif + +ifndef CXXFLAGS + ifeq ($(debugEnabled),1) + compilerFlags = $(debugFlags) + else + compilerFlags = $(releaseFlags) + endif +else + compilerFlags = $(CXXFLAGS) +endif + +CC ?= gcc + +ifndef CFLAGS + ifeq ($(debugEnabled),1) + cCompilerFlags = $(debugFlags) + else + cCompilerFlags = $(releaseFlags) + endif +else + cCompilerFlags = $(CFLAGS) +endif + +FC ?= gfortran + +ifndef FFLAGS + ifeq ($(debugEnabled),1) + fCompilerFlags = $(debugFlags) + else + fCompilerFlags = $(releaseFlags) + endif +else + fCompilerFlags = $(FFLAGS) +endif + +ifeq ($(OCCA_COVERAGE),1) + coverageFlags = --coverage + compilerFlags += $(coverageFlags) + cCompilerFlags += $(coverageFlags) + fCompilerFlags += $(coverageFlags) +endif + +compiler = $(CXX) +cCompiler = $(CC) +fCompiler = $(FC) + +linkerFlags = $(LDFLAGS) +fLinkerFlags = $(LDFLAGS) +#================================================= + + +#---[ Paths/Flags ]------------------------------- +# Include the public API +paths = -I$(OCCA_DIR)/include +# Include the internal API +paths += -I$(OCCA_DIR)/src + +# Link with the library for our examples +paths += -L$(OCCA_DIR)/lib + +# Extra (user-supplied) include and library paths +paths += $(foreach path, $(subst :, ,$(OCCA_INCLUDE_PATH)), -I$(path)) +paths += $(foreach path, $(subst :, ,$(OCCA_LIBRARY_PATH)), -L$(path)) + +# Project (local) include path +ifneq (,$(strip $(wildcard $(incPath)/*))) + paths += -I$(incPath) +endif + +linkerFlags += -locca +#================================================= + + +#---[ Shell Tools ]------------------------------- +ifeq (,$(findstring bash,$(SHELL))) + SHELL := $(shell which bash) + ifeq (,$(SHELL)) + $(error Could not find [bash], set SHELL manually with [export SHELL=/path/to/bash] or compile with [make SHELL=/path/to/bash]) + endif +endif + +libraryFlagsFor = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; libraryFlags "$1") +includeFlagsFor = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; headerFlags "$1") + +compilerVendor = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerVendor "$(compiler)") +compilerReleaseFlags = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerReleaseFlags "$(compiler)") +compilerDebugFlags = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerDebugFlags "$(compiler)") +compilerCpp11Flags = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerCpp11Flags "$(compiler)") +compilerPicFlag = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerPicFlag "$(compiler)") +compilerSharedFlag = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerSharedFlag "$(compiler)") +compilerPthreadFlag = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerPthreadFlag "$(compiler)") + +fCompilerModuleDirFlag = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; fCompilerModuleDirFlag "$(fCompiler)") +fCompilerCppFlag = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; fCompilerCppFlag "$(fCompiler)") + +compilerSupportsOpenMP = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerSupportsOpenMP "$(compiler)") +compilerOpenMPFlag = $(shell . $(OCCA_DIR)/scripts/build/shellTools.sh; compilerOpenMPFlag "$(compiler)") +#================================================= + + +#---[ Compiler Info ]----------------------------- +vendor = $(call compilerVendor) +debugFlags = $(call compilerDebugFlags) +releaseFlags = $(call compilerReleaseFlags) +cpp11Flags = $(call compilerCpp11Flags) +picFlag = $(call compilerPicFlag) +sharedFlag = $(call compilerSharedFlag) +pthreadFlag = $(call compilerPthreadFlag) + +fModuleDirFlag = $(call fCompilerModuleDirFlag) +fCppFlag = $(call fCompilerCppFlag) + +# Workaround for GitHub Actions CI tests on Ubuntu using GCC. +ifdef GITHUB_ACTIONS + ifeq ($(usingLinux),1) + ifeq ($(vendor),GCC) + releaseFlags := $(filter-out -march=native,$(releaseFlags)) + endif + endif +endif +#================================================= + + +#---[ Flags and Libraries ]----------------------- +OCCA_USING_VS := 0 +OCCA_UNSAFE ?= 0 +soNameFlag= +soExt= + +ifeq ($(usingLinux),1) + + ifeq ($(usingWinux),0) + OCCA_OS := OCCA_LINUX_OS + else + OCCA_OS := OCCA_WINUX_OS + endif + + linkerFlags += -lm -lrt -ldl + soNameFlag = -Wl,-soname,libocca.so + soExt = so + +else ifeq ($(usingMacOS),1) + + OCCA_OS := OCCA_MACOS_OS + flags += -Wno-deprecated-declarations + linkerFlags += -framework accelerate -framework CoreServices + soNameFlag = -Wl,-install_name,@rpath/libocca.dylib + soExt = dylib + +else ifeq ($(usingWindows),1) + + ifeq ($(usingWinux),0) + OCCA_OS := OCCA_WINDOWS_OS + OCCA_USING_VS := 1 + else + OCCA_OS := OCCA_WINDOWS_OS + endif + + linkerFlags += + soExt = so + +endif +#================================================= + + +#---[ Variable Dependencies ]--------------------- +fortranEnabled = 0 +mpiEnabled = 0 +openmpEnabled = 0 +cudaEnabled = 0 +hipEnabled = 0 +openclEnabled = 0 +metalEnabled = 0 +dpcppEnabled = 0 +threadSharableEnabled = 0 +maxArgs = 128 + +#---[ Fortran ]------------------------- +ifdef OCCA_FORTRAN_ENABLED + fortranEnabled = $(OCCA_FORTRAN_ENABLED) +endif +include $(OCCA_DIR)/scripts/build/Make.fortran + +#---[ OpenMP ]-------------------------- +ifdef OCCA_OPENMP_ENABLED + openmpEnabled = $(OCCA_OPENMP_ENABLED) +else + openmpEnabled = $(call compilerSupportsOpenMP) +endif +ifeq ($(openmpEnabled),1) + flags += $(call compilerOpenMPFlag) +endif + + +#---[ CUDA ]---------------------------- +ifdef OCCA_CUDA_ENABLED + cudaEnabled = $(OCCA_CUDA_ENABLED) + + ifeq ($(cudaEnabled),1) + ifeq ($(usingLinux),1) + linkerFlags += -lcuda + else ifeq ($(usingMacOS),1) + linkerFlags += -framework CUDA + endif + endif +else + cudaIncFlags = $(call includeFlagsFor,cuda.h) + + ifneq (,$(cudaIncFlags)) + + ifeq ($(usingLinux),1) + cudaLibFlags = $(call libraryFlagsFor,cuda) + else ifeq ($(usingMacOS),1) + cudaLibFlags = $(call libraryFlagsFor,CUDA) + endif + + ifneq (,$(cudaLibFlags)) + cudaEnabled = 1 + paths += $(cudaIncFlags) + linkerFlags += $(cudaLibFlags) + endif + endif +endif + + +#---[ HIP ]----------------------------- +ifdef OCCA_HIP_ENABLED + hipEnabled = $(OCCA_HIP_ENABLED) + + ifeq ($(hipEnabled),1) + #set HIP_PATH if not supplied by user + ifeq (,$(HIP_PATH)) + hipIncFlags = $(call includeFlagsFor,hip/hip_runtime_api.h) + ifneq (,$(hipIncFlags)) + HIP_PATH = ${hipIncFlags:-I%=%}/.. + endif + endif + + #set HIP_PLATFORM if not supplied by user + ifeq (,$(HIP_PLATFORM)) + hipPlatform = $(shell $(HIP_PATH)/bin/hipconfig --platform) + else + hipPlatform = $(HIP_PLATFORM) + endif + + ifeq ($(hipPlatform),nvidia) + linkerFlags += -lcuda + linkerFlags += -lcudart + else ifeq ($(hipPlatform),amd) + #set HIP_COMPILER if not supplied by user + ifeq (,$(HIP_COMPILER)) + hipCompiler = $(shell $(HIP_PATH)/bin/hipconfig --compiler) + else + hipCompiler = $(HIP_COMPILER) + endif + + #Select HIP Runtime + ifeq ($(hipCompiler),clang) + linkerFlags += -lamdhip64 + endif + endif + endif +else + #find HIP_PATH if not supplied by user + ifeq (,$(HIP_PATH)) + hipIncFlags = $(call includeFlagsFor,hip/hip_runtime_api.h) + ifneq (,$(hipIncFlags)) + HIP_PATH = ${hipIncFlags:-I%=%}/.. + endif + endif + + ifneq (,$(HIP_PATH)) + #set HIP_PLATFORM if not supplied by user + ifeq (,$(HIP_PLATFORM)) + hipPlatform = $(shell $(HIP_PATH)/bin/hipconfig --platform) + else + hipPlatform = $(HIP_PLATFORM) + endif + + ifeq ($(hipPlatform),nvidia) + hipLibFlags = $(call libraryFlagsFor,cuda) + hipLibFlags += $(call libraryFlagsFor,cudart) + else ifeq ($(hipPlatform),amd) + #set HIP_COMPILER if not supplied by user + ifeq (,$(HIP_COMPILER)) + hipCompiler = $(shell $(HIP_PATH)/bin/hipconfig --compiler) + else + hipCompiler = $(HIP_COMPILER) + endif + + #Select HIP Runtime + ifeq ($(hipCompiler),clang) + hipLibFlags = $(call libraryFlagsFor,amdhip64) + endif + endif + + ifneq (,$(hipLibFlags)) + hipEnabled = 1 + paths += $(shell $(HIP_PATH)/bin/hipconfig --cpp_config) + paths += $(call includeFlagsFor,hip/hip_runtime_api.h) + linkerFlags += $(hipLibFlags) + endif + endif +endif + +#---[ SYCL/DPC++ ]-------------------------- +ifdef OCCA_DPCPP_ENABLED + dpcppEnabled = $(OCCA_DPCPP_ENABLED) + + ifeq ($(dpcppEnabled),1) + ifeq ($(usingLinux),1) + linkerFlags += -lsycl + else ifeq ($(usingMacOS),1) + linkerFlags += -framework sycl + endif + endif +else + ifeq ($(usingLinux),1) + dpcppLibFlags = $(call libraryFlagsFor,sycl) + ifneq (,$(dpcppLibFlags)) + + dpcppIncFlags = $(call includeFlagsFor,CL/sycl.hpp) + ifneq (,$(dpcppIncFlags)) + dpcppEnabled = 1 + paths += $(dpcppIncFlags) + linkerFlags += $(dpcppLibFlags) + endif + endif + else ifeq ($(usingMacOS),1) + dpcppLibFlags = $(call libraryFlagsFor,sycl) + + ifneq (,$(dpcppLibFlags)) + dpcppEnabled = 1 + linkerFlags += $(dpcppLibFlags) + endif + endif +endif + +#---[ OpenCL ]-------------------------- +ifdef OCCA_OPENCL_ENABLED + openclEnabled = $(OCCA_OPENCL_ENABLED) + + ifeq ($(openclEnabled),1) + ifeq ($(usingLinux),1) + linkerFlags += -lOpenCL + else ifeq ($(usingMacOS),1) + linkerFlags += -framework OpenCL + endif + endif +else + ifeq ($(usingLinux),1) + openclLibFlags = $(call libraryFlagsFor,OpenCL) + ifneq (,$(openclLibFlags)) + + openclIncFlags = $(call includeFlagsFor,CL/cl.h) + ifneq (,$(openclIncFlags)) + openclEnabled = 1 + paths += $(openclIncFlags) + linkerFlags += $(openclLibFlags) + endif + endif + else ifeq ($(usingMacOS),1) + # OpenCL includes are embedded in the framework + openclLibFlags = $(call libraryFlagsFor,OpenCL) + + ifneq (,$(openclLibFlags)) + openclEnabled = 1 + linkerFlags += $(openclLibFlags) + endif + endif +endif + +#---[ Metal ]--------------------------- +# Metal is only supported with +# - MacOS +# - clang++ compiler +# - XCode version >= 10.2.1 +ifeq ($(usingMacOS),1) + ifdef OCCA_METAL_ENABLED + metalEnabled := $(OCCA_METAL_ENABLED) + else + xcodeVersion := $(shell xcodebuild -version | head -n 1 | sed 's/Xcode //g') + minXcodeVersion := $(shell echo -e "$(xcodeVersion)\n10.2.1" | sort -V | head -n 1) + ifeq ($(minXcodeVersion),$(xcodeVersion)) + # Check for Metal Framework + metalLibFlags = $(call libraryFlagsFor,Metal) + + ifneq (,$(metalLibFlags)) + metalEnabled = 1 + linkerFlags += $(metalLibFlags) -framework Foundation -lobjc + endif + endif + endif +endif + +#---[ Other parameters ]--------------------------- +ifdef OCCA_THREAD_SHARABLE_ENABLED + threadSharableEnabled = $(OCCA_THREAD_SHARABLE_ENABLED) +endif + +ifdef OCCA_MAX_ARGS + maxArgs = $(OCCA_MAX_ARGS) +endif + +ifeq ($(cudaEnabled),1) + compilerFlags += -Wno-c++11-long-long +endif +compilerFlags += $(cpp11Flags) + +ifeq ($(checkEnabled),1) + OCCA_CHECK_ENABLED := 1 +else + OCCA_CHECK_ENABLED := 0 +endif + +OCCA_FORTRAN_ENABLED := $(fortranEnabled) +OCCA_OPENMP_ENABLED := $(openmpEnabled) +OCCA_CUDA_ENABLED := $(cudaEnabled) +OCCA_HIP_ENABLED := $(hipEnabled) +OCCA_OPENCL_ENABLED := $(openclEnabled) +OCCA_METAL_ENABLED := $(metalEnabled) +OCCA_DPCPP_ENABLED := $(dpcppEnabled) +OCCA_THREAD_SHARABLE_ENABLED := $(threadSharableEnabled) +OCCA_MAX_ARGS := $(maxArgs) +#================================================= diff --git a/level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp b/level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp new file mode 100644 index 0000000..4b2ed36 --- /dev/null +++ b/level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp @@ -0,0 +1,36 @@ +#ifndef OCCA_DEFINES_COMPILEDDEFINES_HEADER +#define OCCA_DEFINES_COMPILEDDEFINES_HEADER + +#ifndef OCCA_LINUX_OS +# define OCCA_LINUX_OS 1 +#endif + +#ifndef OCCA_MACOS_OS +# define OCCA_MACOS_OS 2 +#endif + +#ifndef OCCA_WINDOWS_OS +# define OCCA_WINDOWS_OS 4 +#endif + +#ifndef OCCA_WINUX_OS +# define OCCA_WINUX_OS (OCCA_LINUX_OS | OCCA_WINDOWS_OS) +#endif + +#define OCCA_OS @@OCCA_OS@@ +#define OCCA_USING_VS @@OCCA_USING_VS@@ +#define OCCA_UNSAFE @@OCCA_UNSAFE@@ + +#define OCCA_OPENMP_ENABLED @@OCCA_OPENMP_ENABLED@@ +#define OCCA_CUDA_ENABLED @@OCCA_CUDA_ENABLED@@ +#define OCCA_HIP_ENABLED @@OCCA_HIP_ENABLED@@ +#define OCCA_OPENCL_ENABLED @@OCCA_OPENCL_ENABLED@@ +#define OCCA_METAL_ENABLED @@OCCA_METAL_ENABLED@@ +#define OCCA_DPCPP_ENABLED @@OCCA_DPCPP_ENABLED@@ + +#define OCCA_THREAD_SHARABLE_ENABLED @@OCCA_THREAD_SHARABLE_ENABLED@@ +#define OCCA_MAX_ARGS @@OCCA_MAX_ARGS@@ + +#define OCCA_BUILD_DIR "@@OCCA_BUILD_DIR@@" + +#endif diff --git a/level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp.in b/level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp.in new file mode 100644 index 0000000..75be10d --- /dev/null +++ b/level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp.in @@ -0,0 +1,37 @@ +#ifndef OCCA_DEFINES_COMPILEDDEFINES_HEADER +#define OCCA_DEFINES_COMPILEDDEFINES_HEADER + +#ifndef OCCA_LINUX_OS +# define OCCA_LINUX_OS 1 +#endif + +#ifndef OCCA_MACOS_OS +# define OCCA_MACOS_OS 2 +#endif + +#ifndef OCCA_WINDOWS_OS +# define OCCA_WINDOWS_OS 4 +#endif + +#ifndef OCCA_WINUX_OS +# define OCCA_WINUX_OS (OCCA_LINUX_OS | OCCA_WINDOWS_OS) +#endif + +#cmakedefine OCCA_OS @OCCA_OS@ +#cmakedefine01 OCCA_USING_VS +#cmakedefine01 OCCA_UNSAFE + +#cmakedefine01 OCCA_OPENMP_ENABLED +#cmakedefine01 OCCA_OPENCL_ENABLED +#cmakedefine01 OCCA_CUDA_ENABLED +#cmakedefine01 OCCA_HIP_ENABLED +#cmakedefine01 OCCA_METAL_ENABLED +#cmakedefine01 OCCA_DPCPP_ENABLED + +#cmakedefine01 OCCA_THREAD_SHARABLE_ENABLED +#cmakedefine OCCA_MAX_ARGS @OCCA_MAX_ARGS@ + +#cmakedefine OCCA_SOURCE_DIR "@OCCA_SOURCE_DIR@" +#cmakedefine OCCA_BUILD_DIR "@OCCA_BUILD_DIR@" + +#endif diff --git a/level2/hipbone/occa/scripts/build/shellTools.sh b/level2/hipbone/occa/scripts/build/shellTools.sh new file mode 100644 index 0000000..75a0734 --- /dev/null +++ b/level2/hipbone/occa/scripts/build/shellTools.sh @@ -0,0 +1,459 @@ +#!/bin/bash + +SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OCCA_SOURCE_DIR="$(cd "$(dirname "${SCRIPTS_DIR}")" && pwd)" +OCCA_SOURCE_SCRIPTS_DIR="${OCCA_SOURCE_DIR}/include/occa/scripts" + +#---[ Library Information ]------------- +function uniqueAddToPath { + local path="$1" + local dir="$2" + + if [ ! -z "$path" ]; then + case ":$path:" in + *":$dir:"*) ;; # Already in the path + *) path="$path:$dir";; + esac + else + path="$dir" + fi + + echo "$path" +} + +function removeDuplicatesInPath { + local path="$1" + + for dir_ in "${path//:/ }"; do + if ls "$dir_" > /dev/null 2>&1; then + path=$(uniqueAddToPath "$path" "$dir_") + fi + done + + echo "$path" +} + +function getIncludePath { + local path="$1" + + path=$(echo "$path:" | sed 's/\/lib[^:]*:/\/include:/g') + + path=$(removeDuplicatesInPath "$path") + + echo "$path" +} + +function dirWithFileInPath { + local path="$1" + local filename="$2" + + if [ ! -z "$path" ]; then + for dir_ in ${path//:/ }; do + if ls "$dir_/$filename" > /dev/null 2>&1; then + echo "$dir_" + return + fi + done + fi + + echo "" +} + +function dirWithFileInIncludePath { + local path=$(getIncludePath "$1") + local filename="$2" + + if [ ! -z "$path" ]; then + for dir_ in ${path//:/ }; do + if ls "$dir_/$filename" > /dev/null 2>&1; then + echo "$dir_" + return + fi + done + fi + + echo "" +} + +function defaultIncludePath { + local mergedPaths="" + + mergedPaths+=":$OCCA_INCLUDE_PATH" + mergedPaths+=":$CPLUS_INCLUDE_PATH" + mergedPaths+=":$C_INCLUDE_PATH" + mergedPaths+=":$INCLUDEPATH" + mergedPaths+=":/opt/rocm*/include" + mergedPaths+=":/opt/rocm*/opencl/include" + mergedPaths+=":/usr/local/cuda*/include" + mergedPaths+=":/Developer/NVIDIA/CUDA*/include" + mergedPaths+=":/usr/local/cuda*/targets/*/include/" + mergedPaths+=":/opt/cuda*/include/" + mergedPaths+=":/usr/include" + + echo "${mergedPaths}" +} + +function defaultLibraryPath { + local mergedPaths="" + + mergedPaths+=":$OCCA_LIBRARY_PATH" + mergedPaths+=":$LD_LIBRARY_PATH" + mergedPaths+=":$DYLD_LIBRARY_PATH" + mergedPaths+=":/opt/rocm*/lib" + mergedPaths+=":/opt/rocm*/opencl/lib/*" + mergedPaths+=":/usr/local/cuda*/lib*" + mergedPaths+=":/usr/local/cuda*/lib*/stubs" + mergedPaths+=":/opt/cuda*/lib*" + mergedPaths+=":/lib:/usr/lib:/usr/lib32:/usr/lib64:" + mergedPaths+=":/usr/lib/*-gnu/" + + echo "${mergedPaths}" +} + +function dirWithLibrary { + local libName="lib$1.so" + local result="" + + result=$(dirWithFileInPath "$(defaultLibraryPath)" "$libName") + + if [ ! -z "$result" ]; then echo "$result"; return; fi + + if hash ldconfig 2> /dev/null; then + echo $(ldconfig -p | grep -m 1 "$libName" | sed 's/.*=>\(.*\/\).*/\1/g') + return + fi + + case "$(uname)" in + Darwin) + if ls "/System/Library/Frameworks/$1.framework" > /dev/null 2>&1; then + echo "Is A System/Library Framework" + return + fi + if ls /Library/Frameworks/$1.framework > /dev/null 2>&1; then + echo "Is A Library Framework" + return + fi;; + esac + + echo "" +} + +function dirWithHeader { + local filename="$1" + local result="" + + result=$(dirWithFileInPath "$(defaultIncludePath)" "$filename") + if [ ! -z "$result" ]; then echo "$result"; return; fi + + result=$(dirWithFileInIncludePath "$(defaultLibraryPath)" "$filename") + + if [ ! -z "$result" ]; then echo "$result"; return; fi + + echo "" +} + +function dirsWithHeaders { + local headers="$1" + local path="" + + if [ ! -z $headers ]; then + for header in "${headers//:/ }"; do + local inc=$(dirWithHeader $header) + + if [ ! -z $inc ]; then + path=$(uniqueAddToPath $path $inc) + else + echo "" + return + fi + done + fi + + echo "$path" +} + +function libraryFlags { + local libName="$1" + + local libDir=$(dirWithLibrary $libName) + local flags="" + local isAFramework=0 + + if [ -z "$libDir" ]; then echo ""; return; fi + + if [ "$libDir" == "Is A System/Library Framework" ]; then + flags="-framework $libName" + isAFramework=1 + elif [ "$libDir" == "Is A Library Framework" ]; then + flags="-F/Library/Frameworks -framework $libName" + isAFramework=1 + else + flags="-L$libDir -l$libName" + fi + + echo "$flags" +} + + +function headerFlags { + local headers="$1" + + local incDirs + local flags="" + + if [ ! -z "$headers" ]; then + incDirs=$(dirsWithHeaders "$headers") + + if [ -z "$incDirs" ]; then echo ""; return; fi + + incDirs="${incDirs%?}" # Remove the last : + flags="-I${incDirs//:/ -I}" # : -> -I + fi + + echo "$flags" +} +#======================================= + + +#---[ Compiler Information ]------------ +function compilerVendor { + local compiler="$1" + + local b_GNU=0 + local b_LLVM=1 + local b_Intel=2 + local b_Pathscale=3 + local b_IBM=4 + local b_PGI=5 + local b_HP=6 + local b_VisualStudio=7 + local b_Cray=8 + local b_PPC=9 + + local testFilename="${OCCA_SOURCE_SCRIPTS_DIR}/findCompilerVendor.cpp" + local binaryFilename="${OCCA_SOURCE_SCRIPTS_DIR}/findCompilerVendor" + + eval "${compiler}" "${testFilename}" -o "${binaryFilename}" > /dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + echo "Failed to build findCompilerVendor:" + eval "${compiler}" "${testFilename}" -o "${binaryFilename}" + fi + + eval "${binaryFilename}" + bit="$?" + + # C/C++ Compilers + case "$bit" in + "${b_GNU}") echo GCC ;; + "${b_LLVM}") echo LLVM ;; + "${b_Intel}") echo INTEL ;; + "${b_IBM}") echo IBM ;; + "${b_PGI}") echo PGI ;; + "${b_Pathscale}") echo PATHSCALE ;; + "${b_HP}") echo HP ;; + "${b_Cray}") echo CRAY ;; + "${b_VisualStudio}") echo VISUALSTUDIO ;; + "${b_PPC}") echo POWERPC ;; + *) echo N/A ;; + esac +} + +function compilerCpp11Flags { + local vendor=$(compilerVendor "$1") + + case "$vendor" in + GCC|LLVM|INTEL|PGI|POWERPC) + echo "-std=c++17";; + CRAY) echo "-hstd=c++17" ;; + IBM) echo "-qlanglvl=extended0x" ;; + # Unknown + PATHSCALE) echo "-std=c++17" ;; + # Unknown + HP) echo "-std=c++17" ;; + *) ;; + esac +} + +function compilerReleaseFlags { + local vendor=$(compilerVendor "$1") + + case "$vendor" in + GCC|LLVM) echo " -O3 -march=native -D __extern_always_inline=inline" ;; + POWERPC) echo " -O3 -mcpu=native -mtune=native -D __extern_always_inline=inline" ;; + INTEL) echo " -O3 -xHost" ;; + CRAY) echo " -O3 -h intrinsics -fast" ;; + IBM) echo " -O3 -qhot=simd" ;; + PGI) echo " -O3 -fast -Mipa=fast,inline -Msmartalloc" ;; + PATHSCALE) echo " -O3 -march=auto" ;; + HP) echo " +O3" ;; + *) ;; + esac +} + +function compilerDebugFlags { + local vendor=$(compilerVendor "$1") + + case "$vendor" in + N/A) ;; + *) echo " -g";; + esac +} + +function compilerPicFlag { + local vendor=$(compilerVendor "$1") + + case "$vendor" in + GCC|LLVM|INTEL|PATHSCALE|CRAY|PGI|POWERPC) + echo "-fPIC";; + IBM) echo "-qpic";; + HP) echo "+z";; + *) echo "" ;; + esac +} + +function compilerSharedFlag { + local vendor=$(compilerVendor "$1") + + case "$vendor" in + GCC|LLVM|INTEL|PATHSCALE|CRAY|PGI|POWERPC) + echo "-shared";; + IBM) echo "-qmkshrobj";; + HP) echo "-b";; + *) echo "";; + esac +} + +function compilerPthreadFlag { + echo "-lpthread" +} + +function compilerOpenMPFlag { + local vendor=$(compilerVendor "$1") + + case "$vendor" in + GCC|LLVM|POWERPC) echo "-fopenmp" ;; + INTEL|PATHSCALE) echo "-openmp" ;; + CRAY) echo "" ;; + IBM) echo "-qsmp" ;; + PGI) echo "-mp" ;; + HP) echo "+Oopenmp" ;; + *) echo "" ;; + esac +} + +function compilerSupportsOpenMP { + local compiler="$1" + local ompFlag=$(compilerOpenMPFlag "${compiler}") + + local filename="${OCCA_SOURCE_SCRIPTS_DIR}/compilerSupportsOpenMP.cpp" + local binary="${OCCA_SOURCE_SCRIPTS_DIR}/compilerSupportsOpenMP" + + rm -f "${binary}" + + # Test compilation + "${compiler}" "${ompFlag}" "${filename}" -o "${binary}" > /dev/null 2>&1 + + if [[ ! -a "${binary}" ]]; then + echo 0 + return + fi + + if [[ "$?" -eq 0 ]]; then + # Test binary + "${binary}" + + if [[ "$?" -eq 0 ]]; then + echo 1 + else + echo 0 + fi + else + echo 0 + fi + + if [ ! -z "${binary}" ]; then + rm -f "${binary}" + fi +} +#======================================= + + +#---[ Fortran Compiler Information ]---- +function fCompilerVendor { + local compiler="$1" + + local filename="${SCRIPTS_DIR}/compiler/findFortranCompilerVendor.F90" + local binary="${SCRIPTS_DIR}/compiler/findFortranCompilerVendor" + + rm -f "$binary" + "$compiler" "$filename" -o "$binary" > /dev/null 2>&1 + if [[ ! -a "$binary" ]]; then + echo "N/A" + return + fi + + vendor=$("$binary" 2>&1) + vendor=${vendor// /} + echo $vendor +} + +function fCompilerModuleDirFlag { + local vendor=$(fCompilerVendor "$1") + + case "$vendor" in + GCC|CRAY) echo "-J" ;; + INTEL|PGI|PATHSCALE) echo "-module" ;; + IBM) echo "-qmoddir" ;; + *) echo "" ;; + esac +} + +function fCompilerCppFlag { + local vendor=$(fCompilerVendor "$1") + + case "$vendor" in + GCC) echo "-lstdc++" ;; + CRAY) echo "" ;; + INTEL) echo "-cxxlib" ;; + PGI) echo "-pgc++libs" ;; + *) echo "" ;; + esac +} + +function fCompilerSupportsMPI { + local compiler="$1" + + local filename="${SCRIPTS_DIR}/compiler/fortranCompilerSupportsMPI.f90" + local binary="${SCRIPTS_DIR}/compiler/fortranCompilerSupportsMPI" + + rm -f "$binary" + + # Test compilation + "$compiler" "$filename" -o "$binary" > /dev/null 2>&1 + + if [[ ! -a "$binary" ]]; then + echo 0 + return + fi + + rm -f "$binary" + echo 1 +} +#======================================= + + +#---[ Commands ]------------------------ +function installOcca { + if [ -z "${PREFIX}" ]; then + return + fi + if [ -d "${PREFIX}" ]; then + echo "Warning: Install PREFIX=${PREFIX} already exists." + else + mkdir -p "${PREFIX}" + fi + cp -r bin "${PREFIX}" + cp -r include "${PREFIX}" + cp -r lib "${PREFIX}" +} +#======================================= diff --git a/level2/miniweather/cpp/build/check_output.sh b/level2/miniweather/cpp/build/check_output.sh new file mode 100755 index 0000000..7721ea7 --- /dev/null +++ b/level2/miniweather/cpp/build/check_output.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +showusage() { + printf "Usage:\n" + printf "./check_output.sh executable mass_relative_tolerance energy_relative_tolerance\n\n" +} +if [[ "$1" == "-h" || "$1" == "--help" ]]; then + showusage + exit 0 +fi +if [[ "$1" == "" ]]; then + printf "ERROR: Did not specify an executable\n\n" + showusage + exit -1 +fi +if [[ "$2" == "" ]]; then + printf "ERROR: Did not specify a mass tolerance\n\n" + showusage + exit -1 +fi +if [[ "$3" == "" ]]; then + printf "ERROR: Did not specify a total energy tolerance\n\n" + showusage + exit -1 +fi + +if [[ ! -f $1 ]]; then + exit -1 +fi +${TEST_MPI_COMMAND} $1 > ${1}.output || exit -1 + +dmass=`grep d_mass ${1}.output | awk '{print $2}'` +if [[ "$dmass" == "NaN" ]]; then + printf "Mass change is NaN\n\n" + exit -1 +fi +if (( `awk "BEGIN {val=$dmass; abs=val>0?val:-val; ret=abs<$2?0:-1; print ret}"` )); then + printf "ERROR: Mass change magnitude is too large\n\n" + printf "Change in mass: $dmass" + printf "Mass tolerance: $2" + exit -1 +fi + +dte=`grep d_te ${1}.output | awk '{print $2}'` +if [[ "$dte" == "NaN" ]]; then + printf "Total energy change is NaN\n\n" + exit -1 +fi +if (( `awk "BEGIN {val=$dte; ret=val<0?0:-1; print ret}"` )); then + printf "ERROR: Total energy change must be negative\n\n" + exit -1 +fi +if (( `awk "BEGIN {val=$dte; abs=val>0?val:-val; ret=abs<$3?0:-1; print ret}"` )); then + printf "ERROR: Total energy change magnitude is too large\n\n" + printf "Change in total energy: $dte" + printf "Total energy tolerance: $3" + exit -1 +fi + +rm -f ${1}.output + diff --git a/level2/tools/tests/run_all.sh b/level2/tools/tests/run_all.sh index 2f71e5d..54ea96c 100644 --- a/level2/tools/tests/run_all.sh +++ b/level2/tools/tests/run_all.sh @@ -14,5 +14,6 @@ run "deps markers" bash "$HERE/test_deps_markers.sh" run "env dep profiles" bash "$HERE/test_env_profiles.sh" run "launcher dry-run" bash "$HERE/test_launcher_dryrun.sh" run "run.sh guards" bash "$HERE/test_run_guards.sh" +run "vendored completeness" bash "$HERE/test_vendored_completeness.sh" [ $rc -eq 0 ] && echo "ALL TEST GROUPS PASSED" || echo "SOME TEST GROUPS FAILED" exit $rc diff --git a/level2/tools/tests/test_vendored_completeness.sh b/level2/tools/tests/test_vendored_completeness.sh new file mode 100755 index 0000000..0f90311 --- /dev/null +++ b/level2/tools/tests/test_vendored_completeness.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Clean-clone completeness of the vendored Level 2 source copies: files that upstream keeps in a directory +# named build/ are swallowed by the repository's global `build/` ignore rule unless explicitly re-included +# (.gitignore negations). These checks fail in a fresh clone when such a file is missing -- the situation +# in which hipBone and miniWeather could not be built on 2026-09-15. +set -u +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; R="$(cd "$HERE/../../.." && pwd)" +pass=0; failn=0 +ok() { echo "ok $*"; pass=$((pass+1)); } +bad() { echo "FAIL $*"; failn=$((failn+1)); } +for f in level2/hipbone/occa/scripts/build/Makefile level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp \ + level2/hipbone/occa/scripts/build/compiledDefinesTemplate.hpp.in level2/hipbone/occa/scripts/build/shellTools.sh \ + level2/hipbone/occa/scripts/build/Make.fortran level2/hipbone/occa/scripts/build/Make.fortran_rules \ + level2/miniweather/cpp/build/check_output.sh; do + [ -f "$R/$f" ] && ok "present: $f" || bad "missing vendored upstream file: $f" +done +# the two directories must be re-included by .gitignore, so `git add` keeps tracking them +for d in level2/hipbone/occa/scripts/build/ level2/miniweather/cpp/build/; do + /usr/bin/grep -qxF "!$d" "$R/.gitignore" && ok ".gitignore re-includes $d" || bad ".gitignore does not re-include $d" +done +# in a git checkout: no ignored file may exist under a vendored Level 2 tree (build products live under /build/) +if git -C "$R" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + stray="$(git -C "$R" ls-files --others --ignored --exclude-standard -- level2 2>/dev/null | /usr/bin/grep -v -E '__pycache__|\.pyc$|^level2/tools/mpi_cuda_check/mpi_cuda_check$' || true)" + [ -z "$stray" ] && ok "no ignored file hides inside a vendored level2/ tree" || bad "ignored files inside level2/ (would be lost in a clean clone): $stray" +fi +echo; echo "test_vendored_completeness: $pass passed, $failn failed"; [ "$failn" -eq 0 ]