[starpu-devel] Compilation and autoconf issues with an NVHPC environment

[Cedric Augonnet <caugonnet@nvidia.com>]

Dear StarPU folks,

I hope you are all doing well, and i apologize in advance for the cryptic mail ! I was trying to compile StarPU (master, commit ac2e893eb1) on my desktop, which relies on NVHPC 24.5 and had various issues ... I don't know if this is supported at the moment ?

At first, i tried to compile using CC=nvc which is set by NVHPC module, but i've run into various compilation errors. For the record, nvc is not nvcc, it's a different compiler. I started with C parsing issues with such a workaround (I did not dig for the root cause)

diff --git a/src/parallel_worker/starpu_parallel_worker_create.c b/src/parallel_worker/starpu_parallel_worker_create.c

index a3317cf630..793f7e042a 100644

--- a/src/parallel_worker/starpu_parallel_worker_create.c

+++ b/src/parallel_worker/starpu_parallel_worker_create.c

@@ -171,7 +171,8 @@ struct starpu_parallel_worker_config *_starpu_parallel_worker_init_varg(hwloc_ob

                }

                else if (arg_type == STARPU_PARALLEL_WORKER_CREATE_FUNC)

                {

-                       machine->params->create_func = va_arg(varg_list, void (*)(void*));

+                       typedef void (*fn)(void*);

+                       machine->params->create_func = va_arg(varg_list, fn);

                }

                else if (arg_type == STARPU_PARALLEL_WORKER_CREATE_FUNC_ARG)

                {

I disabled fortran because i did not want to fix the nvfortran errors as well today ... but there were some which forced me to comment these lines out

diff --git a/include/fstarpu_mod.f90 b/include/fstarpu_mod.f90

index cef51e3fbe..0015804506 100644

--- a/include/fstarpu_mod.f90

+++ b/include/fstarpu_mod.f90

@@ -2571,7 +2571,7 @@ module fstarpu_mod

                         FSTARPU_OPENCL_WORKER   = fstarpu_get_constant(C_CHAR_"FSTARPU_OPENCL_WORKER"//C_NULL_CHAR)

                         FSTARPU_ANY_WORKER   = fstarpu_get_constant(C_CHAR_"FSTARPU_ANY_WORKER"//C_NULL_CHAR)

 

-                        FSTARPU_NMAXBUFS   = int(p_to_ip(fstarpu_get_constant(C_CHAR_"FSTARPU_NMAXBUFS"//C_NULL_CHAR)),c_int)

+                        !!FSTARPU_NMAXBUFS   = int(p_to_ip(fstarpu_get_constant(C_CHAR_"FSTARPU_NMAXBUFS"//C_NULL_CHAR)),c_int)

 

                         FSTARPU_SCHED_CTX_POLICY_NAME    = &

                             fstarpu_get_constant(C_CHAR_"FSTARPU_SCHED_CTX_POLICY_NAME"//C_NULL_CHAR)

@@ -2658,8 +2658,8 @@ module fstarpu_mod

                         FSTARPU_SZ_COMPLEX8        = sz_to_p(c_sizeof(FSTARPU_SZ_COMPLEX8_dummy))

                         FSTARPU_SZ_COMPLEX8        = sz_to_p(c_sizeof(FSTARPU_SZ_COMPLEX8_dummy))

 

-                        FSTARPU_DEFAULT_PRIO =  int(p_to_ip(&

-                                fstarpu_get_constant(C_CHAR_"FSTARPU_DEFAULT_PRIO"//C_NULL_CHAR)),c_int)

+                        ! FSTARPU_DEFAULT_PRIO =  int(p_to_ip(&

+                        !         fstarpu_get_constant(C_CHAR_"FSTARPU_DEFAULT_PRIO"//C_NULL_CHAR)),c_int)

 

                         ! Initialize StarPU

                         if (c_associated(conf)) then 

After compiling, I'm was getting false positives on some warnings at runtime, probably because NVC does not define the same not-so-portable macros as GCC. For example in spite of this, i had a false positive warning that i had compiled in debug mode.

checking whether debug mode should be enabled... no

#ifdef __GNUC__

#ifndef __OPTIMIZE__

»________STARPU_DISP("Warning: StarPU was configured with --enable-debug (-O0), and is thus not optimized\n");

#endif

#endif

Then I started again, using GNU compilers instead, to avoid these pitfalls, and I found another issue in StarPU with NVHPC. 

When using this :

CXX=g++ FC=gfortran CC=gcc ./configure --disable-fortran --disable-mpi --disable-opencl --prefix=$PWD/build/

I saw this (and i was getting weird compilation warnings from an old version of CUFFT headers.)

checking whether CUDA library is available in -L/usr/local/cuda/lib64... 

checking for main in -lcudart... yes

checking for nvcc... /opt/nvidia/hpc_sdk/Linux_x86_64/24.5/compilers/bin/nvcc

While the nvcc was properly taken from my PATH variable, libraries and headers where taken from /usr/local/cuda .... Instead of $NVHPC_ROOT. So i've added that to the candidates when looking for a path in autoconf.

One other issue was that libcuda.so, libcudart.so and math libs are not at the same location anymore. So we cannot simply look for libcudart.so, and assume CUDA is properly configured : we also need to look for libcuda.so. I've implemented that in the autoconf file, and ensured we would update LDFLAGS for all libs one by one. The current meaning of the STARPU_CUDA_LDFLAGS in the autoconf file is quite unclear, we erase it when checking for CUBLAS for example, but this might only be needed to find CUBLAS, CUFFT, CURAND, and not the CUDA runtime ?

Now at runtime, the code does run properly (it would deadlock before),

$ ./build/lib/starpu/examples/axpy

[starpu][_starpu_init_cuda_config] Warning: could not find location of CUDA0, do you have the hwloc CUDA plugin installed?

[starpu][initialize_lws_policy] Warning: you are running the default lws scheduler, which is not a very smart scheduler, while the system has GPUs or several memory nodes. Make sure to read the StarPU documentation about adding performance models in order to be able to use the dmda or dmdas scheduler instead.

BEFORE x[0] = 1.00

BEFORE y[0] = 4.00

timing -> 4914.31 us 40967.42 MB/s

AFTER y[0] = 7.41 (ALPHA = 3.41)

But i'm still having this which suggests we should use some rpath ? (and the version of cusolver is let to the interested reader)

$ ldd ./build/lib/starpu/examples/axpy|grep cuda

      libcublas.so.12 => /usr/local/cuda/lib64/libcublas.so.12 (0x00007ff198800000)

      libcudart.so.12 => /usr/local/cuda/lib64/libcudart.so.12 (0x00007ff195e00000)

      libcusolver.so.11 => /usr/local/cuda/lib64/libcusolver.so.11 (0x00007ff18ec00000)

      libcublasLt.so.12 => /usr/local/cuda/lib64/libcublasLt.so.12 (0x00007ff16ba00000)

      libnvJitLink.so.12 => /usr/local/cuda/lib64/libnvJitLink.so.12 (0x00007ff168200000)

      libcusparse.so.12 => /usr/local/cuda/lib64/libcusparse.so.12 (0x00007ff158400000)

Given the huge amont of fun we always get with autoconf, I wish I could help more, but I'm a bit out of fashion 🙂

$ git clone git@gitlab.inria.fr:starpu/starpu.git
Cloning into 'starpu'...
remote:
remote: ========================================================================
remote:
remote: Your account has been blocked.
remote:
remote: ========================================================================

Best,

Cédric

diff --git a/configure.ac b/configure.ac
index 3fbed3b9be..968edfc740 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1446,7 +1446,7 @@ AC_DEFUN([STARPU_CHECK_CUDA_L],
 [
     __cuda_L=$1
     SAVED_LDFLAGS="${LDFLAGS}"
-    STARPU_CUDA_LDFLAGS="${__cuda_L}"
+    STARPU_CUDA_LDFLAGS="${STARPU_CUDA_LDFLAGS} ${__cuda_L}"
     if test "$__cuda_L" = "" ; then
         AC_MSG_CHECKING(whether CUDA library is available in default location)
     else
@@ -1456,10 +1456,29 @@ AC_DEFUN([STARPU_CHECK_CUDA_L],
     LDFLAGS="${SAVED_LDFLAGS} ${__cuda_L}"
     AC_HAVE_LIBRARY([cudart],[have_valid_cuda=yes],[have_valid_cuda=no])
     unset ac_cv_lib_cudart_main
+    LDFLAGS="${SAVED_LDFLAGS}"
     if test "$have_valid_cuda" = yes ; then
-	LDFLAGS="${SAVED_LDFLAGS} ${STARPU_CUDA_LDFLAGS}"
+	LDFLAGS="${LDFLAGS} ${STARPU_CUDA_LDFLAGS}"
     fi
+])
+AC_DEFUN([STARPU_CHECK_CUDA_DRIVER_L],
+[
+    __cuda_L=$1
+    SAVED_LDFLAGS="${LDFLAGS}"
+    STARPU_CUDA_LDFLAGS="${STARPU_CUDA_LDFLAGS} ${__cuda_L}"
+    if test "$__cuda_L" = "" ; then
+        AC_MSG_CHECKING(whether CUDA driver is available in default location)
+    else
+        AC_MSG_CHECKING(whether CUDA driver is available in $__cuda_L)
+    fi
+    AC_MSG_RESULT()
+    LDFLAGS="${SAVED_LDFLAGS} ${__cuda_L}"
+    AC_HAVE_LIBRARY([cuda],[have_valid_cuda_driver=yes],[have_valid_cuda_driver=no])
+    unset ac_cv_lib_cuda_main
     LDFLAGS="${SAVED_LDFLAGS}"
+    if test "$have_valid_cuda_driver" = yes ; then
+	LDFLAGS="${LDFLAGS} ${STARPU_CUDA_LDFLAGS}"
+    fi
 ])
 AC_DEFUN([STARPU_CHECK_CUDA],
 [
@@ -1509,6 +1528,16 @@ AC_DEFUN([STARPU_CHECK_CUDA],
 			break
 		    fi
 		done
+		for __cuda_libdir in lib64 lib64/stubs/ lib lib/x64 lib/Win32 ; do
+		    STARPU_CHECK_CUDA_DRIVER_L("-L${__cuda_dir}/${__cuda_libdir}")
+		    if test "$have_valid_cuda_driver" = yes ; then
+			break
+		    fi
+		done
+		if test "$have_valid_cuda_driver" = no ; then
+			have_valid_cuda=no
+		fi
+
 	    else
 		STARPU_CHECK_CUDA_L("")
 	    fi
@@ -1572,7 +1601,7 @@ AC_DEFUN([STARPU_CHECK_CUBLAS_L],
 [
     __cublas_L=$1
     SAVED_LDFLAGS="${LDFLAGS}"
-    STARPU_CUDA_LDFLAGS="${__cublas_L}"
+    STARPU_CUDA_LDFLAGS="${STARPU_CUDA_LDFLAGS} ${__cublas_L}"
     if test "$__cublas_L" = "" ; then
         AC_MSG_CHECKING(whether CUBLAS library is available in default location)
     else
@@ -1582,10 +1611,10 @@ AC_DEFUN([STARPU_CHECK_CUBLAS_L],
     LDFLAGS="${SAVED_LDFLAGS} ${__cublas_L}"
     AC_HAVE_LIBRARY([cublas],[have_valid_cublas=yes],[have_valid_cublas=no])
     unset ac_cv_lib_cublas_main
+    LDFLAGS="${SAVED_LDFLAGS}"
     if test "$have_valid_cublas" = yes ; then
-	LDFLAGS="${SAVED_LDFLAGS} ${STARPU_CUDA_LDFLAGS}"
+	LDFLAGS="${LDFLAGS} ${STARPU_CUDA_LDFLAGS}"
     fi
-    LDFLAGS="${SAVED_LDFLAGS}"
 ])
 AC_DEFUN([STARPU_CHECK_CUBLAS],
 [
@@ -1653,7 +1682,7 @@ if test x$enable_cuda = xyes -o x$enable_cuda = xmaybe; then
 	STARPU_CHECK_CUDA("$CUDA_ROOT", "$CUDA_PATH", "$CUDA_INC_PATH", "$CUDA_LIB_PATH")
     fi
     if test "$have_valid_cuda" = "no" ; then
-	for f in "/usr/local/cuda" "/c/cuda" "/cygdrive/c/cuda" "/opt/cuda" "$CUDA_ROOT" "$CUDA_PATH" "$CUDA_INC_PATH/.." "$CUDA_INC/.." "$CUDA_BIN/.." "$CUDA_SDK/.." "$CUDA_INSTALL_PATH" "$CUDA_TOOLKIT"; do
+	for f in "$NVHPC_ROOT/cuda/" "/usr/local/cuda" "/c/cuda" "/cygdrive/c/cuda" "/opt/cuda" "$CUDA_ROOT" "$CUDA_PATH" "$CUDA_INC_PATH/.." "$CUDA_INC/.." "$CUDA_BIN/.." "$CUDA_SDK/.." "$CUDA_INSTALL_PATH" "$CUDA_TOOLKIT"; do
 	    if test -n "$f" ; then
 		STARPU_CHECK_CUDA("$f", "no", "no")
 		if test "$have_valid_cuda" = "yes" ; then
@@ -1667,7 +1696,7 @@ if test x$enable_cuda = xyes -o x$enable_cuda = xmaybe; then
     if test "$have_valid_cuda" = "yes" ; then
         STARPU_CHECK_CUBLAS("$cublas_dir", "$cublas_include_dir", "$cublas_lib_dir")
 	if test "$have_valid_cublas" = "no" ; then
-	    for f in "" "/usr/local/cuda" "/c/cuda" "/cygdrive/c/cuda" "/opt/cuda" "$CUDA_ROOT" "$CUDA_PATH" "$CUDA_INC_PATH/.." "$CUDA_INC/.." "$CUDA_BIN/.." "$CUDA_SDK/.." "$CUDA_INSTALL_PATH" "$CUDA_TOOLKIT"; do
+	    for f in "" "$NVHPC_ROOT/math_libs/" "/usr/local/cuda" "/c/cuda" "/cygdrive/c/cuda" "/opt/cuda" "$CUDA_ROOT" "$CUDA_PATH" "$CUDA_INC_PATH/.." "$CUDA_INC/.." "$CUDA_BIN/.." "$CUDA_SDK/.." "$CUDA_INSTALL_PATH" "$CUDA_TOOLKIT"; do
 	        if test -n "$f" ; then
 		    STARPU_CHECK_CUBLAS("$f", "no", "no")
 		    if test "$have_valid_cublas" = "yes" ; then