From e21720a5a346ecb9d0163ba75ce8f1383eb50638 Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Wed, 30 Jun 2021 19:09:27 -0700 Subject: [PATCH 1/9] [ROCm] Changes to enable build for ROCm platform --- .gitmodules | 3 + CMakeLists.txt | 10 ++ cmake/Hip.cmake | 162 ++++++++++++++++++ cmake/Hipify.cmake | 62 +++++++ cmake/Options.cmake | 6 + tensorpipe/CMakeLists.txt | 60 +++++-- tensorpipe/benchmark/CMakeLists.txt | 31 +++- tensorpipe/test/CMakeLists.txt | 56 ++++-- tools/amd_build/build_amd.py | 60 +++++++ .../amd_build/replace_cuda_with_hip_files.py | 47 +++++ 10 files changed, 459 insertions(+), 38 deletions(-) create mode 100644 cmake/Hip.cmake create mode 100644 cmake/Hipify.cmake create mode 100755 tools/amd_build/build_amd.py create mode 100755 tools/amd_build/replace_cuda_with_hip_files.py diff --git a/.gitmodules b/.gitmodules index 7207ef9b6..0777fa968 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,3 +11,6 @@ [submodule "third_party/libnop"] path = third_party/libnop url = https://github.com/google/libnop.git +[submodule "third_party/hipify"] + path = third_party/hipify + url = https://github.com/ROCmSoftwarePlatform/hipify-torch.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 640054ee3..174746a7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,16 @@ include(Sanitize) # Misc checks to cope with various compiler modes. include(MiscCheck) +# ROCm related +if (TP_USE_ROCM) + include(Hip) + if(TP_HAVE_HIP) + include(Hipify) + else() + message(FATAL_ERROR "Not able to find HIP installation, so cant compile with ROCm support.") + endif() +endif() + add_subdirectory(tensorpipe) install(EXPORT TensorpipeTargets diff --git a/cmake/Hip.cmake b/cmake/Hip.cmake new file mode 100644 index 000000000..79a67a014 --- /dev/null +++ b/cmake/Hip.cmake @@ -0,0 +1,162 @@ +set(TP_HAVE_HIP FALSE) + +IF(NOT DEFINED ENV{ROCM_PATH}) + SET(ROCM_PATH /opt/rocm) +ELSE() + SET(ROCM_PATH $ENV{ROCM_PATH}) +ENDIF() + +# HIP_PATH +IF(NOT DEFINED ENV{HIP_PATH}) + SET(HIP_PATH ${ROCM_PATH}/hip) +ELSE() + SET(HIP_PATH $ENV{HIP_PATH}) +ENDIF() + +IF(NOT EXISTS ${HIP_PATH}) + return() +ENDIF() + +# HCC_PATH +IF(NOT DEFINED ENV{HCC_PATH}) + SET(HCC_PATH ${ROCM_PATH}/hcc) +ELSE() + SET(HCC_PATH $ENV{HCC_PATH}) +ENDIF() + +# HSA_PATH +IF(NOT DEFINED ENV{HSA_PATH}) + SET(HSA_PATH ${ROCM_PATH}/hsa) +ELSE() + SET(HSA_PATH $ENV{HSA_PATH}) +ENDIF() + +# ROCBLAS_PATH +IF(NOT DEFINED ENV{ROCBLAS_PATH}) + SET(ROCBLAS_PATH ${ROCM_PATH}/rocblas) +ELSE() + SET(ROCBLAS_PATH $ENV{ROCBLAS_PATH}) +ENDIF() + +# ROCSPARSE_PATH +IF(NOT DEFINED ENV{ROCSPARSE_PATH}) + SET(ROCSPARSE_PATH ${ROCM_PATH}/rocsparse) +ELSE() + SET(ROCSPARSE_PATH $ENV{ROCSPARSE_PATH}) +ENDIF() + +# ROCFFT_PATH +IF(NOT DEFINED ENV{ROCFFT_PATH}) + SET(ROCFFT_PATH ${ROCM_PATH}/rocfft) +ELSE() + SET(ROCFFT_PATH $ENV{ROCFFT_PATH}) +ENDIF() + +# HIPSPARSE_PATH +IF(NOT DEFINED ENV{HIPSPARSE_PATH}) + SET(HIPSPARSE_PATH ${ROCM_PATH}/hipsparse) +ELSE() + SET(HIPSPARSE_PATH $ENV{HIPSPARSE_PATH}) +ENDIF() + +# THRUST_PATH +IF(DEFINED ENV{THRUST_PATH}) + SET(THRUST_PATH $ENV{THRUST_PATH}) +ELSE() + SET(THRUST_PATH ${ROCM_PATH}/include) +ENDIF() + +# HIPRAND_PATH +IF(NOT DEFINED ENV{HIPRAND_PATH}) + SET(HIPRAND_PATH ${ROCM_PATH}/hiprand) +ELSE() + SET(HIPRAND_PATH $ENV{HIPRAND_PATH}) +ENDIF() + +# ROCRAND_PATH +IF(NOT DEFINED ENV{ROCRAND_PATH}) + SET(ROCRAND_PATH ${ROCM_PATH}/rocrand) +ELSE() + SET(ROCRAND_PATH $ENV{ROCRAND_PATH}) +ENDIF() + +# MIOPEN_PATH +IF(NOT DEFINED ENV{MIOPEN_PATH}) + SET(MIOPEN_PATH ${ROCM_PATH}/miopen) +ELSE() + SET(MIOPEN_PATH $ENV{MIOPEN_PATH}) +ENDIF() + +IF(NOT DEFINED ENV{TP_ROCM_ARCH}) + SET(TP_ROCM_ARCH gfx900;gfx906;gfx908) +ELSE() + SET(TP_ROCM_ARCH $ENV{TP_ROCM_ARCH}) +ENDIF() + +# Add HIP to the CMAKE Module Path +set(CMAKE_MODULE_PATH ${HIP_PATH}/cmake ${CMAKE_MODULE_PATH}) + +# Disable Asserts In Code (Can't use asserts on HIP stack.) +ADD_DEFINITIONS(-DNDEBUG) + +# Find the HIP Package +find_package(HIP 1.0) + +IF(HIP_FOUND) + set(TP_HAVE_HIP TRUE) + + if(HIP_COMPILER STREQUAL clang) + set(hip_library_name amdhip64) + else() + set(hip_library_name hip_hcc) + endif() + message("HIP library name: ${hip_library_name}") + + set(CMAKE_HCC_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + set(CMAKE_HCC_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + FIND_LIBRARY(TP_HIP_HCC_LIBRARIES ${hip_library_name} HINTS ${HIP_PATH}/lib) + + list(APPEND HIP_CXX_FLAGS -fPIC) + list(APPEND HIP_CXX_FLAGS -D__HIP_PLATFORM_HCC__=1) + list(APPEND HIP_CXX_FLAGS -D__HIP_NO_HALF_OPERATORS__=1) + list(APPEND HIP_CXX_FLAGS -D__HIP_NO_HALF_CONVERSIONS__=1) + list(APPEND HIP_CXX_FLAGS -DHIP_VERSION=${HIP_VERSION_MAJOR}) + list(APPEND HIP_CXX_FLAGS -Wno-macro-redefined) + list(APPEND HIP_CXX_FLAGS -Wno-inconsistent-missing-override) + list(APPEND HIP_CXX_FLAGS -Wno-exceptions) + list(APPEND HIP_CXX_FLAGS -Wno-shift-count-negative) + list(APPEND HIP_CXX_FLAGS -Wno-shift-count-overflow) + list(APPEND HIP_CXX_FLAGS -Wno-unused-command-line-argument) + list(APPEND HIP_CXX_FLAGS -Wno-duplicate-decl-specifier) + + set(HIP_CLANG_FLAGS ${HIP_CXX_FLAGS}) + # Ask hcc to generate device code during compilation so we can use + # host linker to link. + list(APPEND HIP_CLANG_FLAGS -fno-gpu-rdc) + list(APPEND HIP_CLANG_FLAGS -Wno-defaulted-function-deleted) + foreach(tp_rocm_arch ${TP_ROCM_ARCH}) + list(APPEND HIP_CLANG_FLAGS --amdgpu-target=${tp_rocm_arch}) + endforeach() + + set(hip_DIR ${HIP_PATH}/lib/cmake/hip) + set(hsa-runtime64_DIR ${ROCM_PATH}/lib/cmake/hsa-runtime64) + set(AMDDeviceLibs_DIR ${ROCM_PATH}/lib/cmake/AMDDeviceLibs) + set(amd_comgr_DIR ${ROCM_PATH}/lib/cmake/amd_comgr) + set(rocrand_DIR ${ROCRAND_PATH}/lib/cmake/rocrand) + set(hiprand_DIR ${HIPRAND_PATH}/lib/cmake/hiprand) + set(rocblas_DIR ${ROCBLAS_PATH}/lib/cmake/rocblas) + set(miopen_DIR ${MIOPEN_PATH}/lib/cmake/miopen) + set(rocfft_DIR ${ROCFFT_PATH}/lib/cmake/rocfft) + set(hipfft_DIR ${HIPFFT_PATH}/lib/cmake/hipfft) + set(hipsparse_DIR ${HIPSPARSE_PATH}/lib/cmake/hipsparse) + set(rccl_DIR ${RCCL_PATH}/lib/cmake/rccl) + set(rocprim_DIR ${ROCPRIM_PATH}/lib/cmake/rocprim) + set(hipcub_DIR ${HIPCUB_PATH}/lib/cmake/hipcub) + set(rocthrust_DIR ${ROCTHRUST_PATH}/lib/cmake/rocthrust) + set(ROCclr_DIR ${ROCM_PATH}/rocclr/lib/cmake/rocclr) + + find_package(hip REQUIRED) + + set(TP_HIP_INCLUDE ${ROCM_PATH}/include ${TP_HIP_INCLUDE}) + set(TP_HIP_INCLUDE ${hip_INCLUDE_DIRS} $ $ ${TP_HIP_INCLUDE}) +ENDIF() diff --git a/cmake/Hipify.cmake b/cmake/Hipify.cmake new file mode 100644 index 000000000..475e369b1 --- /dev/null +++ b/cmake/Hipify.cmake @@ -0,0 +1,62 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# cmake file to trigger hipify + +function(write_file_list FILE_SUFFIX INPUT_LIST) + message(STATUS "Writing ${FILE_SUFFIX} into file - file_${FILE_SUFFIX}.txt") + set(_FULL_FILE_NAME "${CMAKE_BINARY_DIR}/cuda_to_hip_list_${FILE_SUFFIX}.txt") + file(WRITE ${_FULL_FILE_NAME} "") + foreach(_SOURCE_FILE ${INPUT_LIST}) + file(APPEND ${_FULL_FILE_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/${_SOURCE_FILE}) + file(APPEND ${_FULL_FILE_NAME} "\n") + endforeach() +endfunction() + +function(get_file_list FILE_SUFFIX OUTPUT_LIST) + set(_FULL_FILE_NAME "${CMAKE_BINARY_DIR}/cuda_to_hip_list_${FILE_SUFFIX}.txt") + file(STRINGS ${_FULL_FILE_NAME} _FILE_LIST) + set(${OUTPUT_LIST}_HIP ${_FILE_LIST} PARENT_SCOPE) +endfunction() + +function(update_list_with_hip_files FILE_SUFFIX) + set(_SCRIPTS_DIR ${PROJECT_SOURCE_DIR}/tools/amd_build) + set(_FULL_FILE_NAME "${CMAKE_BINARY_DIR}/cuda_to_hip_list_${FILE_SUFFIX}.txt") + set(_EXE_COMMAND + ${_SCRIPTS_DIR}/replace_cuda_with_hip_files.py + --io-file ${_FULL_FILE_NAME} + --dump-dict-directory ${CMAKE_BINARY_DIR}) + execute_process( + COMMAND ${_EXE_COMMAND} + RESULT_VARIABLE _return_value) + if (NOT _return_value EQUAL 0) + message(FATAL_ERROR "Failed to get the list of hipified files!") + endif() +endfunction() + +function(get_hipified_list FILE_SUFFIX INPUT_LIST OUTPUT_LIST) + write_file_list("${FILE_SUFFIX}" "${INPUT_LIST}") + update_list_with_hip_files("${FILE_SUFFIX}") + get_file_list("${FILE_SUFFIX}" __temp_srcs) + set(${OUTPUT_LIST} ${__temp_srcs_HIP} PARENT_SCOPE) +endfunction() + + +set(HIPIFY_SCRIPTS_DIR ${PROJECT_SOURCE_DIR}/tools/amd_build) +set(HIPIFY_COMMAND + ${HIPIFY_SCRIPTS_DIR}/build_amd.py + --project-directory ${PROJECT_SOURCE_DIR} + --output-directory ${PROJECT_SOURCE_DIR} + --dump-dict-directory ${CMAKE_BINARY_DIR} +) + +execute_process( + COMMAND ${HIPIFY_COMMAND} + RESULT_VARIABLE hipify_return_value +) +if (NOT hipify_return_value EQUAL 0) + message(FATAL_ERROR "Failed to hipify files!") +endif() diff --git a/cmake/Options.cmake b/cmake/Options.cmake index 10e09c94e..4a6da4458 100644 --- a/cmake/Options.cmake +++ b/cmake/Options.cmake @@ -31,6 +31,12 @@ endmacro() # TODO: Default to ON if CUDA available. option(TP_USE_CUDA "Enable support for CUDA tensors" OFF) +option(TP_USE_ROCM "Enable support for ROCM tensors" OFF) + +# if both TP_USE_CUDA and TP_USE_ROCM is set then break +if(TP_USE_CUDA AND TP_USE_ROCM) + message(FATAL_ERROR "Tensorpipe can be built either for CUDA or ROCm, TP_USE_CUDA and TP_USE_ROCM both are set, erroring out!!!!") +endif() # Optional features option(TP_BUILD_BENCHMARK "Build benchmarks" OFF) diff --git a/tensorpipe/CMakeLists.txt b/tensorpipe/CMakeLists.txt index 5c3606471..c8beddb93 100644 --- a/tensorpipe/CMakeLists.txt +++ b/tensorpipe/CMakeLists.txt @@ -100,6 +100,7 @@ list(APPEND TP_SRCS list(APPEND TP_PUBLIC_HDRS channel/mpt/factory.h) + ## Transports ### uv @@ -219,9 +220,9 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/config.h DESTINATION ${TP_INSTALL_INCLUDEDIR}/tensorpipe) -## CUDA +## CUDA AND ROCM -if(TP_USE_CUDA) +if(TP_USE_CUDA OR TP_USE_ROCM) # TP_SRCS is the list of source files that we need to build libtensorpipe. set(TP_CUDA_SRCS) @@ -234,9 +235,17 @@ if(TP_USE_CUDA) # TP_INCLUDE_DIRS is list of include path to be used set(TP_CUDA_INCLUDE_DIRS) - find_package(CUDA REQUIRED) - list(APPEND TP_CUDA_LINK_LIBRARIES ${CUDA_LIBRARIES}) - list(APPEND TP_CUDA_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS}) + if (TP_USE_CUDA) + set(TP_GPU_LIB_NAME "tensorpipe_cuda") + find_package(CUDA REQUIRED) + list(APPEND TP_CUDA_LINK_LIBRARIES ${CUDA_LIBRARIES}) + list(APPEND TP_CUDA_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS}) + elseif (TP_USE_ROCM) + set(TP_GPU_LIB_NAME "tensorpipe_hip") + # Finding of HIP package is already before hipifying the files + list(APPEND TP_CUDA_LINK_LIBRARIES ${TP_HIP_HCC_LIBRARIES}) + list(APPEND TP_CUDA_INCLUDE_DIRS ${TP_HIP_INCLUDE}) + endif() list(APPEND TP_CUDA_SRCS common/cuda_buffer.cc) @@ -246,12 +255,16 @@ if(TP_USE_CUDA) ### cuda_xth - list(APPEND TP_CUDA_SRCS - channel/cuda_xth/channel_impl.cc - channel/cuda_xth/context_impl.cc - channel/cuda_xth/factory.cc) - list(APPEND TP_CUDA_PUBLIC_HDRS - channel/cuda_xth/factory.h) + tp_conditional_backend( + TP_ENABLE_HIP_XTH "Enable HIP XTH communication channel" "TP_USE_ROCM") + if(TP_ENABLE_HIP_XTH OR TP_USE_CUDA) + list(APPEND TP_CUDA_SRCS + channel/cuda_xth/channel_impl.cc + channel/cuda_xth/context_impl.cc + channel/cuda_xth/factory.cc) + list(APPEND TP_CUDA_PUBLIC_HDRS + channel/cuda_xth/factory.h) + endif() ### cuda_basic @@ -267,7 +280,9 @@ if(TP_USE_CUDA) tp_conditional_backend( TP_ENABLE_CUDA_IPC "Enable CUDA inter-process communication channel" "TP_USE_CUDA") - if(TP_ENABLE_CUDA_IPC) + tp_conditional_backend( + TP_ENABLE_HIP_IPC "Enable HIP inter-process communication channel" "TP_USE_ROCM") + if(TP_ENABLE_CUDA_IPC OR TP_ENABLE_HIP_IPC) list(APPEND TP_CUDA_SRCS channel/cuda_ipc/channel_impl.cc channel/cuda_ipc/context_impl.cc @@ -281,7 +296,9 @@ if(TP_USE_CUDA) tp_conditional_backend( TP_ENABLE_CUDA_GDR "Enable CUDA GpuDirect (InfiniBand) channel" "LINUX") - if(TP_ENABLE_CUDA_GDR) + tp_conditional_backend( + TP_ENABLE_HIP_GDR "Enable HIP GpuDirect (InfiniBand) channel" "LINUX") + if((TP_ENABLE_CUDA_GDR AND TP_USE_CUDA) OR (TP_ENABLE_HIP_GDR AND TP_USE_ROCM)) list(APPEND TP_CUDA_SRCS common/ibv.cc channel/cuda_gdr/channel_impl.cc @@ -293,19 +310,24 @@ if(TP_USE_CUDA) set(TENSORPIPE_HAS_CUDA_GDR_CHANNEL 1) endif() + if(TP_USE_ROCM) + get_hipified_list("TP_CUDA_SRCS" "${TP_CUDA_SRCS}" TP_CUDA_SRCS) + get_hipified_list("TP_CUDA_PUBLIC_HDRS" "${TP_CUDA_PUBLIC_HDRS}" TP_CUDA_PUBLIC_HDRS) + endif() + configure_file(config_cuda.h.in config_cuda.h) - add_library(tensorpipe_cuda ${TP_STATIC_OR_SHARED} ${TP_CUDA_SRCS}) + add_library(${TP_GPU_LIB_NAME} ${TP_STATIC_OR_SHARED} ${TP_CUDA_SRCS}) if(BUILD_SHARED_LIBS) - set_target_properties(tensorpipe_cuda PROPERTIES POSITION_INDEPENDENT_CODE 1) + set_target_properties(${TP_GPU_LIB_NAME} PROPERTIES POSITION_INDEPENDENT_CODE 1) endif() - target_link_libraries(tensorpipe_cuda PUBLIC tensorpipe) - target_link_libraries(tensorpipe_cuda PRIVATE ${TP_CUDA_LINK_LIBRARIES}) - target_include_directories(tensorpipe_cuda PUBLIC ${TP_CUDA_INCLUDE_DIRS}) + target_link_libraries(${TP_GPU_LIB_NAME} PUBLIC tensorpipe) + target_link_libraries(${TP_GPU_LIB_NAME} PRIVATE ${TP_CUDA_LINK_LIBRARIES}) + target_include_directories(${TP_GPU_LIB_NAME} PUBLIC ${TP_CUDA_INCLUDE_DIRS}) - install(TARGETS tensorpipe_cuda + install(TARGETS ${TP_GPU_LIB_NAME} EXPORT TensorpipeTargets LIBRARY DESTINATION ${TP_INSTALL_LIBDIR} ARCHIVE DESTINATION ${TP_INSTALL_LIBDIR}) diff --git a/tensorpipe/benchmark/CMakeLists.txt b/tensorpipe/benchmark/CMakeLists.txt index d12c030aa..e254129dd 100644 --- a/tensorpipe/benchmark/CMakeLists.txt +++ b/tensorpipe/benchmark/CMakeLists.txt @@ -6,8 +6,33 @@ # TODO: Make those separate CMake projects. -add_executable(benchmark_transport benchmark_transport.cc options.cc transport_registry.cc channel_registry.cc) +set(TP_BM_TRANS_SRCS) + +set(TP_BM_PIPE_SRCS) + +list(APPEND TP_BM_TRANS_SRCS + benchmark_transport + benchmark_transport.cc + options.cc + transport_registry.cc + channel_registry.cc) + +list(APPEND TP_BM_PIPE_SRCS + benchmark_pipe.cc + options.cc + transport_registry.cc + channel_registry.cc) + +if(TP_USE_ROCM) + get_hipified_list("TP_BM_TRANS_SRCS" "${TP_BM_TRANS_SRCS}" TP_BM_TRANS_SRCS) + get_hipified_list("TP_BM_PIPE_SRCS" "${TP_BM_PIPE_SRCS}" TP_BM_PIPE_SRCS) + set(TP_GPU_LIB "tensorpipe_hip") +elseif(TP_USE_CUDA) + set(TP_GPU_LIB "tensorpipe_cuda") +endif() + +add_executable(benchmark_transport ${TP_BM_TRANS_SRCS}) target_link_libraries(benchmark_transport PRIVATE tensorpipe) -add_executable(benchmark_pipe benchmark_pipe.cc options.cc transport_registry.cc channel_registry.cc) -target_link_libraries(benchmark_pipe PRIVATE tensorpipe tensorpipe_cuda) +add_executable(benchmark_pipe ${TP_BM_PIPE_SRCS}) +target_link_libraries(benchmark_pipe PRIVATE tensorpipe ${TP_GPU_LIB}) diff --git a/tensorpipe/test/CMakeLists.txt b/tensorpipe/test/CMakeLists.txt index 43ae51a56..1034e1891 100644 --- a/tensorpipe/test/CMakeLists.txt +++ b/tensorpipe/test/CMakeLists.txt @@ -68,11 +68,22 @@ if(TP_ENABLE_CMA) add_subdirectory(channel/cma) endif() -if(TP_USE_CUDA) - find_package(CUDA REQUIRED) - list(APPEND TP_TEST_LINK_LIBRARIES ${CUDA_LIBRARIES}) - list(APPEND TP_TEST_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS}) - list(APPEND TP_TEST_COMPILE_DEFS TP_USE_CUDA) +if(TP_USE_CUDA OR TP_USE_ROCM) + if(TP_USE_CUDA) + set(TP_GPU_KERNEL_LIB_NAME "tensorpipe_cuda_kernel") + set(TP_GPU_LIB_NAME "tensorpipe_cuda") + find_package(CUDA REQUIRED) + list(APPEND TP_TEST_LINK_LIBRARIES ${CUDA_LIBRARIES}) + list(APPEND TP_TEST_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS}) + list(APPEND TP_TEST_COMPILE_DEFS TP_USE_CUDA) + elseif(TP_USE_ROCM) + set(TP_GPU_KERNEL_LIB_NAME "tensorpipe_hip_kernel") + set(TP_GPU_LIB_NAME "tensorpipe_hip") + # Finding of HIP package is already before hipifying the files + list(APPEND TP_TEST_LINK_LIBRARIES ${TP_HIP_HCC_LIBRARIES}) + list(APPEND TP_TEST_INCLUDE_DIRS ${TP_HIP_INCLUDE}) + list(APPEND TP_TEST_COMPILE_DEFS TP_USE_ROCM) + endif() list(APPEND TP_TEST_SRCS channel/channel_test_cuda.cc @@ -82,27 +93,40 @@ if(TP_USE_CUDA) core/pipe_cuda_test.cc ) - list(APPEND TP_TEST_SRCS - channel/cuda_xth/cuda_xth_test.cc - channel/cuda_basic/cuda_basic_test.cc - ) + if(TP_ENABLE_HIP_XTH OR TP_USE_CUDA) + list(APPEND TP_TEST_SRCS + channel/cuda_xth/cuda_xth_test.cc + channel/cuda_basic/cuda_basic_test.cc + ) + endif() - if(TP_ENABLE_CUDA_IPC) + if(TP_ENABLE_CUDA_IPC OR TP_ENABLE_HIP_IPC) list(APPEND TP_TEST_SRCS channel/cuda_ipc/cuda_ipc_test.cc ) endif() - list(APPEND TP_TEST_SRCS - channel/cuda_gdr/cuda_gdr_test.cc - ) + if((TP_ENABLE_CUDA_GDR AND TP_USE_CUDA) OR (TP_ENABLE_HIP_GDR AND TP_USE_ROCM)) + list(APPEND TP_TEST_SRCS + channel/cuda_gdr/cuda_gdr_test.cc + ) + endif() - cuda_add_library(tensorpipe_cuda_kernel channel/kernel.cu) - list(APPEND TP_TEST_LINK_LIBRARIES tensorpipe_cuda_kernel) + if(TP_USE_CUDA) + cuda_add_library(${TP_GPU_KERNEL_LIB_NAME} channel/kernel.cu) + elseif(TP_USE_ROCM) + # Update the TP_TEST_SRCS file list with hipified file names + get_hipified_list("TP_TEST_SRCS" "${TP_TEST_SRCS}" TP_TEST_SRCS) + hip_add_library(${TP_GPU_KERNEL_LIB_NAME} channel/kernel.hip) + target_compile_options(${TP_GPU_KERNEL_LIB_NAME} PUBLIC ${HIP_CLANG_FLAGS}) + endif() + + list(APPEND TP_TEST_LINK_LIBRARIES ${TP_GPU_KERNEL_LIB_NAME}) - list(APPEND TP_TEST_LINK_LIBRARIES tensorpipe_cuda) + list(APPEND TP_TEST_LINK_LIBRARIES ${TP_GPU_LIB_NAME}) endif() + add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/googletest ${PROJECT_BINARY_DIR}/third_party/googletest EXCLUDE_FROM_ALL) diff --git a/tools/amd_build/build_amd.py b/tools/amd_build/build_amd.py new file mode 100755 index 000000000..7f4d9d946 --- /dev/null +++ b/tools/amd_build/build_amd.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import sys +import argparse +import json + +sys.path.append(os.path.realpath(os.path.join( + os.path.dirname(__file__), + os.path.pardir, + os.path.pardir, + 'third_party'))) + +from hipify import hipify_python + +parser = argparse.ArgumentParser(description='Top-level script for HIPifying, filling in most common parameters') +parser.add_argument( + '--project-directory', + type=str, + help="The root of the project. (default: %(default)s)", + required=True) + +parser.add_argument( + '--output-directory', + type=str, + help="The Directory to Store the Hipified Project", + required=True) + +parser.add_argument( + '--dump-dict-directory', + type=str, + help="The Directory to Store the return dict output after hipification", + required=True) + +args = parser.parse_args() + +includes = [ + '*' +] + +ignores = [ +] + +# capturing the return value which is a dict[filename]:HipifyResult +HipifyFinalResult = hipify_python.hipify( + project_directory=args.project_directory, + output_directory=args.output_directory, + includes=includes, + ignores=ignores, + is_pytorch_extension=True) + +dict_file_name = args.dump_dict_directory + "/hipify_output_dict_dump.txt" +with open(dict_file_name, 'w') as dict_file: + dict_file.write(json.dumps(HipifyFinalResult)) diff --git a/tools/amd_build/replace_cuda_with_hip_files.py b/tools/amd_build/replace_cuda_with_hip_files.py new file mode 100755 index 000000000..3b077d881 --- /dev/null +++ b/tools/amd_build/replace_cuda_with_hip_files.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import sys +import argparse +import json + +parser = argparse.ArgumentParser(description="") +parser.add_argument( + '--io-file', + type=str, + help="Input file to read the list of files", + required=True) + +parser.add_argument( + '--dump-dict-directory', + type=str, + help="The Directory when the dictionary output of hipified is stored", + required=True) + +args = parser.parse_args() + +dict_file_name = args.dump_dict_directory + "/hipify_output_dict_dump.txt" +file_obj = open(dict_file_name, mode='r') +json_string = file_obj.read() +file_obj.close() +hipfied_result = json.loads(json_string) + +out_list = [] +with open(args.io_file) as inp_file: + for line in inp_file: + line = line.strip() + if line in hipfied_result: + out_list.append(hipfied_result[line]['hipified_path']) + else: + out_list.append(line) + +w_file_obj = open(args.io_file, mode='w') +for f in out_list: + w_file_obj.write(f+"\n") +w_file_obj.close() From d9f3bc6ed304a96fe47c894f88b36e04fc032a4c Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Mon, 19 Jul 2021 12:42:28 -0700 Subject: [PATCH 2/9] Hipify related files moved to hipify-torch repo --- CMakeLists.txt | 2 +- cmake/Hipify.cmake | 62 ------------------- tools/amd_build/build_amd.py | 60 ------------------ .../amd_build/replace_cuda_with_hip_files.py | 47 -------------- 4 files changed, 1 insertion(+), 170 deletions(-) delete mode 100644 cmake/Hipify.cmake delete mode 100755 tools/amd_build/build_amd.py delete mode 100755 tools/amd_build/replace_cuda_with_hip_files.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 174746a7e..4f1faff42 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ project(tensorpipe LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 14) -list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${PROJECT_SOURCE_DIR}/third_party/hipify/cmake") # Expose build options. include(Options) diff --git a/cmake/Hipify.cmake b/cmake/Hipify.cmake deleted file mode 100644 index 475e369b1..000000000 --- a/cmake/Hipify.cmake +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# cmake file to trigger hipify - -function(write_file_list FILE_SUFFIX INPUT_LIST) - message(STATUS "Writing ${FILE_SUFFIX} into file - file_${FILE_SUFFIX}.txt") - set(_FULL_FILE_NAME "${CMAKE_BINARY_DIR}/cuda_to_hip_list_${FILE_SUFFIX}.txt") - file(WRITE ${_FULL_FILE_NAME} "") - foreach(_SOURCE_FILE ${INPUT_LIST}) - file(APPEND ${_FULL_FILE_NAME} ${CMAKE_CURRENT_SOURCE_DIR}/${_SOURCE_FILE}) - file(APPEND ${_FULL_FILE_NAME} "\n") - endforeach() -endfunction() - -function(get_file_list FILE_SUFFIX OUTPUT_LIST) - set(_FULL_FILE_NAME "${CMAKE_BINARY_DIR}/cuda_to_hip_list_${FILE_SUFFIX}.txt") - file(STRINGS ${_FULL_FILE_NAME} _FILE_LIST) - set(${OUTPUT_LIST}_HIP ${_FILE_LIST} PARENT_SCOPE) -endfunction() - -function(update_list_with_hip_files FILE_SUFFIX) - set(_SCRIPTS_DIR ${PROJECT_SOURCE_DIR}/tools/amd_build) - set(_FULL_FILE_NAME "${CMAKE_BINARY_DIR}/cuda_to_hip_list_${FILE_SUFFIX}.txt") - set(_EXE_COMMAND - ${_SCRIPTS_DIR}/replace_cuda_with_hip_files.py - --io-file ${_FULL_FILE_NAME} - --dump-dict-directory ${CMAKE_BINARY_DIR}) - execute_process( - COMMAND ${_EXE_COMMAND} - RESULT_VARIABLE _return_value) - if (NOT _return_value EQUAL 0) - message(FATAL_ERROR "Failed to get the list of hipified files!") - endif() -endfunction() - -function(get_hipified_list FILE_SUFFIX INPUT_LIST OUTPUT_LIST) - write_file_list("${FILE_SUFFIX}" "${INPUT_LIST}") - update_list_with_hip_files("${FILE_SUFFIX}") - get_file_list("${FILE_SUFFIX}" __temp_srcs) - set(${OUTPUT_LIST} ${__temp_srcs_HIP} PARENT_SCOPE) -endfunction() - - -set(HIPIFY_SCRIPTS_DIR ${PROJECT_SOURCE_DIR}/tools/amd_build) -set(HIPIFY_COMMAND - ${HIPIFY_SCRIPTS_DIR}/build_amd.py - --project-directory ${PROJECT_SOURCE_DIR} - --output-directory ${PROJECT_SOURCE_DIR} - --dump-dict-directory ${CMAKE_BINARY_DIR} -) - -execute_process( - COMMAND ${HIPIFY_COMMAND} - RESULT_VARIABLE hipify_return_value -) -if (NOT hipify_return_value EQUAL 0) - message(FATAL_ERROR "Failed to hipify files!") -endif() diff --git a/tools/amd_build/build_amd.py b/tools/amd_build/build_amd.py deleted file mode 100755 index 7f4d9d946..000000000 --- a/tools/amd_build/build_amd.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import os -import sys -import argparse -import json - -sys.path.append(os.path.realpath(os.path.join( - os.path.dirname(__file__), - os.path.pardir, - os.path.pardir, - 'third_party'))) - -from hipify import hipify_python - -parser = argparse.ArgumentParser(description='Top-level script for HIPifying, filling in most common parameters') -parser.add_argument( - '--project-directory', - type=str, - help="The root of the project. (default: %(default)s)", - required=True) - -parser.add_argument( - '--output-directory', - type=str, - help="The Directory to Store the Hipified Project", - required=True) - -parser.add_argument( - '--dump-dict-directory', - type=str, - help="The Directory to Store the return dict output after hipification", - required=True) - -args = parser.parse_args() - -includes = [ - '*' -] - -ignores = [ -] - -# capturing the return value which is a dict[filename]:HipifyResult -HipifyFinalResult = hipify_python.hipify( - project_directory=args.project_directory, - output_directory=args.output_directory, - includes=includes, - ignores=ignores, - is_pytorch_extension=True) - -dict_file_name = args.dump_dict_directory + "/hipify_output_dict_dump.txt" -with open(dict_file_name, 'w') as dict_file: - dict_file.write(json.dumps(HipifyFinalResult)) diff --git a/tools/amd_build/replace_cuda_with_hip_files.py b/tools/amd_build/replace_cuda_with_hip_files.py deleted file mode 100755 index 3b077d881..000000000 --- a/tools/amd_build/replace_cuda_with_hip_files.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import os -import sys -import argparse -import json - -parser = argparse.ArgumentParser(description="") -parser.add_argument( - '--io-file', - type=str, - help="Input file to read the list of files", - required=True) - -parser.add_argument( - '--dump-dict-directory', - type=str, - help="The Directory when the dictionary output of hipified is stored", - required=True) - -args = parser.parse_args() - -dict_file_name = args.dump_dict_directory + "/hipify_output_dict_dump.txt" -file_obj = open(dict_file_name, mode='r') -json_string = file_obj.read() -file_obj.close() -hipfied_result = json.loads(json_string) - -out_list = [] -with open(args.io_file) as inp_file: - for line in inp_file: - line = line.strip() - if line in hipfied_result: - out_list.append(hipfied_result[line]['hipified_path']) - else: - out_list.append(line) - -w_file_obj = open(args.io_file, mode='w') -for f in out_list: - w_file_obj.write(f+"\n") -w_file_obj.close() From 5681a98c36f8589582308031a60867654237aa02 Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Mon, 19 Jul 2021 13:37:29 -0700 Subject: [PATCH 3/9] Reordering of parameter to get_hipified_list function --- tensorpipe/CMakeLists.txt | 5 +++-- tensorpipe/benchmark/CMakeLists.txt | 5 +++-- tensorpipe/test/CMakeLists.txt | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tensorpipe/CMakeLists.txt b/tensorpipe/CMakeLists.txt index c8beddb93..753e0b6e1 100644 --- a/tensorpipe/CMakeLists.txt +++ b/tensorpipe/CMakeLists.txt @@ -311,8 +311,9 @@ if(TP_USE_CUDA OR TP_USE_ROCM) endif() if(TP_USE_ROCM) - get_hipified_list("TP_CUDA_SRCS" "${TP_CUDA_SRCS}" TP_CUDA_SRCS) - get_hipified_list("TP_CUDA_PUBLIC_HDRS" "${TP_CUDA_PUBLIC_HDRS}" TP_CUDA_PUBLIC_HDRS) + # Replace the cuda file names in TP_CUDA_SRCS & TP_CUDA_PUBLIC_HDRS file lists with hipified file names + get_hipified_list("${TP_CUDA_SRCS}" TP_CUDA_SRCS "TP_CUDA_SRCS") + get_hipified_list("${TP_CUDA_PUBLIC_HDRS}" TP_CUDA_PUBLIC_HDRS "TP_CUDA_PUBLIC_HDRS") endif() configure_file(config_cuda.h.in config_cuda.h) diff --git a/tensorpipe/benchmark/CMakeLists.txt b/tensorpipe/benchmark/CMakeLists.txt index e254129dd..202bcb4d0 100644 --- a/tensorpipe/benchmark/CMakeLists.txt +++ b/tensorpipe/benchmark/CMakeLists.txt @@ -24,8 +24,9 @@ list(APPEND TP_BM_PIPE_SRCS channel_registry.cc) if(TP_USE_ROCM) - get_hipified_list("TP_BM_TRANS_SRCS" "${TP_BM_TRANS_SRCS}" TP_BM_TRANS_SRCS) - get_hipified_list("TP_BM_PIPE_SRCS" "${TP_BM_PIPE_SRCS}" TP_BM_PIPE_SRCS) + # Replace the cuda file names in TP_BM_TRANS_SRCS & TP_BM_PIPE_SRCS file lists with hipified file names + get_hipified_list("${TP_BM_TRANS_SRCS}" TP_BM_TRANS_SRCS "TP_BM_TRANS_SRCS") + get_hipified_list("${TP_BM_PIPE_SRCS}" TP_BM_PIPE_SRCS "TP_BM_PIPE_SRCS") set(TP_GPU_LIB "tensorpipe_hip") elseif(TP_USE_CUDA) set(TP_GPU_LIB "tensorpipe_cuda") diff --git a/tensorpipe/test/CMakeLists.txt b/tensorpipe/test/CMakeLists.txt index 1034e1891..8c1d49c21 100644 --- a/tensorpipe/test/CMakeLists.txt +++ b/tensorpipe/test/CMakeLists.txt @@ -115,8 +115,8 @@ if(TP_USE_CUDA OR TP_USE_ROCM) if(TP_USE_CUDA) cuda_add_library(${TP_GPU_KERNEL_LIB_NAME} channel/kernel.cu) elseif(TP_USE_ROCM) - # Update the TP_TEST_SRCS file list with hipified file names - get_hipified_list("TP_TEST_SRCS" "${TP_TEST_SRCS}" TP_TEST_SRCS) + # Replace the cuda file names in TP_TEST_SRCS file list with hipified file names + get_hipified_list("${TP_TEST_SRCS}" TP_TEST_SRCS "TP_TEST_SRCS") hip_add_library(${TP_GPU_KERNEL_LIB_NAME} channel/kernel.hip) target_compile_options(${TP_GPU_KERNEL_LIB_NAME} PUBLIC ${HIP_CLANG_FLAGS}) endif() From 07ebeb942661e7c1126cb8714223126b98a7d17e Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Mon, 19 Jul 2021 15:53:46 -0700 Subject: [PATCH 4/9] Nit update related to cmake macro --- tensorpipe/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorpipe/CMakeLists.txt b/tensorpipe/CMakeLists.txt index 753e0b6e1..2cb6701ad 100644 --- a/tensorpipe/CMakeLists.txt +++ b/tensorpipe/CMakeLists.txt @@ -79,7 +79,7 @@ list(APPEND TP_PUBLIC_HDRS ### cma -tp_conditional_backend( +TP_CONDITIONAL_BACKEND( TP_ENABLE_CMA "Enable cross-memory attach channel" "LINUX") if(TP_ENABLE_CMA) list(APPEND TP_SRCS @@ -125,7 +125,7 @@ list(APPEND TP_LINK_LIBRARIES uv::uv) ### shm -tp_conditional_backend( +TP_CONDITIONAL_BACKEND( TP_ENABLE_SHM "Enable shared-memory transport" "LINUX") if(TP_ENABLE_SHM) list(APPEND TP_SRCS @@ -144,7 +144,7 @@ endif() ### ibv -tp_conditional_backend( +TP_CONDITIONAL_BACKEND( TP_ENABLE_IBV "Enable InfiniBand transport" "LINUX") if(TP_ENABLE_IBV) list(APPEND TP_SRCS @@ -255,7 +255,7 @@ if(TP_USE_CUDA OR TP_USE_ROCM) ### cuda_xth - tp_conditional_backend( + TP_CONDITIONAL_BACKEND( TP_ENABLE_HIP_XTH "Enable HIP XTH communication channel" "TP_USE_ROCM") if(TP_ENABLE_HIP_XTH OR TP_USE_CUDA) list(APPEND TP_CUDA_SRCS @@ -278,9 +278,9 @@ if(TP_USE_CUDA OR TP_USE_ROCM) ### cuda_ipc - tp_conditional_backend( + TP_CONDITIONAL_BACKEND( TP_ENABLE_CUDA_IPC "Enable CUDA inter-process communication channel" "TP_USE_CUDA") - tp_conditional_backend( + TP_CONDITIONAL_BACKEND( TP_ENABLE_HIP_IPC "Enable HIP inter-process communication channel" "TP_USE_ROCM") if(TP_ENABLE_CUDA_IPC OR TP_ENABLE_HIP_IPC) list(APPEND TP_CUDA_SRCS @@ -294,9 +294,9 @@ if(TP_USE_CUDA OR TP_USE_ROCM) ### cuda_gdr - tp_conditional_backend( + TP_CONDITIONAL_BACKEND( TP_ENABLE_CUDA_GDR "Enable CUDA GpuDirect (InfiniBand) channel" "LINUX") - tp_conditional_backend( + TP_CONDITIONAL_BACKEND( TP_ENABLE_HIP_GDR "Enable HIP GpuDirect (InfiniBand) channel" "LINUX") if((TP_ENABLE_CUDA_GDR AND TP_USE_CUDA) OR (TP_ENABLE_HIP_GDR AND TP_USE_ROCM)) list(APPEND TP_CUDA_SRCS From 64174b8a72d2da33726531ca1febfb20f3a882d9 Mon Sep 17 00:00:00 2001 From: Jithun Nair Date: Wed, 28 Jul 2021 15:38:16 +0000 Subject: [PATCH 5/9] Add comment --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f1faff42..6059de80f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ project(tensorpipe LANGUAGES C CXX) set(CMAKE_CXX_STANDARD 14) +# ROCm related list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${PROJECT_SOURCE_DIR}/third_party/hipify/cmake") # Expose build options. From a0d23474e768e2f22a81ec96fde8e313bbbe9817 Mon Sep 17 00:00:00 2001 From: Jithun Nair Date: Wed, 28 Jul 2021 15:39:16 +0000 Subject: [PATCH 6/9] Update fatal error message --- cmake/Options.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Options.cmake b/cmake/Options.cmake index 4a6da4458..6034d45bb 100644 --- a/cmake/Options.cmake +++ b/cmake/Options.cmake @@ -35,7 +35,7 @@ option(TP_USE_ROCM "Enable support for ROCM tensors" OFF) # if both TP_USE_CUDA and TP_USE_ROCM is set then break if(TP_USE_CUDA AND TP_USE_ROCM) - message(FATAL_ERROR "Tensorpipe can be built either for CUDA or ROCm, TP_USE_CUDA and TP_USE_ROCM both are set, erroring out!!!!") + message(FATAL_ERROR "TensorPipe does not support building for CUDA and ROCM at the same time. Please unset either TP_USE_CUDA or TP_USE_ROCM.") endif() # Optional features From 38800561c43ad3cb82212e283481f78e469d6cb5 Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Sun, 22 Aug 2021 22:10:54 -0700 Subject: [PATCH 7/9] Changes related to updated get_hipified_list() api --- tensorpipe/CMakeLists.txt | 4 ++-- tensorpipe/benchmark/CMakeLists.txt | 4 ++-- tensorpipe/test/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorpipe/CMakeLists.txt b/tensorpipe/CMakeLists.txt index 2cb6701ad..48d05b360 100644 --- a/tensorpipe/CMakeLists.txt +++ b/tensorpipe/CMakeLists.txt @@ -312,8 +312,8 @@ if(TP_USE_CUDA OR TP_USE_ROCM) if(TP_USE_ROCM) # Replace the cuda file names in TP_CUDA_SRCS & TP_CUDA_PUBLIC_HDRS file lists with hipified file names - get_hipified_list("${TP_CUDA_SRCS}" TP_CUDA_SRCS "TP_CUDA_SRCS") - get_hipified_list("${TP_CUDA_PUBLIC_HDRS}" TP_CUDA_PUBLIC_HDRS "TP_CUDA_PUBLIC_HDRS") + get_hipified_list("${TP_CUDA_SRCS}" TP_CUDA_SRCS) + get_hipified_list("${TP_CUDA_PUBLIC_HDRS}" TP_CUDA_PUBLIC_HDRS) endif() configure_file(config_cuda.h.in config_cuda.h) diff --git a/tensorpipe/benchmark/CMakeLists.txt b/tensorpipe/benchmark/CMakeLists.txt index 202bcb4d0..034cf6287 100644 --- a/tensorpipe/benchmark/CMakeLists.txt +++ b/tensorpipe/benchmark/CMakeLists.txt @@ -25,8 +25,8 @@ list(APPEND TP_BM_PIPE_SRCS if(TP_USE_ROCM) # Replace the cuda file names in TP_BM_TRANS_SRCS & TP_BM_PIPE_SRCS file lists with hipified file names - get_hipified_list("${TP_BM_TRANS_SRCS}" TP_BM_TRANS_SRCS "TP_BM_TRANS_SRCS") - get_hipified_list("${TP_BM_PIPE_SRCS}" TP_BM_PIPE_SRCS "TP_BM_PIPE_SRCS") + get_hipified_list("${TP_BM_TRANS_SRCS}" TP_BM_TRANS_SRCS) + get_hipified_list("${TP_BM_PIPE_SRCS}" TP_BM_PIPE_SRCS) set(TP_GPU_LIB "tensorpipe_hip") elseif(TP_USE_CUDA) set(TP_GPU_LIB "tensorpipe_cuda") diff --git a/tensorpipe/test/CMakeLists.txt b/tensorpipe/test/CMakeLists.txt index 8c1d49c21..29e53a423 100644 --- a/tensorpipe/test/CMakeLists.txt +++ b/tensorpipe/test/CMakeLists.txt @@ -116,7 +116,7 @@ if(TP_USE_CUDA OR TP_USE_ROCM) cuda_add_library(${TP_GPU_KERNEL_LIB_NAME} channel/kernel.cu) elseif(TP_USE_ROCM) # Replace the cuda file names in TP_TEST_SRCS file list with hipified file names - get_hipified_list("${TP_TEST_SRCS}" TP_TEST_SRCS "TP_TEST_SRCS") + get_hipified_list("${TP_TEST_SRCS}" TP_TEST_SRCS) hip_add_library(${TP_GPU_KERNEL_LIB_NAME} channel/kernel.hip) target_compile_options(${TP_GPU_KERNEL_LIB_NAME} PUBLIC ${HIP_CLANG_FLAGS}) endif() From 15b1bd88c238bc9915d21943944c5bded4502415 Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Wed, 25 Aug 2021 12:24:36 -0700 Subject: [PATCH 8/9] Adding guard to enable skip of not supportted HIP APIs --- tensorpipe/CMakeLists.txt | 7 +++++++ tensorpipe/common/cuda_loop.h | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/tensorpipe/CMakeLists.txt b/tensorpipe/CMakeLists.txt index 48d05b360..8df78e991 100644 --- a/tensorpipe/CMakeLists.txt +++ b/tensorpipe/CMakeLists.txt @@ -235,6 +235,9 @@ if(TP_USE_CUDA OR TP_USE_ROCM) # TP_INCLUDE_DIRS is list of include path to be used set(TP_CUDA_INCLUDE_DIRS) + # TP_CUDA_COMPILE_DEFS is list of compiler definitions to be used + set(TP_CUDA_COMPILE_DEFS) + if (TP_USE_CUDA) set(TP_GPU_LIB_NAME "tensorpipe_cuda") find_package(CUDA REQUIRED) @@ -245,6 +248,9 @@ if(TP_USE_CUDA OR TP_USE_ROCM) # Finding of HIP package is already before hipifying the files list(APPEND TP_CUDA_LINK_LIBRARIES ${TP_HIP_HCC_LIBRARIES}) list(APPEND TP_CUDA_INCLUDE_DIRS ${TP_HIP_INCLUDE}) + list(APPEND TP_CUDA_COMPILE_DEFS TP_USE_ROCM) + option(TP_ROCM_TMP_COMPILE "This variable enables temp code for compilation on ROCm platform" OFF) + list(APPEND TP_CUDA_COMPILE_DEFS TP_ROCM_TMP_COMPILE) endif() list(APPEND TP_CUDA_SRCS @@ -327,6 +333,7 @@ if(TP_USE_CUDA OR TP_USE_ROCM) target_link_libraries(${TP_GPU_LIB_NAME} PUBLIC tensorpipe) target_link_libraries(${TP_GPU_LIB_NAME} PRIVATE ${TP_CUDA_LINK_LIBRARIES}) target_include_directories(${TP_GPU_LIB_NAME} PUBLIC ${TP_CUDA_INCLUDE_DIRS}) + target_compile_definitions(${TP_GPU_LIB_NAME} PUBLIC ${TP_CUDA_COMPILE_DEFS}) install(TARGETS ${TP_GPU_LIB_NAME} EXPORT TensorpipeTargets diff --git a/tensorpipe/common/cuda_loop.h b/tensorpipe/common/cuda_loop.h index 6fc17f0f2..87f2a71af 100644 --- a/tensorpipe/common/cuda_loop.h +++ b/tensorpipe/common/cuda_loop.h @@ -20,6 +20,10 @@ #include +#ifdef TP_USE_ROCM +#define CUDART_CB +#endif + namespace tensorpipe { class CudaLoop { From 9d9bca1c25ca85b9ed15822883986eed2db61fc0 Mon Sep 17 00:00:00 2001 From: Pruthvi Madugundu Date: Wed, 25 Aug 2021 13:07:38 -0700 Subject: [PATCH 9/9] Updates to use new hipify() cmake API --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6059de80f..54dfc3ea4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,7 @@ if (TP_USE_ROCM) include(Hip) if(TP_HAVE_HIP) include(Hipify) + hipify(CUDA_SOURCE_DIR ${PROJECT_SOURCE_DIR}) else() message(FATAL_ERROR "Not able to find HIP installation, so cant compile with ROCm support.") endif()