Issue with CMAKE_SOURCE_DIR in CMakeFiles.txt

[arachid1]

Hello,

I am trying to complete the "Training an audio keyword spotter with PyTorch" tutorial: https://microsoft.github.io/ELL/tutorials/Training-audio-keyword-spotter-with-pytorch/

I am stuck at the step where I need to compile the wrapped ELL:

pushd compiled_featurizer
bash script.sh
popd

with script.sh being the following:

#!/bin/bash
mkdir build
cd build
cmake ..
make
cd ..

It fails at 'cmake ..'

'cmake ..' required a CMakeFiles.txt file but the tutorial doesn't tell what it looks like, so I assumed we could use the CMakesFiles.txt at the root directory:

#
# cmake file for Embedded Learning Library subprojects
#

cmake_minimum_required(VERSION 3.8 FATAL_ERROR)

# Include modules in the CMake directory.
list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMake")
message(${CMAKE_SOURCE_DIR})
include(CompilerCache)

project(ELL)

file(STRINGS "VERSION" ELL_VERSION)
message(STATUS "ELL version ${ELL_VERSION}")

# Define custom configuration types for Visual Studio
if(CMAKE_CONFIGURATION_TYPES)
  # Copy important configuration info from Release config to new Documentation config
  list(APPEND CMAKE_CONFIGURATION_TYPES Documentation)
  set(CMAKE_CXX_FLAGS_DOCUMENTATION ${CMAKE_CXX_FLAGS_RELEASE})
  set(CMAKE_C_FLAGS_DOCUMENTATION ${CMAKE_C_FLAGS_RELEASE})
  set(CMAKE_EXE_LINKER_FLAGS_DOCUMENTATION ${CMAKE_EXE_LINKER_FLAGS_RELEASE})
  set(CMAKE_EXE_LINKER_FLAGS_DOCUMENTATION ${CMAKE_EXE_LINKER_FLAGS_RELEASE})
  set(CMAKE_SHARED_LINKER_FLAGS_DOCUMENTATION ${CMAKE_SHARED_LINKER_FLAGS_RELEASE})
  set(CMAKE_MODULE_LINKER_FLAGS_DOCUMENTATION ${CMAKE_MODULE_LINKER_FLAGS_RELEASE})
endif()

# Try to create a compilation database, which is useful to have when working
# with clang tooling
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

# Options
# To set an option:
# cmake -DMyOption=ON|OFF buildDirectory
# (so if we're running cmake from a 'build' directory inside the main directory, do this:
#      cmake -DBUILD_DOC=OFF ..  )
option(BUILD_DOC "Build Doxygen documentation" ON)
option(STRICT_MODE "Build with 'warnings as errors'" OFF)
option(PROFILING "Enable performance profiling tests" OFF)
option(DISABLE_PYTHON "Explicitly disable building python modules" OFF)
option(CNTK "Enable CNTK importer and related unit tests (requires CNTK python module)" OFF)
option(ONNX "Enable ONNX importer and related unit tests (requires PyTorch and ONNX python modules)" OFF)

set(ELL_ROOT "${CMAKE_SOURCE_DIR}")
set(FLAKE8_CONFIG "${CMAKE_SOURCE_DIR}/.flake8")
set(TEST_MODELS_REPO "https://github.com/Microsoft/ell-test-models" CACHE DOCUMENTATION "URL to the git repo containing test models" )
message(STATUS "Configuring tests to use TEST_MODELS_REPO at: ${TEST_MODELS_REPO}")
set(EXTERNAL_DIR "${CMAKE_SOURCE_DIR}/external" CACHE DOCUMENTATION "Directory to install external dependencies" )
set(RPI_PASSWORD "$ENV{RPI_PASSWORD}")
set(RPI_CLUSTER "$ENV{RPI_CLUSTER}")
set(RPI_KEY "$ENV{RPI_APIKEY}")

if(DISABLE_PYTHON)
  message(STATUS "Python has been explicitly disabled")
else()
  foreach(PYTHON_VERSION 3.7 3.6)
    find_package(PythonInterp ${PYTHON_VERSION} QUIET)
    find_package(PythonLibs ${PYTHON_VERSION} QUIET)

    message(STATUS "Looking for python ${PYTHON_VERSION}: PYTHONINTERP_FOUND=${PYTHONINTERP_FOUND}, PYTHONLIBS_FOUND=${PYTHONLIBS_FOUND}")
    if(${PYTHONINTERP_FOUND} AND ${PYTHONLIBS_FOUND})
      message(STATUS "PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}")
      message(STATUS "PYTHON_VERSION_STRING=${PYTHON_VERSION_STRING}")
      message(STATUS "PYTHON_VERSION_MAJOR=${PYTHON_VERSION_MAJOR}")

      message(STATUS "PYTHON_LIBRARIES=${PYTHON_LIBRARIES}")
      message(STATUS "PYTHON_INCLUDE_PATH=${PYTHON_INCLUDE_PATH}")
      message(STATUS "PYTHON_INCLUDE_DIRS=${PYTHON_INCLUDE_DIRS}")
      message(STATUS "PYTHON_DEBUG_LIBRARIES=${PYTHON_DEBUG_LIBRARIES}")
      message(STATUS "PYTHONLIBS_VERSION_STRING=${PYTHONLIBS_VERSION_STRING}")

      break()
    endif()
  endforeach()

  if(${PYTHONLIBS_FOUND})
    set(PYTHON_ENABLED ON)
  else()
    set(PYTHON_ENABLED OFF)
    message(STATUS "### Python 3.6 or later not found, so python features will be disabled")
  endif()
endif()

include(Flake8)

# Turn on ability to create folders to organize projects
set_property(GLOBAL PROPERTY USE_FOLDERS ON)

# Set C++ version
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

# Apply -fPIC where applicable to the platform
set(CMAKE_POSITION_INDEPENDENT_CODE ON)

# Find the platform-specific way of working with threads
set(THREADS_PREFER_PTHREAD_FLAG ON)
set(CMAKE_THREAD_PREFER_PTHREAD ON)
find_package(Threads REQUIRED)

if(STRICT_MODE)
  message("-- Turning on strict mode with warnings as errors.")
endif()

# Turn on ctest tests
enable_testing()

# Set up global variables to help find NuGet projects
set(PACKAGE_ROOT ${EXTERNAL_DIR})
include(OpenBLASSetup)
include(LLVMSetup)
include(SWIGSetup)
include(CopySharedLibraries)
include(AddPrecompiledHeader)

if(MSVC)
  # Set Visual Studio-specific options
  add_definitions(-DUNICODE)
  add_compile_options(/utf-8)
  add_compile_options(/MP)
  add_compile_options(/bigobj)
  add_compile_options(/W4)
  add_compile_options(/permissive-)
  if(STRICT_MODE)
    add_compile_options(/WX)
  endif()
  # the debug linker needs to match how LLVM was built (either /MD or /MDd)
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${LLVM_MD_OPTION}")
  set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${LLVM_MD_OPTION}")
else()
  # Set Clang/GCC-specific options
  add_compile_options(-Wall)
  if(STRICT_MODE)
    add_compile_options(-Werror)
  endif()
  add_compile_options(-Wsign-compare)
  add_compile_options(-Wno-missing-braces)
  add_compile_options(-Wmissing-field-initializers)
  add_compile_options(-fvisibility-inlines-hidden)
  add_compile_options(-Wno-unknown-pragmas)
  add_compile_options(-Wno-backslash-newline-escape)
  add_compile_options(-Wno-comment)
  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -ggdb3 -O0")
  set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb3 -O0")
  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ggdb3")
  set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -ggdb3")
  if(${CMAKE_CXX_COMPILER_ID} STREQUAL Clang)
    add_compile_options(-Wno-self-assign)
  endif()
endif()

# Print helpful message if LLVM not found on Windows
if(NOT LLVM_FOUND AND WIN32)
  message(WARNING "LLVM not found. Run the following command from the main project directory:\n    nuget.exe restore external/packages.config -PackagesDirectory external")
endif()

# Include cmake projects for libraries and executables
add_subdirectory(libraries)
add_subdirectory(tools)
include(CommonInterfaces)
add_subdirectory(interfaces)
# Add examples (has dependencies on SWIG-generated interfaces)
add_subdirectory(examples)

# Add user directories to ELL build if requested
if(EXISTS "${CMAKE_SOURCE_DIR}/user")
  # Add root user directory if it has a CMakeLists.txt file and INCLUDE_IN_ELL_BUILD.txt file
  if(EXISTS"${CMAKE_SOURCE_DIR}/user/CMakeLists.txt" AND EXISTS "${CMAKE_SOURCE_DIR}/user/INCLUDE_IN_ELL_BUILD.txt")
    message(STATUS "Adding user directory to ELL build")
    add_subdirectory(user)
  endif()

  # Now add all child directories that have CMakeLists.txt files and INCLUDE_IN_ELL_BUILD.txt file
  file(GLOB children RELATIVE "${CMAKE_SOURCE_DIR}/user" "${CMAKE_SOURCE_DIR}/user/*")
  foreach(child ${children})
    if(IS_DIRECTORY "${CMAKE_SOURCE_DIR}/user/${child}" AND EXISTS "${CMAKE_SOURCE_DIR}/user/${child}/CMakeLists.txt" AND EXISTS "${CMAKE_SOURCE_DIR}/user/${child}/INCLUDE_IN_ELL_BUILD.txt")
      message(STATUS "Adding user directory ${child} to ELL build")
      add_subdirectory("user/${child}")
    endif()
  endforeach()
endif()

message(STATUS "Writing: ${CMAKE_BINARY_DIR}/config.json")
set(JSON "{ \"test_models_repo\": \"${TEST_MODELS_REPO}\" }")
file(WRITE "${CMAKE_BINARY_DIR}/config.json" ${JSON})
configure_file("${CMAKE_BINARY_DIR}/config.json" "config.json" COPYONLY)

# Add project for solution-level documentation
set (DOC README.md
         StyleGuide.md)
add_custom_target(documentation ALL DEPENDS ${DOC} SOURCES ${DOC})
set_property(TARGET documentation PROPERTY FOLDER "documentation")

# Generate doxygen documentation
if(BUILD_DOC)
  find_package(Doxygen 1.8 QUIET)

  # search external NuGet package directory also
  if(NOT DOXYGEN_FOUND)
    set(DOXYGEN_PACKAGE_NAME Doxygen)
    set(DOXYGEN_PACKAGE_VERSION 1.8.13)
    set(DOXYGEN_PACKAGE_DIR ${PACKAGE_ROOT}/${DOXYGEN_PACKAGE_NAME}.${DOXYGEN_PACKAGE_VERSION})
    find_program(DOXYGEN_EXECUTABLE doxygen
      HINTS "${DOXYGEN_PACKAGE_DIR}/tools")
      if(DOXYGEN_EXECUTABLE)
        set(DOXYGEN_FOUND TRUE)
      endif()
  endif()

  if(DOXYGEN_FOUND)
    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
    set (DOXYFILE_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile")

    add_custom_target(doc
      COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_LOCATION}
      WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
      COMMENT "Building Doxygen documentation" VERBATIM
      SOURCES Doxyfile)
    set_property(TARGET doc PROPERTY FOLDER "documentation")
  else()
    message(WARNING "Doxygen processor not found")
  endif()
endif()

I get the following error message:

mkdir: build: File exists
/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tutorial/compiled_featurizer
CMake Error at CMakeLists.txt:10 (include):
  include could not find load file:

    CompilerCache


CMake Error at CMakeLists.txt:14 (file):
  file STRINGS file
  "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tutorial/compiled_featurizer/VERSION"
  cannot be read.


-- ELL version
CMake Warning (dev) at CMakeLists.txt:47 (set):
  implicitly converting 'DOCUMENTATION' to 'STRING' type.
This warning is for project developers.  Use -Wno-dev to suppress it.

-- Configuring tests to use TEST_MODELS_REPO at: https://github.com/Microsoft/ell-test-models
CMake Warning (dev) at CMakeLists.txt:49 (set):
  implicitly converting 'DOCUMENTATION' to 'STRING' type.
This warning is for project developers.  Use -Wno-dev to suppress it.

-- Looking for python 3.7: PYTHONINTERP_FOUND=TRUE, PYTHONLIBS_FOUND=TRUE
-- PYTHON_EXECUTABLE=/usr/local/bin/python3.7
-- PYTHON_VERSION_STRING=3.7.4
-- PYTHON_VERSION_MAJOR=3
-- PYTHON_LIBRARIES=/usr/local/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib
-- PYTHON_INCLUDE_PATH=/usr/local/Frameworks/Python.framework/Versions/3.7/include/python3.7m
-- PYTHON_INCLUDE_DIRS=/usr/local/Frameworks/Python.framework/Versions/3.7/include/python3.7m
-- PYTHON_DEBUG_LIBRARIES=
-- PYTHONLIBS_VERSION_STRING=3.7.4
CMake Error at CMakeLists.txt:85 (include):
  include could not find load file:

    Flake8


CMake Error at CMakeLists.txt:111 (include):
  include could not find load file:

    OpenBLASSetup


CMake Error at CMakeLists.txt:112 (include):
  include could not find load file:

    LLVMSetup


CMake Error at CMakeLists.txt:113 (include):
  include could not find load file:

    SWIGSetup


CMake Error at CMakeLists.txt:114 (include):
  include could not find load file:

    CopySharedLibraries


CMake Error at CMakeLists.txt:115 (include):
  include could not find load file:

    AddPrecompiledHeader


CMake Error at CMakeLists.txt:159 (add_subdirectory):
  add_subdirectory given source "libraries" which is not an existing
  directory.


CMake Error at CMakeLists.txt:160 (add_subdirectory):
  add_subdirectory given source "tools" which is not an existing directory.


CMake Error at CMakeLists.txt:161 (include):
  include could not find load file:

    CommonInterfaces


CMake Error at CMakeLists.txt:162 (add_subdirectory):
  add_subdirectory given source "interfaces" which is not an existing
  directory.


CMake Error at CMakeLists.txt:164 (add_subdirectory):
  add_subdirectory given source "examples" which is not an existing
  directory.


-- Writing: /Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tutorial/compiled_featurizer/build/config.json
CMake Warning at CMakeLists.txt:222 (message):
  Doxygen processor not found


-- Configuring incomplete, errors occurred!
See also "/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tutorial/compiled_featurizer/build/CMakeFiles/CMakeOutput.log".
make: *** No targets specified and no makefile found.  Stop.

Clearly, it's not able to find the folders and modules needed, and I suspect the reason is CMAKE_SOURCE_DIR is not pointing to ELL. Indeed, I ran in CMakesFiles.txt:

message(${CMAKE_SOURCE_DIR})

and get:

/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL/tutorial/compiled_featurizer

(You can also refer to the second line of the error message).

I'm not sure if my understanding is wrong, but I would have expected the output to be:

/Users/alirachidi/Documents/Sonavi_Labs/KWS/ELL

I tried changing the CMAKE_SOURCE_DIR variable using cmake -S but I get an error saying the source folder is not the same.

I would really appreciate some help!

[lovettchris]

If the "compiled_featurizer" folder is created by the "tools/wrap/wrap.py" script then it should already contain a generated CMakeLists.txt file.