Support AArch64/ARM64, upgrade LLVM and C++ version

.gitignore

@@ -115,4 +115,4 @@ CMakeSettings.json
 .ccls-cache/*
 
 # ignore config
-config.txt
+config.txt

CMakeLists.txt

@@ -1,8 +1,9 @@
-cmake_minimum_required(VERSION 3.1) # we use target_sources()
+cmake_minimum_required(VERSION 3.19) # we use target_sources()
 project(Extempore VERSION 0.8.9)
 
-# for backwards compatibility with CMake older than 3.19
-cmake_policy(SET CMP0114 OLD)
+if(POLICY CMP0135)
+  cmake_policy(SET CMP0135 NEW)
+endif()
 
 option(ASSETS "download multimedia assets (approx 500MB)" OFF)
 option(BUILD_TESTS "build test targets (including examples)" ON)
@@ -39,9 +40,20 @@ endif()
 
 # packaging (binary distribution)
 
+# ARM64 requires macOS 11.0 (Big Sur) minimum
+if(APPLE)
+  if(NOT DEFINED CMAKE_OSX_DEPLOYMENT_TARGET)
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64" OR CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "arm64")
+      set(CMAKE_OSX_DEPLOYMENT_TARGET 11.0)
+    endif()
+  endif()
+endif()
+
 if(PACKAGE)
-  # this needs to be set before project() is called
-  set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12)
+  # For packaged binaries on Intel, enforce a 10.12 floor if still unset
+  if(NOT DEFINED CMAKE_OSX_DEPLOYMENT_TARGET)
+    set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12)
+  endif()
   set(ASSETS ON) # necessary for packaging
   message(STATUS "Building Extempore for binary distribution (assets directory will be downloaded)")
 endif()
@@ -66,8 +78,15 @@ if(EXTERNAL_SHLIBS)
   set(EXT_DEPS_INSTALL_DIR ${CMAKE_BINARY_DIR}/deps-install)
   set(EXT_PLATFORM_SHLIBS_DIR ${CMAKE_SOURCE_DIR}/libs/platform-shlibs)
   if(PACKAGE)
+    # Use generic tuning for x86_64 packages, but native tuning for ARM64
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64")
+      set(EXT_DEPS_C_FLAGS "${CMAKE_C_FLAGS_RELEASE}")
+      set(EXT_DEPS_CXX_FLAGS "${CMAKE_CXX_FLAGS_RELEASE}")
+      message(STATUS "ARM64 detected: using native CPU tuning")
+    else()
     set(EXT_DEPS_C_FLAGS "${CMAKE_C_FLAGS_RELEASE} -mtune=generic")
     set(EXT_DEPS_CXX_FLAGS "${CMAKE_CXX_FLAGS_RELEASE} -mtune=generic")
+    endif()
     message(STATUS "compiler flags for packaging:\nC    ${EXT_DEPS_C_FLAGS}\nCXX  ${EXT_DEPS_CXX_FLAGS}")
   endif()
 endif()
@@ -119,7 +138,7 @@ if(APPLE)
   execute_process(COMMAND sw_vers -productVersion
     OUTPUT_VARIABLE EXTEMPORE_SYSTEM_VERSION
     OUTPUT_STRIP_TRAILING_WHITESPACE)
-  string(REGEX MATCH "^10.[0-9]+" EXTEMPORE_SYSTEM_VERSION ${EXTEMPORE_SYSTEM_VERSION})
+  string(REGEX MATCH "^[0-9]+\\.?[0-9]*" EXTEMPORE_SYSTEM_VERSION ${EXTEMPORE_SYSTEM_VERSION})
   set(EXTEMPORE_SYSTEM_ARCHITECTURE ${UNAME_MACHINE_NAME})
 elseif(UNIX)
   # try lsb_release first - better at giving the distro name
@@ -171,8 +190,10 @@ target_compile_definitions(pcre
   )
 
 if(PACKAGE)
+  if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64")
   target_compile_options(pcre
     PRIVATE -mtune=generic)
+  endif()
 endif()
 
 #############
@@ -195,67 +216,90 @@ ExternalProject_Add(portaudio_static
   -DCMAKE_CXX_FLAGS=${EXT_DEPS_CXX_FLAGS}
   -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/portaudio)
 
-##############
-# LLVM 3.8.0 #
-##############
+###############
+# LLVM 21.1.7 #
+###############
 
 # if you need to build LLVM by hand, the command will be something like
-# cmake .. -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_ZLIB=OFF -DCMAKE_INSTALL_PREFIX=c:/Users/ben/Code/extempore/llvm-3.8.0-release
+# cmake .. -DLLVM_TARGETS_TO_BUILD=X86 -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_TERMINFO=OFF -DLLVM_ENABLE_ZLIB=OFF -DCMAKE_INSTALL_PREFIX=c:/Users/ben/Code/extempore/llvm-21.1.7-release
+
+# Detect target architecture for LLVM.
+if(APPLE)
+  if(UNAME_MACHINE_NAME STREQUAL "arm64")
+    set(LLVM_TARGET_ARCH "AArch64")
+    set(LLVM_ARCH_PREFIX "AArch64")
+  else()
+    set(LLVM_TARGET_ARCH "X86")
+    set(LLVM_ARCH_PREFIX "X86")
+  endif()
+elseif(UNIX)
+  if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+    set(LLVM_TARGET_ARCH "AArch64")
+    set(LLVM_ARCH_PREFIX "AArch64")
+  else()
+    set(LLVM_TARGET_ARCH "X86")
+    set(LLVM_ARCH_PREFIX "X86")
+  endif()
+else()
+  # Only x86_64 currently supported for Windows.
+  set(LLVM_TARGET_ARCH "X86")
+  set(LLVM_ARCH_PREFIX "X86")
+endif()
+
+message(STATUS "LLVM target architecture: ${LLVM_TARGET_ARCH}")
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 if(NOT BUILD_LLVM)
   add_custom_target(LLVM)
 else()
   include(ExternalProject)
 
-  if(PACKAGE)
     ExternalProject_Add(LLVM
       PREFIX llvm
-	  URL https://github.com/digego/extempore/releases/download/v0.8.9/llvm-3.8.0.src-patched-for-extempore.tar.xz
-      URL_MD5 600ee9a94d2e104f53be739568f3508e
+    URL https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.7/llvm-project-21.1.7.src.tar.xz
+    SOURCE_SUBDIR llvm
       CMAKE_ARGS
-      -DLLVM_TARGETS_TO_BUILD=X86
+      -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET_ARCH}
       -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
       -DLLVM_ENABLE_TERMINFO=OFF
       -DLLVM_ENABLE_ZLIB=OFF
+      -DLLVM_ENABLE_ZSTD=OFF
+      -DLLVM_ENABLE_LIBXML2=OFF
       -DLLVM_INCLUDE_TOOLS=ON
       -DLLVM_BUILD_TOOLS=ON
       -DLLVM_INCLUDE_UTILS=OFF
       -DLLVM_BUILD_RUNTIME=OFF
       -DLLVM_INCLUDE_EXAMPLES=OFF
       -DLLVM_INCLUDE_TESTS=OFF
-      -DLLVM_INCLUDE_GO_TESTS=OFF
-      -DLLVM_INCLUDE_GO_TESTS=OFF
+      -DLLVM_INCLUDE_BENCHMARKS=OFF
       -DLLVM_INCLUDE_DOCS=OFF
+      -DLLVM_ENABLE_BINDINGS=OFF
+      -DLLVM_ENABLE_OCAMLDOC=OFF
       -DCMAKE_C_FLAGS=${EXT_DEPS_C_FLAGS}
       -DCMAKE_CXX_FLAGS=${EXT_DEPS_CXX_FLAGS}
       -DCMAKE_INSTALL_PREFIX=${EXT_LLVM_DIR})
-  else()
-    ExternalProject_Add(LLVM
-      PREFIX llvm
-	  URL https://github.com/digego/extempore/releases/download/v0.8.9/llvm-3.8.0.src-patched-for-extempore.tar.xz
-      URL_MD5 600ee9a94d2e104f53be739568f3508e
-      CMAKE_ARGS
-      -DLLVM_TARGETS_TO_BUILD=X86
-      -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-      -DLLVM_ENABLE_TERMINFO=OFF
-      -DLLVM_ENABLE_ZLIB=OFF
-      -DLLVM_INCLUDE_TOOLS=ON
-      -DLLVM_BUILD_TOOLS=ON
-      -DLLVM_INCLUDE_UTILS=OFF
-      -DLLVM_BUILD_RUNTIME=OFF
-      -DLLVM_INCLUDE_EXAMPLES=OFF
-      -DLLVM_INCLUDE_TESTS=OFF
-      -DLLVM_INCLUDE_GO_TESTS=OFF
-      -DLLVM_INCLUDE_GO_TESTS=OFF
-      -DLLVM_INCLUDE_DOCS=OFF
-      -DCMAKE_INSTALL_PREFIX=${EXT_LLVM_DIR})
-  endif()
   ExternalProject_Add_StepTargets(LLVM install)
 endif()
 
-# the ordering of these libs matters, especially with the gcc linker.
-# Check the output of "llvm-config --libnames" to be sure
-set(EXT_LLVM_LIBRARIES "LLVMLTO;LLVMObjCARCOpts;LLVMSymbolize;LLVMDebugInfoPDB;LLVMDebugInfoDWARF;LLVMMIRParser;LLVMLibDriver;LLVMOption;LLVMTableGen;LLVMOrcJIT;LLVMPasses;LLVMipo;LLVMVectorize;LLVMLinker;LLVMIRReader;LLVMAsmParser;LLVMX86Disassembler;LLVMX86AsmParser;LLVMX86CodeGen;LLVMSelectionDAG;LLVMAsmPrinter;LLVMX86Desc;LLVMMCDisassembler;LLVMX86Info;LLVMX86AsmPrinter;LLVMX86Utils;LLVMMCJIT;LLVMLineEditor;LLVMDebugInfoCodeView;LLVMInterpreter;LLVMExecutionEngine;LLVMRuntimeDyld;LLVMCodeGen;LLVMTarget;LLVMScalarOpts;LLVMInstCombine;LLVMInstrumentation;LLVMProfileData;LLVMObject;LLVMMCParser;LLVMTransformUtils;LLVMMC;LLVMBitWriter;LLVMBitReader;LLVMAnalysis;LLVMCore;LLVMSupport")
+# Architecture-independent libraries (order matters for linker!)
+set(EXT_LLVM_LIBRARIES_COMMON
+  "LLVMWindowsManifest;LLVMXRay;LLVMLibDriver;LLVMDlltoolDriver;LLVMTextAPIBinaryReader;LLVMCoverage;LLVMLineEditor;LLVMSandboxIR;LLVMOrcDebugging;LLVMOrcJIT;LLVMWindowsDriver;LLVMMCJIT;LLVMJITLink;LLVMInterpreter;LLVMExecutionEngine;LLVMRuntimeDyld;LLVMOrcTargetProcess;LLVMOrcShared;LLVMDWP;LLVMDebugInfoLogicalView;LLVMDebugInfoGSYM;LLVMOption;LLVMObjectYAML;LLVMObjCopy;LLVMMCA;LLVMMCDisassembler;LLVMLTO;LLVMCFGuard;LLVMCFIVerify;LLVMFrontendOpenACC;LLVMFrontendHLSL;LLVMFrontendDriver;LLVMFrontendDirective;LLVMExtensions;LLVMPasses;LLVMHipStdPar;LLVMCoroutines;LLVMipo;LLVMInstrumentation;LLVMVectorize;LLVMLinker;LLVMFrontendOpenMP;LLVMFrontendOffloading;LLVMDWARFLinkerParallel;LLVMDWARFLinkerClassic;LLVMDWARFLinker;LLVMDWARFCFIChecker;LLVMGlobalISel;LLVMMIRParser;LLVMAsmPrinter;LLVMSelectionDAG;LLVMCodeGen;LLVMCGData;LLVMTarget;LLVMObjCARCOpts;LLVMCodeGenTypes;LLVMIRPrinter;LLVMInterfaceStub;LLVMFileCheck;LLVMFuzzMutate;LLVMFuzzerCLI;LLVMScalarOpts;LLVMInstCombine;LLVMAggressiveInstCombine;LLVMTransformUtils;LLVMBitWriter;LLVMAnalysis;LLVMProfileData;LLVMDebuginfod;LLVMSymbolize;LLVMDebugInfoBTF;LLVMDebugInfoPDB;LLVMDebugInfoMSF;LLVMDebugInfoDWARF;LLVMDebugInfoDWARFLowLevel;LLVMObject;LLVMTextAPI;LLVMMCParser;LLVMIRReader;LLVMAsmParser;LLVMMC;LLVMDebugInfoCodeView;LLVMBitReader;LLVMFrontendAtomic;LLVMCore;LLVMRemarks;LLVMBitstreamReader;LLVMBinaryFormat;LLVMTargetParser;LLVMTableGen;LLVMTableGenBasic;LLVMTableGenCommon;LLVMTelemetry;LLVMSupport;LLVMDemangle")
+
+# Architecture-specific libraries
+set(EXT_LLVM_LIBRARIES_X86 "LLVMX86Disassembler;LLVMX86AsmParser;LLVMX86CodeGen;LLVMX86Desc;LLVMX86Info")
+set(EXT_LLVM_LIBRARIES_AARCH64 "LLVMAArch64Disassembler;LLVMAArch64AsmParser;LLVMAArch64CodeGen;LLVMAArch64Desc;LLVMAArch64Info;LLVMAArch64Utils")
+
+# Select the appropriate libraries.
+if(LLVM_TARGET_ARCH STREQUAL "AArch64")
+  set(EXT_LLVM_LIBRARIES_ARCH ${EXT_LLVM_LIBRARIES_AARCH64})
+else()
+  set(EXT_LLVM_LIBRARIES_ARCH ${EXT_LLVM_LIBRARIES_X86})
+endif()
+
+# Combine into final list (arch-specific libs need to come before common libs for linker)
+set(EXT_LLVM_LIBRARIES "${EXT_LLVM_LIBRARIES_ARCH};${EXT_LLVM_LIBRARIES_COMMON}")
 foreach(llvm_lib ${EXT_LLVM_LIBRARIES})
   get_filename_component(LLVM_LIB_FULLPATH "${EXT_LLVM_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${llvm_lib}${CMAKE_STATIC_LIBRARY_SUFFIX}" ABSOLUTE)
   list(APPEND LLVM_LIBRARIES ${LLVM_LIB_FULLPATH})
@@ -281,14 +325,15 @@ if (EXT_DYLIB)
     runtime/scheme.xtm
   )
 
-  add_library(extempore SHARED 
+  add_library(extempore SHARED
     src/Extempore.cpp
     src/AudioDevice.cpp
     src/EXTZones.cpp
     src/EXTClosureAddressTable.cpp
     src/EXTLLVM.cpp
     src/EXTThread.cpp
     src/Extempore.cpp
+    src/shims/__hash_memory.cpp
     src/OSC.cpp
     src/Scheme.cpp
     src/SchemeFFI.cpp
@@ -306,6 +351,7 @@ else()
     src/EXTLLVM.cpp
     src/EXTThread.cpp
     src/Extempore.cpp
+    src/shims/__hash_memory.cpp
     src/OSC.cpp
     src/Scheme.cpp
     src/SchemeFFI.cpp
@@ -361,14 +407,16 @@ endif()
 if(PACKAGE)
   target_compile_definitions(extempore
     PRIVATE -DEXT_SHARE_DIR=".")
+  if(NOT CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64")
   target_compile_options(extempore
     PRIVATE -mtune=generic)
+  endif()
 elseif(EXT_SHARE_DIR)
   target_compile_definitions(extempore
     PRIVATE -DEXT_SHARE_DIR="${EXT_SHARE_DIR}")
 elseif(EXT_DYLIB)
   target_compile_definitions(extempore
-    PRIVATE -DEXT_DYLIB=1 
+    PRIVATE -DEXT_DYLIB=1
     PRIVATE -DEXT_SHARE_DIR="."
   )
 else()
@@ -385,7 +433,7 @@ if(UNIX)
     PRIVATE -D__STDC_FORMAT_MACROS
     PRIVATE -D__STDC_LIMIT_MACROS)
   target_compile_options(extempore
-    PRIVATE -std=c++11
+    PRIVATE -std=c++17
     PRIVATE -fvisibility-inlines-hidden
     # PRIVATE -fno-exceptions
     PRIVATE -fno-rtti
@@ -399,13 +447,7 @@ endif()
 if(WIN32)
   target_compile_definitions(extempore
     PRIVATE -DPCRE_STATIC
-    PRIVATE -D_CRT_SECURE_NO_WARNINGS
-	# NOTE: this next define is necessary because VS2019 deprecated the std::tr2
-	# namespace, but setting CXX_STANDARD to c++17 (required for "normal"
-	# std::filesystem) breaks a bunch of LLVM 3.8. So, when we finally upgrade
-	# LLVM, we should switch to std::filesystem, but for now let's just hold our
-	# nose and do this.
-	PRIVATE -D_SILENCE_EXPERIMENTAL_FILESYSTEM_DEPRECATION_WARNING)
+    PRIVATE -D_CRT_SECURE_NO_WARNINGS)
   set_source_files_properties(
     PROPERTIES
     COMPILE_FLAGS "/EHsc")
@@ -535,6 +577,8 @@ else(WIN32)
     if(NOT ${group} STREQUAL "core")
       add_dependencies(${targetname} external_shlibs_${group})
       add_dependencies(aot_external_${group} ${targetname})
+    else()
+      add_dependencies(aot_core ${targetname})
     endif()
     foreach(dep ${ARGN})
       add_dependencies(${targetname} aot_${dep})

include/EXTClosureAddressTable.h

@@ -4,6 +4,8 @@
 
 #include <cinttypes>
 
+struct llvm_zone_t;
+
 namespace extemp {
 namespace ClosureAddressTable {
     ///////////////////////////////////////////////////////////////////////
@@ -19,6 +21,7 @@ namespace ClosureAddressTable {
     };
 
     EXPORT closure_address_table* get_address_table(const char *name, extemp::ClosureAddressTable::closure_address_table *table);
+    EXPORT closure_address_table* add_address_table(llvm_zone_t* zone, char* name, uint32_t offset, char* type, int alloctype, closure_address_table* table);
 
     EXPORT uint32_t get_address_offset(uint64_t id, closure_address_table* table);
     EXPORT bool check_address_exists(uint64_t id, closure_address_table* table);

include/EXTLLVM.h

@@ -45,6 +45,9 @@
 #include <string>
 #include <memory>
 
+#include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
+#include "llvm/Support/Error.h"
+
 
 struct _llvm_callback_struct_ {
     void(*fptr)(void*,llvm_zone_t*);
@@ -94,15 +97,11 @@ class GlobalVariable;
 class GlobalValue;
 class Function;
 class StructType;
-class ModuleProvider;
-class SectionMemoryManager;
-class ExecutionEngine;
-
-namespace legacy
-{
-
-class PassManager;
+class LLVMContext;
 
+namespace orc {
+class LLJIT;
+class ThreadSafeContext;
 }
 
 } // end llvm namespace
@@ -113,16 +112,23 @@ namespace extemp
 namespace EXTLLVM
 {
 
-uint64_t getSymbolAddress(const std::string&);
+uint64_t getFunctionAddress(const std::string&);
 void addModule(llvm::Module* m);
 
-extern llvm::ExecutionEngine* EE; // TODO: nobody should need this (?)
-extern llvm::Module* M;
+extern std::unique_ptr<llvm::orc::LLJIT> JIT;
+
+extern std::unique_ptr<llvm::orc::ThreadSafeContext> TSC;
+
+llvm::orc::ThreadSafeContext& getThreadSafeContext();
+bool removeSymbol(const std::string& name);
+void removeFromGlobalMap(const std::string& name);
+
+llvm::Error addTrackedModule(llvm::orc::ThreadSafeModule TSM, const std::vector<std::string>& symbolNames);
+
 extern int64_t LLVM_COUNT;
 extern bool OPTIMIZE_COMPILES;
 extern bool VERIFY_COMPILES;
-extern llvm::legacy::PassManager* PM;
-extern llvm::legacy::PassManager* PM_NO;
+extern int OPTIMIZATION_LEVEL;  // 0=O0, 1=O1, 2=O2, 3=O3
 extern std::vector<llvm::Module*> Ms;
 
 void initLLVM();

include/EXTZones.h

@@ -41,11 +41,21 @@ namespace EXTZones {
   EXPORT void llvm_zone_destroy(llvm_zone_t* Zone);
   llvm_zone_t* llvm_zone_reset(llvm_zone_t* Zone);
   EXPORT void* llvm_zone_malloc(llvm_zone_t* zone, uint64_t size);
+  EXPORT void* llvm_zone_malloc_from_current_zone(uint64_t size);
+  EXPORT void llvm_zone_print(llvm_zone_t* zone);
+  EXPORT uint64_t llvm_zone_ptr_size(void* ptr);
+  EXPORT bool llvm_zone_copy_ptr(void* ptr1, void* ptr2);
+  EXPORT bool llvm_ptr_in_zone(llvm_zone_t* zone, void* ptr);
+  EXPORT bool llvm_ptr_in_current_zone(void* ptr);
   llvm_zone_stack* llvm_threads_get_zone_stack();
   void llvm_threads_set_zone_stack(llvm_zone_stack* Stack);
   void llvm_push_zone_stack(llvm_zone_t* Zone);
   llvm_zone_t* llvm_peek_zone_stack();
   EXPORT llvm_zone_t* llvm_pop_zone_stack();
+  EXPORT llvm_zone_t* llvm_peek_zone_stack_extern();
+  EXPORT void llvm_push_zone_stack_extern(llvm_zone_t* Zone);
+  EXPORT llvm_zone_t* llvm_zone_create_extern(uint64_t Size);
+  EXPORT llvm_zone_t* llvm_zone_callback_setup();
   void llvm_threads_inc_zone_stacksize();
   void llvm_threads_dec_zone_stacksize();
   uint64_t llvm_threads_get_zone_stacksize();

include/UNIV.h

@@ -187,7 +187,7 @@ inline void ascii_text_color(bool Bold, unsigned Foreground, unsigned Background
     }
 #ifdef _WIN32
     extern int WINDOWS_COLORS[];
-    extern int WINDOWS_BGCOLORS[];    
+    extern int WINDOWS_BGCOLORS[];
     if (unlikely(extemp::UNIV::EXT_TERM == 1)) {
       Foreground = (Foreground > 7) ? 7 : Foreground;
       Background = (Background > 7) ? 0 : Background;