Support LLVM 12 (#503)

Works around ORCv1 removal by returning to MCJIT. Includes an update to the Makefile to improve compatibility on a variety of systems and linkers. Co-authored-by: Elliott Slaughter <slaughter@cs.stanford.edu>
terralang · Jul 26, 2021 · 13b8b11 · 13b8b11
1 parent d8d9078
commit 13b8b11
Show file tree

Hide file tree

Showing 8 changed files with 137 additions and 63 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu-16.04', 'ubuntu-18.04', 'macos-10.15']
-        llvm: ['3.8', '5.0', '6.0', '7', '8', '9', '10', '11']
+        llvm: ['3.8', '5.0', '6.0', '7', '8', '9', '10', '11', '12']
         cmake: ['0', '1']
         cuda: ['0', '1']
         static: ['0', '1']
@@ -38,10 +38,12 @@ jobs:
             llvm: '10'
           - os: 'ubuntu-16.04'
             llvm: '11'
+          - os: 'ubuntu-16.04'
+            llvm: '12'
           - os: 'ubuntu-18.04'
             llvm: '3.8'
 
-          # macOS: exclude LLVM 5.0, 8-11 make, cuda/no-static/no-slib
+          # macOS: exclude LLVM 5.0, 8-12 make, cuda/no-static/no-slib
           - os: 'macos-10.15'
             llvm: '5.0'
           - os: 'macos-10.15'
@@ -56,6 +58,9 @@ jobs:
           - os: 'macos-10.15'
             llvm: '11'
             cmake: '0'
+          - os: 'macos-10.15'
+            llvm: '12'
+            cmake: '0'
           - os: 'macos-10.15'
             cuda: '1'
           - os: 'macos-10.15'
@@ -103,6 +108,8 @@ jobs:
             cuda: '1'
           - llvm: '11'
             cuda: '1'
+          - llvm: '12'
+            cuda: '1'
 
           # no-static: only LLVM 8
           - llvm: '3.8'
@@ -119,6 +126,8 @@ jobs:
             static: '0'
           - llvm: '11'
             static: '0'
+          - llvm: '12'
+            static: '0'
 
           # no-slib: only LLVM 9
           - llvm: '3.8'
@@ -135,6 +144,8 @@ jobs:
             slib: '0'
           - llvm: '11'
             slib: '0'
+          - llvm: '12'
+            slib: '0'
 
           # Moonjit: only LLVM 9
           - llvm: '3.8'
@@ -151,6 +162,8 @@ jobs:
             lua: 'moonjit'
           - llvm: '11'
             lua: 'moonjit'
+          - llvm: '12'
+            lua: 'moonjit'
     steps:
       - uses: actions/checkout@v1
       - run: ./travis.sh

diff --git a/Makefile b/Makefile
@@ -86,7 +86,7 @@ LLVMVERGT4 := $(shell expr $(LLVM_VERSION) \>= 40)
 
 FLAGS += -DLLVM_VERSION=$(LLVM_VERSION)
 
-LLVM_NEEDS_CXX14="100 110 111"
+LLVM_NEEDS_CXX14="100 110 111 120"
 ifneq (,$(findstring $(LLVM_VERSION),$(LLVM_NEEDS_CXX14)))
 CPPFLAGS += -std=c++1y # GCC 5 does not support -std=c++14 flag
 else
@@ -101,37 +101,37 @@ DYNFLAGS = -dynamiclib -single_module -fPIC -install_name "@rpath/terra.dylib"
 TERRA_STATIC_LIBRARY =  -Wl,-force_load,$(LIBRARY)
 endif
 
-LLVM_LIBRARY_FLAGS += $(LUAJIT_LIB)
-LLVM_LIBRARY_FLAGS += $(shell $(LLVM_CONFIG) --ldflags) -L$(CLANG_PREFIX)/lib
-LLVM_LIBRARY_FLAGS += -lclangFrontend -lclangDriver \
-                      -lclangSerialization -lclangCodeGen -lclangParse -lclangSema \
-                      -lclangAnalysis \
-                      -lclangEdit -lclangAST -lclangLex -lclangBasic
-
-CLANG_AST_MATCHERS = "80 90 100 110 111"
+CLANG_LIBS += libclangFrontend.a \
+	libclangDriver.a \
+	libclangSerialization.a \
+	libclangCodeGen.a \
+	libclangParse.a \
+	libclangSema.a \
+	libclangAnalysis.a \
+	libclangEdit.a \
+	libclangAST.a \
+	libclangLex.a \
+	libclangBasic.a
+
+CLANG_AST_MATCHERS = "80 90 100 110 111 120"
 ifneq (,$(findstring $(LLVM_VERSION),$(CLANG_AST_MATCHERS)))
-LLVM_LIBRARY_FLAGS += -lclangASTMatchers
+CLANG_LIBS += libclangASTMatchers.a
 endif
 
-# by default, Terra includes only the pieces of the LLVM libraries it needs,
-#  but this can be a problem if third-party-libraries that also need LLVM are
-#  used - allow the user to request that some/all of the LLVM components be
-#  included and re-exported in their entirety
+# Get full path to clang libaries
+CLANG_LIBFILES := $(patsubst %, $(CLANG_PREFIX)/lib/%, $(CLANG_LIBS))
+
 ifeq "$(LLVMVERGT4)" "1"
     LLVM_LIBS += $(shell $(LLVM_CONFIG) --libs --link-static)
+	LLVM_LIBFILES := $(shell $(LLVM_CONFIG) --libfiles --link-static)
 else
 	LLVM_LIBS += $(shell $(LLVM_CONFIG) --libs)
+	LLVM_LIBFILES := $(shell $(LLVM_CONFIG) --libfiles)
 endif
-ifneq ($(REEXPORT_LLVM_COMPONENTS),)
-  REEXPORT_LIBS := $(shell $(LLVM_CONFIG) --libs $(REEXPORT_LLVM_COMPONENTS))
-  ifneq ($(findstring $(UNAME), Linux FreeBSD),)
-    LLVM_LIBRARY_FLAGS += -Wl,--whole-archive $(REEXPORT_LIBS) -Wl,--no-whole-archive
-  else
-    LLVM_LIBRARY_FLAGS += $(REEXPORT_LIBS:%=-Wl,-force_load,%)
-  endif
-  LLVM_LIBRARY_FLAGS += $(filter-out $(REEXPORT_LIBS),$(LLVM_LIBS))
-else
-  LLVM_LIBRARY_FLAGS += $(LLVM_LIBS)
+
+LLVM_POLLY = "100 110 111 120"
+ifneq (,$(findstring $(LLVM_VERSION),$(LLVM_POLLY)))
+	LLVM_LIBFILES += $(shell $(LLVM_CONFIG) --libdir)/libPolly*.a
 endif
 
 # llvm sometimes requires ncurses and libz, check if they have the symbols, and add them if they do
@@ -149,6 +149,8 @@ ifeq ($(UNAME), FreeBSD)
 SUPPORT_LIBRARY_FLAGS += -lexecinfo -pthread
 endif
 
+SUPPORT_LIBRARY_FLAGS += -lffi -ledit -lxml2
+
 PACKAGE_DEPS += $(LUAJIT_LIB)
 
 #makes luajit happy on osx 10.6 (otherwise luaL_newstate returns NULL)
@@ -229,11 +231,15 @@ release/include/terra/%.h:  $(LUAJIT_INCLUDE)/%.h $(LUAJIT_LIB)
 
 build/llvm_objects/llvm_list:    $(addprefix build/, $(LIBOBJS) $(EXEOBJS))
 	mkdir -p build/llvm_objects/luajit
-	$(CXX) -o /dev/null $(addprefix build/, $(LIBOBJS) $(EXEOBJS)) $(LLVM_LIBRARY_FLAGS) $(SUPPORT_LIBRARY_FLAGS) $(LFLAGS) -Wl,-t 2>&1 | egrep "lib(LLVM|clang)"  > build/llvm_objects/llvm_list
-	# extract needed LLVM objects based on a dummy linker invocation
-	< build/llvm_objects/llvm_list $(LUAJIT) src/unpacklibraries.lua build/llvm_objects
-	# include all luajit objects, since the entire lua interface is used in terra 
-
+	# Extract Luajit + all LLVM & Clang libraries
+	cd build/llvm_objects; for lib in $(LUAJIT_LIB) $(LLVM_LIBFILES) $(CLANG_LIBFILES); do \
+		echo Extracing objects from $$lib; \
+		DIR=$$(basename $$lib .a); \
+		mkdir -p $$DIR; \
+		cd $$DIR; \
+		ar x $$lib; \
+		cd ..; \
+	done
 
 build/lua_objects/lj_obj.o:    $(LUAJIT_LIB)
 	mkdir -p build/lua_objects

diff --git a/src/llvmheaders.h b/src/llvmheaders.h
@@ -62,6 +62,8 @@
 #include "llvmheaders_100.h"
 #elif LLVM_VERSION >= 110 && LLVM_VERSION < 120
 #include "llvmheaders_110.h"
+#elif LLVM_VERSION == 120
+#include "llvmheaders_120.h"
 #else
 #error "unsupported LLVM version"
 // for OSX code completion

diff --git a/src/llvmheaders_120.h b/src/llvmheaders_120.h
@@ -0,0 +1,37 @@
+#include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/Analysis/CallGraphSCCPass.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/IR/DIBuilder.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/Mangler.h"
+//#include "llvm/ExecutionEngine/ObjectImage.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Linker/Linker.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+
+#include "llvm/Support/VirtualFileSystem.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Object/SymbolSize.h"
+
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Support/Error.h"
+
+#define LLVM_PATH_TYPE std::string
+#define RAW_FD_OSTREAM_NONE sys::fs::F_None
+#define RAW_FD_OSTREAM_BINARY sys::fs::F_None
+#define HASFNATTR(attr) \
+    getAttributes().hasAttribute(AttributeSet::FunctionIndex, Attribute ::attr)
+#define ADDFNATTR(attr) addFnAttr(Attribute ::attr)
+#define ATTRIBUTE Attributes
diff --git a/src/tcompiler.cpp b/src/tcompiler.cpp
@@ -23,8 +23,10 @@ extern "C" {
 #if LLVM_VERSION < 50
 #include "llvm/ExecutionEngine/MCJIT.h"
 #else
+#if LLVM_VERSION < 120
 #include "llvm/ExecutionEngine/OrcMCJITReplacement.h"
 #endif
+#endif
 
 #include "llvm/Support/Atomic.h"
 #include "llvm/Support/FileSystem.h"
@@ -390,7 +392,10 @@ static void InitializeJIT(TerraCompilationUnit *CU) {
 #else
             .setMCJITMemoryManager(std::make_unique<TerraSectionMemoryManager>(CU))
 #endif
-            .setUseOrcMCJITReplacement(true);
+#if LLVM_VERSION < 120
+            .setUseOrcMCJITReplacement(true)
+#endif
+            ;
 #endif
 
     CU->ee = eb.create();
@@ -952,13 +957,21 @@ struct CCallingConv {
     }
 
     template <typename FnOrCall>
-    void addSRetAttr(FnOrCall *r, int idx) {
+    void addSRetAttr(FnOrCall *r, int idx, Type *ty) {
+#if LLVM_VERSION < 120
         r->addAttribute(idx, Attribute::StructRet);
+#else
+        r->addAttribute(idx, Attribute::getWithStructRetType(*CU->TT->ctx, ty));
+#endif
         r->addAttribute(idx, Attribute::NoAlias);
     }
     template <typename FnOrCall>
-    void addByValAttr(FnOrCall *r, int idx) {
+    void addByValAttr(FnOrCall *r, int idx, Type *ty) {
+#if LLVM_VERSION < 120
         r->addAttribute(idx, Attribute::ByVal);
+#else
+        r->addAttribute(idx, Attribute::getWithByValType(*CU->TT->ctx, ty));
+#endif
     }
     template <typename FnOrCall>
     void addExtAttrIfNeeded(TType *t, FnOrCall *r, int idx) {
@@ -971,14 +984,14 @@ struct CCallingConv {
         addExtAttrIfNeeded(info->returntype.type, r, 0);
         int argidx = 1;
         if (info->returntype.kind == C_AGGREGATE_MEM) {
-            addSRetAttr(r, argidx);
+            addSRetAttr(r, argidx, info->returntype.cctype);
             argidx++;
         }
         for (size_t i = 0; i < info->paramtypes.size(); i++) {
             Argument *v = &info->paramtypes[i];
             if (v->kind == C_AGGREGATE_MEM) {
 #ifndef _WIN32
-                addByValAttr(r, argidx);
+                addByValAttr(r, argidx, v->cctype);
 #endif
             }
             addExtAttrIfNeeded(v->type, r, argidx);
@@ -2488,9 +2501,16 @@ struct FunctionEmitter {
         DEBUG_ONLY(T) {
             MDNode *scope = debugScopeForFile(customfilename ? customfilename
                                                              : obj->string("filename"));
+#if LLVM_VERSION < 120
             B->SetCurrentDebugLocation(DebugLoc::get(
                     customfilename ? customlinenumber : obj->number("linenumber"), 0,
                     scope));
+#else
+            B->SetCurrentDebugLocation(DILocation::get(
+                    scope->getContext(),
+                    customfilename ? customlinenumber : obj->number("linenumber"), 0,
+                    scope));
+#endif
         }
     }
 

diff --git a/src/tcwrapper.cpp b/src/tcwrapper.cpp
@@ -328,8 +328,14 @@ class IncludeCVisitor : public RecursiveASTVisitor<IncludeCVisitor> {
     }
     CStyleCastExpr *CreateCast(QualType Ty, CastKind Kind, Expr *E) {
         TypeSourceInfo *TInfo = Context->getTrivialTypeSourceInfo(Ty, SourceLocation());
+#if LLVM_VERSION < 120
         return CStyleCastExpr::Create(*Context, Ty, VK_RValue, Kind, E, 0, TInfo,
                                       SourceLocation(), SourceLocation());
+#else
+        return CStyleCastExpr::Create(*Context, Ty, VK_RValue, Kind, E, 0,
+                                      FPOptionsOverride::getFromOpaqueInt(0), TInfo,
+                                      SourceLocation(), SourceLocation());
+#endif
     }
     IntegerLiteral *LiteralZero() {
         unsigned IntSize = static_cast<unsigned>(Context->getTypeSize(Context->IntTy));

diff --git a/src/unpacklibraries.lua b/src/unpacklibraries.lua
diff --git a/travis.sh b/travis.sh
@@ -26,7 +26,16 @@ fi
 
 if [[ $(uname) = Linux ]]; then
   sudo apt-get update -qq
-  if [[ $LLVM_CONFIG = llvm-config-11 ]]; then
+  if [[ $LLVM_CONFIG = llvm-config-12 ]]; then
+    wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+    sudo add-apt-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main"
+    for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done
+    sudo apt-get install -y llvm-12-dev clang-12 libclang-12-dev libedit-dev
+    export CMAKE_PREFIX_PATH=/usr/lib/llvm-12:/usr/share/llvm-12
+    if [[ -n $STATIC_LLVM && $STATIC_LLVM -eq 0 ]]; then
+        export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/llvm-12/lib"
+    fi
+  elif [[ $LLVM_CONFIG = llvm-config-11 ]]; then
     wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
     sudo add-apt-repository -y "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-11 main"
     for i in {1..5}; do sudo apt-get update -qq && break || sleep 15; done
@@ -81,7 +90,13 @@ if [[ $(uname) = Linux ]]; then
 fi
 
 if [[ $(uname) = Darwin ]]; then
-  if [[ $LLVM_CONFIG = llvm-config-11 ]]; then
+  if [[ $LLVM_CONFIG = llvm-config-12 ]]; then
+    curl -L -O https://github.com/elliottslaughter/llvm-build/releases/download/llvm-12.0.1/clang+llvm-12.0.1-x86_64-apple-darwin.tar.xz
+    tar xf clang+llvm-12.0.1-x86_64-apple-darwin.tar.xz
+    ln -s clang+llvm-12.0.1-x86_64-apple-darwin/bin/llvm-config llvm-config-12
+    ln -s clang+llvm-12.0.1-x86_64-apple-darwin/bin/clang clang-12
+    export CMAKE_PREFIX_PATH=$PWD/clang+llvm-12.0.1-x86_64-apple-darwin
+  elif [[ $LLVM_CONFIG = llvm-config-11 ]]; then
     curl -L -O https://github.com/elliottslaughter/llvm-build/releases/download/llvm-11.0.1/clang+llvm-11.0.1-x86_64-apple-darwin.tar.xz
     tar xf clang+llvm-11.0.1-x86_64-apple-darwin.tar.xz
     ln -s clang+llvm-11.0.1-x86_64-apple-darwin/bin/llvm-config llvm-config-11