Add a clang libTooling example

Surprisingly hard to get the build to work.  I feel like no one must use
libTooling for out-of-tree projects.  Also, the binary can only be run from
the directory in which it would have been placed for an in-tree project.

It was going to be a refcount discipline checker, but I'm not sure that's
a good use of time right now.  I want to check it in though so that we have it.

CMakeLists.txt

@@ -356,11 +356,17 @@ else()
   add_custom_target(docs COMMAND ${CMAKE_COMMAND} -E echo "Can't create docs, doxygen not installed \(try sudo apt-get install doxygen grpahviz on Ubuntu and then rerun cmake\)" VERBATIM)
 endif()
 
-set(CMAKE_EXECUTABLE_FORMAT "ELF")
-install(TARGETS pyston DESTINATION ".")
+add_subdirectory(plugins/refcount_checker EXCLUDE_FROM_ALL)
+
+
+# CPack config.  I think this needs to come after any other code, since it will
+# look at whatever install targets have already been set up.
 
 set(CPACK_GENERATOR "TGZ")
 
+set(CMAKE_EXECUTABLE_FORMAT "ELF") # Otherwise cmake thinks this is a cross-compile
+install(TARGETS pyston DESTINATION ".")
+
 set(CPACK_PACKAGE_VERSION_MAJOR "0")
 set(CPACK_PACKAGE_VERSION_MINOR "4")
 set(CPACK_PACKAGE_VERSION_PATCH "0")
@@ -378,5 +384,6 @@ install(FILES test/lib/virtualenv/virtualenv_support/setuptools-12.0.5-py2.py3-n
 
 include(CPack)
 
+
 # last file added (need to change this if we add a file that is added via a glob):
 # from_cpython/Lib/test/test_zipimport.py

Makefile

@@ -647,17 +647,19 @@ cmake_check:
 	@cmake --version >/dev/null || (echo "cmake not available"; false)
 	@$(NINJA) --version >/dev/null || (echo "ninja not available"; false)
 
+COMMON_CMAKE_OPTIONS := $(SRC_DIR) -DTEST_THREADS=$(TEST_THREADS) $(CMAKE_VALGRIND) -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -GNinja
+
 .PHONY: cmake_check clang_check
 $(CMAKE_SETUP_DBG):
 	@$(MAKE) cmake_check
 	@$(MAKE) clang_check
 	@mkdir -p $(CMAKE_DIR_DBG)
-	cd $(CMAKE_DIR_DBG); CC='clang' CXX='clang++' cmake -GNinja $(SRC_DIR) -DTEST_THREADS=$(TEST_THREADS) -DCMAKE_BUILD_TYPE=Debug $(CMAKE_VALGRIND)
+	cd $(CMAKE_DIR_DBG); CC='clang' CXX='clang++' cmake $(COMMON_CMAKE_OPTIONS) -DCMAKE_BUILD_TYPE=Debug
 $(CMAKE_SETUP_RELEASE):
 	@$(MAKE) cmake_check
 	@$(MAKE) clang_check
 	@mkdir -p $(CMAKE_DIR_RELEASE)
-	cd $(CMAKE_DIR_RELEASE); CC='clang' CXX='clang++' cmake -GNinja $(SRC_DIR) -DTEST_THREADS=$(TEST_THREADS) -DCMAKE_BUILD_TYPE=Release
+	cd $(CMAKE_DIR_RELEASE); CC='clang' CXX='clang++' cmake $(COMMON_CMAKE_OPTIONS) -DCMAKE_BUILD_TYPE=Release
 
 # Shared modules (ie extension modules that get built using pyston on setup.py) that we will ask CMake
 # to build.  You can flip this off to allow builds to continue even if self-hosting the sharedmods would fail.
@@ -675,7 +677,7 @@ CMAKE_SETUP_GCC := $(CMAKE_DIR_GCC)/build.ninja
 $(CMAKE_SETUP_GCC):
 	@$(MAKE) cmake_check
 	@mkdir -p $(CMAKE_DIR_GCC)
-	cd $(CMAKE_DIR_GCC); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Debug $(CMAKE_VALGRIND)
+	cd $(CMAKE_DIR_GCC); CC='$(GCC)' CXX='$(GPP)' cmake $(COMMON_CMAKE_OPTIONS) -DCMAKE_BUILD_TYPE=Debug
 .PHONY: pyston_gcc
 pyston_gcc: $(CMAKE_SETUP_GCC)
 	$(NINJA) -C $(CMAKE_DIR_GCC) pyston copy_stdlib copy_libpyston $(CMAKE_SHAREDMODS) ext_cpython $(NINJAFLAGS)
@@ -685,7 +687,7 @@ CMAKE_SETUP_RELEASE_GCC := $(CMAKE_DIR_RELEASE_GCC)/build.ninja
 $(CMAKE_SETUP_RELEASE_GCC):
 	@$(MAKE) cmake_check
 	@mkdir -p $(CMAKE_DIR_RELEASE_GCC)
-	cd $(CMAKE_DIR_RELEASE_GCC); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Release $(CMAKE_VALGRIND)
+	cd $(CMAKE_DIR_RELEASE_GCC); CC='$(GCC)' CXX='$(GPP)' cmake $(COMMON_CMAKE_OPTIONS)  -DCMAKE_BUILD_TYPE=Release
 .PHONY: pyston_release_gcc
 pyston_release_gcc: $(CMAKE_SETUP_RELEASE_GCC)
 	$(NINJA) -C $(CMAKE_DIR_RELEASE_GCC) pyston copy_stdlib copy_libpyston $(CMAKE_SHAREDMODS) ext_cpython $(NINJAFLAGS)
@@ -697,7 +699,7 @@ CMAKE_SETUP_RELEASE_GCC_PGO := $(CMAKE_DIR_RELEASE_GCC_PGO)/build.ninja
 $(CMAKE_SETUP_RELEASE_GCC_PGO):
 	@$(MAKE) cmake_check
 	@mkdir -p $(CMAKE_DIR_RELEASE_GCC_PGO)
-	cd $(CMAKE_DIR_RELEASE_GCC_PGO); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Release $(CMAKE_VALGRIND) -DENABLE_PGO=ON -DPROFILE_STATE=use
+	cd $(CMAKE_DIR_RELEASE_GCC_PGO); CC='$(GCC)' CXX='$(GPP)' cmake $(COMMON_CMAKE_OPTIONS) -DCMAKE_BUILD_TYPE=Release -DENABLE_PGO=ON -DPROFILE_STATE=use
 .PHONY: pyston_release_gcc_pgo
 pyston_release_gcc_pgo: $(CMAKE_SETUP_RELEASE_GCC_PGO) $(CMAKE_DIR_RELEASE_GCC_PGO)/.trained
 	$(NINJA) -C $(CMAKE_DIR_RELEASE_GCC_PGO) pyston copy_stdlib copy_libpyston $(CMAKE_SHAREDMODS) ext_cpython $(NINJAFLAGS)
@@ -707,7 +709,7 @@ CMAKE_SETUP_RELEASE_GCC_PGO_INSTRUMENTED := $(CMAKE_DIR_RELEASE_GCC_PGO_INSTRUME
 $(CMAKE_SETUP_RELEASE_GCC_PGO_INSTRUMENTED):
 	@$(MAKE) cmake_check
 	@mkdir -p $(CMAKE_DIR_RELEASE_GCC_PGO_INSTRUMENTED)
-	cd $(CMAKE_DIR_RELEASE_GCC_PGO_INSTRUMENTED); CC='$(GCC)' CXX='$(GPP)' cmake -GNinja $(SRC_DIR) -DCMAKE_BUILD_TYPE=Release $(CMAKE_VALGRIND) -DENABLE_PGO=ON -DPROFILE_STATE=generate -DPROFILE_DIR=$(CMAKE_DIR_RELEASE_GCC_PGO)
+	cd $(CMAKE_DIR_RELEASE_GCC_PGO_INSTRUMENTED); CC='$(GCC)' CXX='$(GPP)' cmake $(COMMON_CMAKE_OPTIONS) -DCMAKE_BUILD_TYPE=Release -DENABLE_PGO=ON -DPROFILE_STATE=generate -DPROFILE_DIR=$(CMAKE_DIR_RELEASE_GCC_PGO)
 
 .PHONY: pyston_release_gcc_pgo_instrumented
 pyston_release_gcc_pgo_instrumented: $(CMAKE_SETUP_RELEASE_GCC_PGO_INSTRUMENTED)
@@ -1047,15 +1049,16 @@ update_section_ordering: pyston_release
 
 # TESTING:
 
-plugins/clang_linter.o: plugins/clang_linter.cpp $(BUILD_SYSTEM_DEPS)
-	ninja -C $(CMAKE_DIR_DBG) llvm/bin/llvm-config clangASTMatchers clangTooling LLVMLTO LLVMDebugInfoPDB LLVMLineEditor LLVMInterpreter LLVMOrcJIT
-	$(CXX) $< -o $@ -std=c++11 $(shell $(LLVM_BIN_DBG)/llvm-config --cxxflags) -fno-rtti -O0 -I$(LLVM_SRC)/tools/clang/include -I$(LLVM_INC_DBG)/tools/clang/include -c
+PLUGINS := $(wildcard plugins/*.cpp)
+$(patsubst %.cpp,%.o,$(PLUGINS)): plugins/%.o: plugins/%.cpp $(BUILD_SYSTEM_DEPS)
+	ninja -C $(CMAKE_DIR_DBG) llvm/bin/llvm-config clangASTMatchers clangTooling LLVMLTO LLVMDebugInfoPDB LLVMLineEditor LLVMInterpreter LLVMOrcJIT llvm/bin/clang
+	$(CMAKE_DIR_DBG)/llvm/bin/clang $< -o $@ -std=c++11 $(shell $(LLVM_BIN_DBG)/llvm-config --cxxflags) -fno-rtti -O0 -I$(LLVM_SRC)/tools/clang/include -I$(LLVM_INC_DBG)/tools/clang/include -c
 
-plugins/clang_linter.so: plugins/clang_linter.o
-	$(CXX) $< -o $@ -shared -lclangASTMatchers -lclangTooling $(shell $(LLVM_BIN_DBG)/llvm-config --ldflags)
+$(patsubst %.cpp,%.so,$(PLUGINS)): plugins/%.so: plugins/%.o
+	$(CMAKE_DIR_DBG)/llvm/bin/clang $< -o $@ -shared -lclangASTMatchers -lclangTooling $(shell $(LLVM_BIN_DBG)/llvm-config --ldflags)
 	# $(CXX) $< -o $@ -lclangASTMatchers -lclangRewrite -lclangFrontend -lclangDriver -lclangTooling -lclangParse -lclangSema -lclangAnalysis -lclangAST -lclangEdit -lclangLex -lclangBasic -lclangSerialization $(shell $(LLVM_BIN_DBG)/llvm-config --ldflags --system-libs --libs all)
 
-plugins/clang_linter: plugins/clang_linter.o $(BUILD_SYSTEM_DEPS)
+$(patsubst %.cpp,%,$(PLUGINS)): plugins/%: plugins/%.o $(BUILD_SYSTEM_DEPS)
 	$(CXX) $< -o $@ -lclangASTMatchers -lclangRewrite -lclangFrontend -lclangDriver -lclangTooling -lclangParse -lclangSema -lclangAnalysis -lclangAST -lclangEdit -lclangLex -lclangBasic -lclangSerialization $(shell $(LLVM_BIN_DBG)/llvm-config --ldflags --system-libs --libs all)
 
 .PHONY: tool_test
@@ -1071,6 +1074,9 @@ lint_%: %.cpp plugins/clang_linter.so
 	$(ECHO) Linting $<
 	$(VERB) $(CLANG_CXX) -Xclang -load -Xclang plugins/clang_linter.so -Xclang -plugin -Xclang pyston-linter src/runtime/float.cpp $< -c -Isrc/ -Ifrom_cpython/Include -Ibuild/Debug/from_cpython/Include $(shell $(LLVM_BIN_DBG)/llvm-config --cxxflags) $(COMMON_CXXFLAGS) -no-pedantic -Wno-unused-variable -DNVALGRIND -Wno-invalid-offsetof -Wno-mismatched-tags -Wno-unused-function -Wno-unused-private-field -Wno-sign-compare -DLLVMREV=$(LLVM_REVISION) -Ibuild_deps/lz4/lib -DBINARY_SUFFIX= -DBINARY_STRIPPED_SUFFIX=_stripped  -Ibuild_deps/libpypa/src/ -Wno-covered-switch-default -Ibuild/Debug/libunwind/include -Wno-extern-c-compat -Wno-unused-local-typedef -Wno-inconsistent-missing-override
 
+refcount_checker:
+	$(NINJA) -C $(CMAKE_DIR_DBG) refcount_checker
+
 .PHONY: clang_lint
 clang_lint: $(foreach FN,$(MAIN_SRCS),$(dir $(FN))lint_$(notdir $(FN:.cpp=)))
 

plugins/refcount_checker/CMakeLists.txt

+# from llvm-trunk/tools/clang/CMakeLists.txt:
+set(LLVM_RUNTIME_OUTPUT_INTDIR llvm/bin)
+set(LLVM_LIBRARY_OUTPUT_INTDIR llvm/lib${LLVM_LIBDIR_SUFFIX})
+
+# Pyston addition:
+include_directories("${DEPS_DIR}/llvm-trunk/tools/clang/include")
+include_directories("${CMAKE_BINARY_DIR}/llvm/tools/clang/include")
+
+# From llvm-trunk/tools/clang/tools/clang-check/CMakeLists.txt
+set(LLVM_LINK_COMPONENTS
+  Option
+  Support
+  )
+
+add_clang_executable(refcount_checker
+  refcount_checker.cpp
+  )
+
+target_link_libraries(refcount_checker
+  clangAST
+  clangBasic
+  clangDriver
+  clangFrontend
+  clangRewriteFrontend
+  clangStaticAnalyzerFrontend
+  clangTooling
+  )

plugins/refcount_checker/refcount_checker.cpp

+// Declares clang::SyntaxOnlyAction.
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+// Declares llvm::cl::extrahelp.
+#include "llvm/Support/CommandLine.h"
+
+using namespace clang::tooling;
+using namespace llvm;
+
+// Apply a custom category to all command-line options so that they are the
+// only ones displayed.
+static cl::OptionCategory MyToolCategory("my-tool options");
+
+// CommonOptionsParser declares HelpMessage with a description of the common
+// command-line options related to the compilation database and input files.
+// It's nice to have this help message in all tools.
+static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage);
+
+// A help message for this specific tool can be added afterwards.
+static cl::extrahelp MoreHelp("\nMore help text...");
+
+int main(int argc, const char **argv) {
+  CommonOptionsParser OptionsParser(argc, argv, MyToolCategory);
+  ClangTool Tool(OptionsParser.getCompilations(),
+                 OptionsParser.getSourcePathList());
+  return Tool.run(newFrontendActionFactory<clang::SyntaxOnlyAction>().get());
+}