Change how we create and represent patchpoints

src/asm_writing/icinfo.cpp

@@ -198,8 +198,8 @@ ICInfo::ICInfo(void* start_addr, void* continue_addr, StackInfo stack_info, int
 }
 
 static std::unordered_map<void*, ICInfo*> ics_by_return_addr;
-void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, PatchpointSetupInfo* pp,
-                                StackInfo stack_info, std::unordered_set<int> live_outs) {
+void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, const ICSetupInfo* pp, StackInfo stack_info,
+                                std::unordered_set<int> live_outs) {
     int size = pp->totalSize();
     uint8_t* end_addr = start_addr + size;
     uint8_t* slowpath_addr = end_addr;

src/asm_writing/icinfo.h

@@ -127,9 +127,9 @@ public:
     friend class ICSlotRewrite;
 };
 
-class PatchpointSetupInfo;
+class ICSetupInfo;
 class CompiledFunction;
-void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, PatchpointSetupInfo*, StackInfo stack_info,
+void registerCompiledPatchpoint(CompiledFunction* cf, uint8_t* start_addr, const ICSetupInfo*, StackInfo stack_info,
                                 std::unordered_set<int> live_outs);
 
 ICInfo* getICInfo(void* rtn_addr);

src/codegen/irgen.h

@@ -52,7 +52,7 @@ public:
     void setEmitter(IREmitter* emitter) { this->emitter = emitter; }
 };
 
-class PatchpointSetupInfo;
+class ICSetupInfo;
 
 class IREmitter {
 public:
@@ -69,13 +69,13 @@ public:
 
     virtual llvm::Function* getIntrinsic(llvm::Intrinsic::ID) = 0;
 
-    virtual llvm::CallSite createCall(ExcInfo exc_info, llvm::Value* callee, const std::vector<llvm::Value*>& args) = 0;
-    virtual llvm::CallSite createCall(ExcInfo exc_info, llvm::Value* callee, llvm::Value* arg1) = 0;
-    virtual llvm::CallSite createCall2(ExcInfo exc_info, llvm::Value* callee, llvm::Value* arg1, llvm::Value* arg2) = 0;
-    virtual llvm::CallSite createCall3(ExcInfo exc_info, llvm::Value* callee, llvm::Value* arg1, llvm::Value* arg2,
-                                       llvm::Value* arg3) = 0;
-    virtual llvm::CallSite createPatchpoint(const PatchpointSetupInfo* pp, void* func_addr,
-                                            const std::vector<llvm::Value*>& args, ExcInfo exc_info) = 0;
+    virtual llvm::Value* createCall(ExcInfo exc_info, llvm::Value* callee, const std::vector<llvm::Value*>& args) = 0;
+    virtual llvm::Value* createCall(ExcInfo exc_info, llvm::Value* callee, llvm::Value* arg1) = 0;
+    virtual llvm::Value* createCall2(ExcInfo exc_info, llvm::Value* callee, llvm::Value* arg1, llvm::Value* arg2) = 0;
+    virtual llvm::Value* createCall3(ExcInfo exc_info, llvm::Value* callee, llvm::Value* arg1, llvm::Value* arg2,
+                                     llvm::Value* arg3) = 0;
+    virtual llvm::Value* createIC(const ICSetupInfo* pp, void* func_addr, const std::vector<llvm::Value*>& args,
+                                  ExcInfo exc_info) = 0;
 };
 
 CompiledFunction* doCompile(SourceInfo* source, const OSREntryDescriptor* entry_descriptor,

src/codegen/irgen/hooks.cpp

@@ -129,7 +129,8 @@ static void compileIR(CompiledFunction* cf, EffortLevel::EffortLevel effort) {
     }
 
     StackMap* stackmap = parseStackMap();
-    patchpoints::processStackmap(cf, stackmap);
+    processStackmap(cf, stackmap);
+    delete stackmap;
 }
 
 static std::unordered_map<std::string, CompiledFunction*> machine_name_to_cf;

src/codegen/irgen/irgenerator.h

@@ -199,7 +199,7 @@ public:
 };
 
 class IREmitter;
-IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock);
+IREmitter* createIREmitter(IRGenState* irstate, llvm::BasicBlock*& curblock, IRGenerator* irgenerator = NULL);
 IRGenerator* createIRGenerator(IRGenState* irstate, std::unordered_map<CFGBlock*, llvm::BasicBlock*>& entry_blocks,
                                CFGBlock* myblock, TypeAnalysis* types, GuardList& out_guards,
                                const GuardList& in_guards, bool is_partial);

src/codegen/llvm_interpreter.cpp

@@ -27,6 +27,7 @@
 #include "codegen/codegen.h"
 #include "codegen/irgen/hooks.h"
 #include "codegen/irgen/util.h"
+#include "codegen/patchpoints.h"
 #include "core/common.h"
 #include "core/stats.h"
 #include "core/thread_utils.h"
@@ -566,32 +567,36 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                 llvm::InvokeInst* invoke = llvm::dyn_cast<llvm::InvokeInst>(inst);
 
                 void* f;
-                int arg_start;
+                int arg_start, num_args;
                 if (cs.getCalledFunction()
                     && (cs.getCalledFunction()->getName() == "llvm.experimental.patchpoint.void"
-                        || cs.getCalledFunction()->getName() == "llvm.experimental.patchpoint.i64")) {
-                    // cs.dump();
-                    assert(0 && "shouldn't be generating patchpoints for interpretation!");
+                        || cs.getCalledFunction()->getName() == "llvm.experimental.patchpoint.i64"
+                        || cs.getCalledFunction()->getName() == "llvm.experimental.patchpoint.double")) {
+// cs.dump();
+#ifndef NDEBUG
+                    // We use size == CALL_ONLY_SIZE to imply that the call isn't patchable
+                    int pp_size = (int64_t)fetch(cs.getArgument(1), dl, symbols).n;
+                    ASSERT(pp_size == CALL_ONLY_SIZE, "shouldn't be generating patchpoints for interpretation");
+#endif
+
                     f = (void*)fetch(cs.getArgument(2), dl, symbols).n;
                     arg_start = 4;
+                    num_args = (int64_t)fetch(cs.getArgument(3), dl, symbols).n;
                 } else {
                     f = (void*)fetch(cs.getCalledValue(), dl, symbols).n;
                     arg_start = 0;
+                    num_args = cs.arg_size();
                 }
 
                 if (VERBOSITY("interpreter") >= 2)
                     printf("calling %s\n", g.func_addr_registry.getFuncNameAtAddress(f, true).c_str());
 
                 std::vector<Val> args;
-                int nargs = cs.arg_size();
-                for (int i = arg_start; i < nargs; i++) {
+                for (int i = arg_start; i < arg_start + num_args; i++) {
                     // cs.getArgument(i)->dump();
                     args.push_back(fetch(cs.getArgument(i), dl, symbols));
                 }
 
-                int npassed_args = nargs - arg_start;
-// printf("%d %d %d\n", nargs, arg_start, npassed_args);
-
 #ifdef TIME_INTERPRETS
                 this_us += _t.end();
 #endif
@@ -603,7 +608,7 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                 else
                     mask = 2;
 
-                for (int i = 0; i < npassed_args; i++) {
+                for (int i = arg_start; i < arg_start + num_args; i++) {
                     mask <<= 1;
                     if (cs.getArgument(i)->getType() == g.double_)
                         mask |= 1;
@@ -636,6 +641,9 @@ Box* interpretFunction(llvm::Function* f, int nargs, Box* closure, Box* generato
                         case 0b1011:
                             r = reinterpret_cast<int64_t (*)(double, double)>(f)(args[0].d, args[1].d);
                             break;
+                        case 0b1100:
+                            r = reinterpret_cast<double (*)(int64_t, int64_t)>(f)(args[0].n, args[1].n);
+                            break;
                         case 0b1111:
                             r = reinterpret_cast<double (*)(double, double)>(f)(args[0].d, args[1].d);
                             break;

src/codegen/patchpoints.h

@@ -20,85 +20,102 @@
 
 #include "llvm/IR/CallingConv.h"
 
+#include "codegen/stackmaps.h"
+#include "core/common.h"
+
 namespace pyston {
 
+class CompiledFunction;
+class CompilerType;
+struct StackMap;
 class TypeRecorder;
+class ICSetupInfo;
 
-namespace patchpoints {
-
-enum PatchpointType {
-    Generic,
-    Callsite,
-    GetGlobal,
-    Getattr,
-    Setattr,
-    Delattr,
-    Getitem,
-    Setitem,
-    Delitem,
-    Binexp,
-    Nonzero,
-};
-}
+static const int CALL_ONLY_SIZE = 13;
 
-class CompiledFunction;
+void processStackmap(CompiledFunction* cf, StackMap* stackmap);
 
-class PatchpointSetupInfo {
+struct PatchpointInfo {
 private:
-    PatchpointSetupInfo(int64_t pp_id, patchpoints::PatchpointType type, int num_slots, int slot_size,
-                        CompiledFunction* parent_cf, bool has_return_value, TypeRecorder* type_recorder)
-        : pp_id(pp_id), type(type), num_slots(num_slots), slot_size(slot_size), has_return_value(has_return_value),
-          parent_cf(parent_cf), type_recorder(type_recorder) {}
+    CompiledFunction* const parent_cf;
+    const ICSetupInfo* icinfo;
+    int num_ic_stackmap_args;
+
+    PatchpointInfo(CompiledFunction* parent_cf, const ICSetupInfo* icinfo, int num_ic_stackmap_args)
+        : parent_cf(parent_cf), icinfo(icinfo), num_ic_stackmap_args(num_ic_stackmap_args) {}
+
+public:
+    const ICSetupInfo* getICInfo() { return icinfo; }
+
+    int patchpointSize();
+    CompiledFunction* parentFunction() { return parent_cf; }
+
+    int scratchStackmapArg() { return 0; }
+    int scratchSize() { return 80; }
+
+    int icStackmapArgsStart() { return 1; }
+    int numICStackmapArgs() { return num_ic_stackmap_args; }
 
-    const int64_t pp_id;
+    int totalStackmapArgs() { return icStackmapArgsStart() + numICStackmapArgs(); }
 
+    static PatchpointInfo* create(CompiledFunction* parent_cf, const ICSetupInfo* icinfo, int num_ic_stackmap_args);
+};
+
+class ICSetupInfo {
 public:
-    const patchpoints::PatchpointType type;
+    enum ICType {
+        Generic,
+        Callsite,
+        GetGlobal,
+        Getattr,
+        Setattr,
+        Delattr,
+        Getitem,
+        Setitem,
+        Delitem,
+        Binexp,
+        Nonzero,
+    };
+
+private:
+    ICSetupInfo(ICType type, int num_slots, int slot_size, bool has_return_value, TypeRecorder* type_recorder)
+        : type(type), num_slots(num_slots), slot_size(slot_size), has_return_value(has_return_value),
+          type_recorder(type_recorder) {}
+
+public:
+    const ICType type;
 
     const int num_slots, slot_size;
     const bool has_return_value;
-    CompiledFunction* const parent_cf;
     TypeRecorder* const type_recorder;
 
     int totalSize() const;
-    int64_t getPatchpointId() const;
     bool hasReturnValue() const { return has_return_value; }
 
-    int numScratchBytes() const { return 64; }
-
     llvm::CallingConv::ID getCallingConvention() const {
         // The plan is to switch probably everything over to PreseveAll (and potentially AnyReg),
         // but for only switch Getattr so the testing can be localized:
-        if (type == patchpoints::Getattr || type == patchpoints::Setattr)
+        if (type == Getattr || type == Setattr)
             return llvm::CallingConv::PreserveAll;
 
         return llvm::CallingConv::C;
     }
 
-    static PatchpointSetupInfo* initialize(bool has_return_value, int num_slots, int slot_size,
-                                           CompiledFunction* parent_cf, patchpoints::PatchpointType type,
-                                           TypeRecorder* type_recorder);
+    static ICSetupInfo* initialize(bool has_return_value, int num_slots, int slot_size, ICType type,
+                                   TypeRecorder* type_recorder);
 };
 
-struct StackMap;
-
-namespace patchpoints {
-
-void processStackmap(CompiledFunction* cf, StackMap* stackmap);
-
-PatchpointSetupInfo* createGenericPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder,
-                                             bool has_return_value, int size);
-PatchpointSetupInfo* createCallsitePatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder, int num_args);
-PatchpointSetupInfo* createGetGlobalPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createGetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createSetattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createDelattrPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createGetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createSetitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createDelitemPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createBinexpPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-PatchpointSetupInfo* createNonzeroPatchpoint(CompiledFunction* parent_cf, TypeRecorder* type_recorder);
-}
+ICSetupInfo* createGenericIC(TypeRecorder* type_recorder, bool has_return_value, int size);
+ICSetupInfo* createCallsiteIC(TypeRecorder* type_recorder, int num_args);
+ICSetupInfo* createGetGlobalIC(TypeRecorder* type_recorder);
+ICSetupInfo* createGetattrIC(TypeRecorder* type_recorder);
+ICSetupInfo* createSetattrIC(TypeRecorder* type_recorder);
+ICSetupInfo* createDelattrIC(TypeRecorder* type_recorder);
+ICSetupInfo* createGetitemIC(TypeRecorder* type_recorder);
+ICSetupInfo* createSetitemIC(TypeRecorder* type_recorder);
+ICSetupInfo* createDelitemIC(TypeRecorder* type_recorder);
+ICSetupInfo* createBinexpIC(TypeRecorder* type_recorder);
+ICSetupInfo* createNonzeroIC(TypeRecorder* type_recorder);
 
 } // namespace pyston
 

src/codegen/stackmaps.h

@@ -32,7 +32,14 @@ struct StackMap {
 
     struct Record {
         struct __attribute__((__packed__)) Location {
-            uint8_t type;
+            enum LocationType : uint8_t {
+                Register = 0x1,
+                Direct = 0x2,
+                Indirect = 0x3,
+                Constant = 0x4,
+                ConstIndex = 0x5,
+            } type;
+
             uint8_t flags;
             uint16_t regnum;
             int32_t offset;