Merge pull request #693 from undingen/bjit_pp

baseline jit: directly emit patchpoints instead of using runtime ICs

src/asm_writing/assembler.cpp

@@ -1034,6 +1034,16 @@ void Assembler::emitAnnotation(int num) {
     nop();
 }
 
+void Assembler::skipBytes(int num) {
+    if (addr + num >= end_addr) {
+        addr = end_addr;
+        failed = true;
+        return;
+    }
+
+    addr += num;
+}
+
 ForwardJump::ForwardJump(Assembler& assembler, ConditionCode condition)
     : assembler(assembler), condition(condition), jmp_inst(assembler.curInstPointer()) {
     assembler.jmp_cond(JumpDestination::fromStart(assembler.bytesWritten() + max_jump_size), condition);

src/asm_writing/assembler.h

@@ -193,6 +193,7 @@ public:
     void fillWithNops();
     void fillWithNopsExcept(int bytes);
     void emitAnnotation(int num);
+    void skipBytes(int num);
 
     uint8_t* startAddr() const { return start_addr; }
     int bytesLeft() const { return end_addr - addr; }

src/asm_writing/rewriter.cpp

@@ -792,10 +792,8 @@ RewriterVar* Rewriter::call(bool has_side_effects, void* func_addr, const Rewrit
     return result;
 }
 
-void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr, const RewriterVar::SmallVector& args,
-                     const RewriterVar::SmallVector& args_xmm) {
-    assembler->comment("_call");
-
+void Rewriter::_setupCall(RewriterVar* result, bool has_side_effects, const RewriterVar::SmallVector& args,
+                          const RewriterVar::SmallVector& args_xmm) {
     if (has_side_effects)
         assert(done_guarding);
 
@@ -828,9 +826,6 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
         }
     }
 
-    // RewriterVarUsage scratch = createNewVar(Location::any());
-    assembler::Register r = allocReg(assembler::R11);
-
     for (int i = 0; i < args.size(); i++) {
         Location l(Location::forArg(i));
         RewriterVar* var = args[i];
@@ -950,6 +945,19 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
         assert(!l.isClobberedByCall());
     }
 #endif
+}
+
+void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr, const RewriterVar::SmallVector& args,
+                     const RewriterVar::SmallVector& args_xmm) {
+    assembler->comment("_call");
+
+    // RewriterVarUsage scratch = createNewVar(Location::any());
+    assembler::Register r = allocReg(assembler::R11);
+
+    _setupCall(result, has_side_effects, args, args_xmm);
+
+    // make sure setupCall doesn't use R11
+    assert(vars_by_location.count(assembler::R11) == 0);
 
     uint64_t asm_address = (uint64_t)assembler->curInstPointer() + 5;
     uint64_t real_asm_address = asm_address + (uint64_t)rewrite->getSlotStart() - (uint64_t)assembler->startAddr();
@@ -962,10 +970,12 @@ void Rewriter::_call(RewriterVar* result, bool has_side_effects, void* func_addr
         assert(assembler->hasFailed() || asm_address == (uint64_t)assembler->curInstPointer());
     }
 
-    assert(vars_by_location.count(assembler::RAX) == 0);
-    result->initializeInReg(assembler::RAX);
+    if (!failed) {
+        assert(vars_by_location.count(assembler::RAX) == 0);
+        result->initializeInReg(assembler::RAX);
 
-    assertConsistent();
+        assertConsistent();
+    }
 
     result->releaseIfNoUses();
 }
@@ -1737,10 +1747,10 @@ Rewriter::Rewriter(std::unique_ptr<ICSlotRewrite> rewrite, int num_args, const s
     // the entire scratch space.
     bool VALIDATE_SCRATCH_SPACE = false;
     if (VALIDATE_SCRATCH_SPACE) {
-        int scratch_size = rewrite->getScratchSize();
+        int scratch_size = this->rewrite->getScratchSize();
         for (int i = 0; i < scratch_size; i += 8) {
             assembler->movq(assembler::Immediate(0x12345678UL),
-                            assembler::Indirect(assembler::RSP, i + rewrite->getScratchRspOffset()));
+                            assembler::Indirect(assembler::RSP, i + this->rewrite->getScratchRspOffset()));
         }
     }
 }

src/asm_writing/rewriter.h

@@ -19,6 +19,7 @@
 #include <memory>
 #include <tuple>
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallSet.h"
 
@@ -351,7 +352,7 @@ protected:
     Rewriter(std::unique_ptr<ICSlotRewrite> rewrite, int num_args, const std::vector<int>& live_outs);
 
     llvm::SmallVector<RewriterAction, 32> actions;
-    void addAction(std::function<void()> action, std::vector<RewriterVar*> const& vars, ActionType type) {
+    void addAction(std::function<void()> action, llvm::ArrayRef<RewriterVar*> vars, ActionType type) {
         assertPhaseCollecting();
         for (RewriterVar* var : vars) {
             assert(var != NULL);
@@ -415,6 +416,8 @@ protected:
 
     void _trap();
     void _loadConst(RewriterVar* result, int64_t val);
+    void _setupCall(RewriterVar* result, bool has_side_effects, const RewriterVar::SmallVector& args,
+                    const RewriterVar::SmallVector& args_xmm);
     void _call(RewriterVar* result, bool has_side_effects, void* func_addr, const RewriterVar::SmallVector& args,
                const RewriterVar::SmallVector& args_xmm);
     void _add(RewriterVar* result, RewriterVar* a, int64_t b, Location dest);

src/codegen/baseline_jit.h

@@ -19,6 +19,7 @@
 
 #include "asm_writing/rewriter.h"
 #include "codegen/ast_interpreter.h"
+#include "codegen/patchpoints.h"
 #include "gc/heap.h"
 #include "runtime/ics.h"
 
@@ -32,17 +33,6 @@ class BoxedDict;
 class BoxedList;
 class BoxedTuple;
 
-class SetGlobalIC;
-class GetGlobalIC;
-class SetAttrIC;
-class GetAttrIC;
-class SetItemIC;
-class GetItemIC;
-class CompareIC;
-class AugBinopIC;
-class UnaryopIC;
-class RuntimeCallIC;
-
 class JitFragmentWriter;
 
 // This JIT tier is designed as Pystons entry level JIT tier (executed after a only a few dozens runs of a basic block)
@@ -100,6 +90,7 @@ class JitFragmentWriter;
 //      mov    %rsp,%rbp
 //      sub    $0x108,%rsp          ; setup scratch, 0x108 = scratch_size + 8 (=stack alignment)
 //      push   %rdi                 ; save the pointer to ASTInterpreter instance
+//      sub    $0x16,%rsp           ; space for two func args passed on the stack
 //      jmpq   *0x8(%rsi)           ; jump to block->code
 //                                      possible values: first_JitFragment, second_JitFragment,...
 //
@@ -130,10 +121,14 @@ class JitFragmentWriter;
 //
 //
 class JitCodeBlock {
-private:
+public:
     static constexpr int scratch_size = 256;
-    static constexpr int code_size = 4096 * 2;
+    static constexpr int code_size = 32768;
+    static constexpr int num_stack_args = 2;
+    static constexpr int interpreter_ptr_offset = num_stack_args * 8;
 
+
+private:
     EHFrameManager frame_manager;
     std::unique_ptr<uint8_t[]> code;
     int entry_offset;
@@ -178,6 +173,16 @@ private:
     // it in this field and process it only when we know we successfully generated the code.
     std::pair<CFGBlock*, int /* offset from fragment start*/> side_exit_patch_location;
 
+    struct PPInfo {
+        void* func_addr;
+        uint8_t* start_addr;
+        uint8_t* end_addr;
+        std::unique_ptr<ICSetupInfo> ic;
+        StackInfo stack_info;
+    };
+
+    llvm::SmallVector<PPInfo, 8> pp_infos;
+
 public:
     JitFragmentWriter(CFGBlock* block, std::unique_ptr<ICInfo> ic_info, std::unique_ptr<ICSlotRewrite> rewrite,
                       int code_offset, int num_bytes_overlapping, void* entry_code, JitCodeBlock& code_block);
@@ -250,35 +255,24 @@ private:
 #endif
     RewriterVar* getInterp();
 
-    static Box* augbinopICHelper(AugBinopIC* ic, Box* lhs, Box* rhs, int op);
-    static Box* binopICHelper(BinopIC* ic, Box* lhs, Box* rhs, int op);
+    RewriterVar* emitPPCall(void* func_addr, llvm::ArrayRef<RewriterVar*> args, int num_slots, int slot_size);
+
     static Box* callattrHelper(Box* obj, BoxedString* attr, CallattrFlags flags, Box** args,
                                std::vector<BoxedString*>* keyword_names);
-    static Box* compareICHelper(CompareIC* ic, Box* lhs, Box* rhs, int op);
     static Box* createDictHelper(uint64_t num, Box** keys, Box** values);
     static Box* createListHelper(uint64_t num, Box** data);
     static Box* createSetHelper(uint64_t num, Box** data);
     static Box* createTupleHelper(uint64_t num, Box** data);
     static Box* exceptionMatchesHelper(Box* obj, Box* cls);
-    static Box* getAttrICHelper(GetAttrIC* ic, Box* o, BoxedString* attr);
-    static Box* getGlobalICHelper(GetGlobalIC* ic, Box* o, BoxedString* s);
-    static Box* getitemICHelper(GetItemIC* ic, Box* o, Box* attr);
     static Box* hasnextHelper(Box* b);
     static Box* nonzeroHelper(Box* b);
     static Box* notHelper(Box* b);
     static Box* runtimeCallHelper(Box* obj, ArgPassSpec argspec, Box** args, std::vector<BoxedString*>* keyword_names);
-    static Box* setAttrICHelper(SetAttrIC* ic, Box* o, BoxedString* attr, Box* value);
-    static Box* setGlobalICHelper(SetGlobalIC* ic, Box* o, BoxedString* s, Box* v);
-    static Box* setitemICHelper(SetItemIC* ic, Box* o, Box* attr, Box* value);
-    static Box* unaryopICHelper(UnaryopIC* ic, Box* obj, int op);
-
-#if ENABLE_BASELINEJIT_ICS
-    static Box* callattrHelperIC(Box* obj, BoxedString* attr, CallattrFlags flags, CallattrIC* ic, Box** args);
-    static Box* runtimeCallHelperIC(Box* obj, ArgPassSpec argspec, RuntimeCallIC* ic, Box** args);
-#endif
 
     void _emitJump(CFGBlock* b, RewriterVar* block_next, int& size_of_exit_to_interp);
     void _emitOSRPoint(RewriterVar* result, RewriterVar* node_var);
+    void _emitPPCall(RewriterVar* result, void* func_addr, const RewriterVar::SmallVector& args, int num_slots,
+                     int slot_size);
     void _emitReturn(RewriterVar* v);
     void _emitSideExit(RewriterVar* var, RewriterVar* val_constant, CFGBlock* next_block, RewriterVar* false_path);
 };

src/runtime/ics.h

@@ -77,29 +77,6 @@ public:
     }
 };
 
-class RuntimeCallIC : public RuntimeIC {
-public:
-    RuntimeCallIC() : RuntimeIC((void*)runtimeCall, 2, 320) {}
-
-    Box* call(Box* obj, ArgPassSpec argspec, Box* arg1, Box* arg2, Box* arg3, Box** args) {
-        return (Box*)call_ptr(obj, argspec, arg1, arg2, arg3, args);
-    }
-};
-
-class UnaryopIC : public RuntimeIC {
-public:
-    UnaryopIC() : RuntimeIC((void*)unaryop, 2, 160) {}
-
-    Box* call(Box* obj, int op_type) { return (Box*)call_ptr(obj, op_type); }
-};
-
-class AugBinopIC : public RuntimeIC {
-public:
-    AugBinopIC() : RuntimeIC((void*)augbinop, 2, 240) {}
-
-    Box* call(Box* lhs, Box* rhs, int op_type) { return (Box*)call_ptr(lhs, rhs, op_type); }
-};
-
 class BinopIC : public RuntimeIC {
 public:
     BinopIC() : RuntimeIC((void*)binop, 2, 240) {}
@@ -107,55 +84,6 @@ public:
     Box* call(Box* lhs, Box* rhs, int op_type) { return (Box*)call_ptr(lhs, rhs, op_type); }
 };
 
-class CompareIC : public RuntimeIC {
-public:
-    CompareIC() : RuntimeIC((void*)compare, 2, 240) {}
-
-    Box* call(Box* lhs, Box* rhs, int op_type) { return (Box*)call_ptr(lhs, rhs, op_type); }
-};
-
-class GetItemIC : public RuntimeIC {
-public:
-    GetItemIC() : RuntimeIC((void*)getitem, 2, 512) {}
-
-    Box* call(Box* obj, Box* attr) { return (Box*)call_ptr(obj, attr); }
-};
-
-class SetItemIC : public RuntimeIC {
-public:
-    SetItemIC() : RuntimeIC((void*)setitem, 2, 512) {}
-
-    Box* call(Box* obj, Box* attr, Box* v) { return (Box*)call_ptr(obj, attr, v); }
-};
-
-class GetAttrIC : public RuntimeIC {
-public:
-    GetAttrIC() : RuntimeIC((void*)getattr, 2, 512) {}
-
-    Box* call(Box* obj, BoxedString* attr) { return (Box*)call_ptr(obj, attr); }
-};
-
-class SetAttrIC : public RuntimeIC {
-public:
-    SetAttrIC() : RuntimeIC((void*)setattr, 2, 512) {}
-
-    Box* call(Box* obj, BoxedString* attr, Box* v) { return (Box*)call_ptr(obj, attr, v); }
-};
-
-class GetGlobalIC : public RuntimeIC {
-public:
-    GetGlobalIC() : RuntimeIC((void*)getGlobal, 2, 512) {}
-
-    Box* call(Box* obj, BoxedString* s) { return (Box*)call_ptr(obj, s); }
-};
-
-class SetGlobalIC : public RuntimeIC {
-public:
-    SetGlobalIC() : RuntimeIC((void*)setGlobal, 2, 512) {}
-
-    Box* call(Box* obj, BoxedString* s, Box* v) { return (Box*)call_ptr(obj, s, v); }
-};
-
 class NonzeroIC : public RuntimeIC {
 public:
     NonzeroIC() : RuntimeIC((void*)nonzero, 1, 40) {}

src/runtime/types.cpp

@@ -932,6 +932,8 @@ static Box* typeCallInner(CallRewriteArgs* rewrite_args, ArgPassSpec argspec, Bo
                 rewrite_args->rewriter->call(true, (void*)assertInitNone, srewrite_args.out_rtn);
             }
         } else {
+            rewrite_args = NULL;
+
             init_attr = processDescriptor(init_attr, made, cls);
 
             ArgPassSpec init_argspec = argspec;