1 //===-- JITEmitter.cpp - Write machine code to executable memory ----------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file defines a MachineCodeEmitter object that is used by the JIT to 11 // write machine code to memory and remember where relocatable values are. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #define DEBUG_TYPE "jit" 16 #include "JIT.h" 17 #include "JITDwarfEmitter.h" 18 #include "llvm/ADT/OwningPtr.h" 19 #include "llvm/Constants.h" 20 #include "llvm/DebugInfo.h" 21 #include "llvm/DerivedTypes.h" 22 #include "llvm/Module.h" 23 #include "llvm/CodeGen/JITCodeEmitter.h" 24 #include "llvm/CodeGen/MachineFunction.h" 25 #include "llvm/CodeGen/MachineCodeInfo.h" 26 #include "llvm/CodeGen/MachineConstantPool.h" 27 #include "llvm/CodeGen/MachineJumpTableInfo.h" 28 #include "llvm/CodeGen/MachineModuleInfo.h" 29 #include "llvm/CodeGen/MachineRelocation.h" 30 #include "llvm/ExecutionEngine/GenericValue.h" 31 #include "llvm/ExecutionEngine/JITEventListener.h" 32 #include "llvm/ExecutionEngine/JITMemoryManager.h" 33 #include "llvm/Target/TargetData.h" 34 #include "llvm/Target/TargetInstrInfo.h" 35 #include "llvm/Target/TargetJITInfo.h" 36 #include "llvm/Target/TargetMachine.h" 37 #include "llvm/Target/TargetOptions.h" 38 #include "llvm/Support/Debug.h" 39 #include "llvm/Support/ErrorHandling.h" 40 #include "llvm/Support/ManagedStatic.h" 41 #include "llvm/Support/MutexGuard.h" 42 #include "llvm/Support/ValueHandle.h" 43 #include "llvm/Support/raw_ostream.h" 44 #include "llvm/Support/Disassembler.h" 45 #include "llvm/Support/Memory.h" 46 #include "llvm/ADT/DenseMap.h" 47 #include "llvm/ADT/SmallPtrSet.h" 48 #include "llvm/ADT/SmallVector.h" 49 #include "llvm/ADT/Statistic.h" 50 #include "llvm/ADT/ValueMap.h" 51 #include <algorithm> 52 #ifndef NDEBUG 53 #include <iomanip> 54 #endif 55 using namespace llvm; 56 57 STATISTIC(NumBytes, "Number of bytes of machine code compiled"); 58 STATISTIC(NumRelos, "Number of relocations applied"); 59 STATISTIC(NumRetries, "Number of retries with more memory"); 60 61 62 // A declaration may stop being a declaration once it's fully read from bitcode. 63 // This function returns true if F is fully read and is still a declaration. 64 static bool isNonGhostDeclaration(const Function *F) { 65 return F->isDeclaration() && !F->isMaterializable(); 66 } 67 68 //===----------------------------------------------------------------------===// 69 // JIT lazy compilation code. 70 // 71 namespace { 72 class JITEmitter; 73 class JITResolverState; 74 75 template<typename ValueTy> 76 struct NoRAUWValueMapConfig : public ValueMapConfig<ValueTy> { 77 typedef JITResolverState *ExtraData; 78 static void onRAUW(JITResolverState *, Value *Old, Value *New) { 79 llvm_unreachable("The JIT doesn't know how to handle a" 80 " RAUW on a value it has emitted."); 81 } 82 }; 83 84 struct CallSiteValueMapConfig : public NoRAUWValueMapConfig<Function*> { 85 typedef JITResolverState *ExtraData; 86 static void onDelete(JITResolverState *JRS, Function *F); 87 }; 88 89 class JITResolverState { 90 public: 91 typedef ValueMap<Function*, void*, NoRAUWValueMapConfig<Function*> > 92 FunctionToLazyStubMapTy; 93 typedef std::map<void*, AssertingVH<Function> > CallSiteToFunctionMapTy; 94 typedef ValueMap<Function *, SmallPtrSet<void*, 1>, 95 CallSiteValueMapConfig> FunctionToCallSitesMapTy; 96 typedef std::map<AssertingVH<GlobalValue>, void*> GlobalToIndirectSymMapTy; 97 private: 98 /// FunctionToLazyStubMap - Keep track of the lazy stub created for a 99 /// particular function so that we can reuse them if necessary. 100 FunctionToLazyStubMapTy FunctionToLazyStubMap; 101 102 /// CallSiteToFunctionMap - Keep track of the function that each lazy call 103 /// site corresponds to, and vice versa. 104 CallSiteToFunctionMapTy CallSiteToFunctionMap; 105 FunctionToCallSitesMapTy FunctionToCallSitesMap; 106 107 /// GlobalToIndirectSymMap - Keep track of the indirect symbol created for a 108 /// particular GlobalVariable so that we can reuse them if necessary. 109 GlobalToIndirectSymMapTy GlobalToIndirectSymMap; 110 111 #ifndef NDEBUG 112 /// Instance of the JIT this ResolverState serves. 113 JIT *TheJIT; 114 #endif 115 116 public: 117 JITResolverState(JIT *jit) : FunctionToLazyStubMap(this), 118 FunctionToCallSitesMap(this) { 119 #ifndef NDEBUG 120 TheJIT = jit; 121 #endif 122 } 123 124 FunctionToLazyStubMapTy& getFunctionToLazyStubMap( 125 const MutexGuard& locked) { 126 assert(locked.holds(TheJIT->lock)); 127 return FunctionToLazyStubMap; 128 } 129 130 GlobalToIndirectSymMapTy& getGlobalToIndirectSymMap(const MutexGuard& lck) { 131 assert(lck.holds(TheJIT->lock)); 132 return GlobalToIndirectSymMap; 133 } 134 135 std::pair<void *, Function *> LookupFunctionFromCallSite( 136 const MutexGuard &locked, void *CallSite) const { 137 assert(locked.holds(TheJIT->lock)); 138 139 // The address given to us for the stub may not be exactly right, it 140 // might be a little bit after the stub. As such, use upper_bound to 141 // find it. 142 CallSiteToFunctionMapTy::const_iterator I = 143 CallSiteToFunctionMap.upper_bound(CallSite); 144 assert(I != CallSiteToFunctionMap.begin() && 145 "This is not a known call site!"); 146 --I; 147 return *I; 148 } 149 150 void AddCallSite(const MutexGuard &locked, void *CallSite, Function *F) { 151 assert(locked.holds(TheJIT->lock)); 152 153 bool Inserted = CallSiteToFunctionMap.insert( 154 std::make_pair(CallSite, F)).second; 155 (void)Inserted; 156 assert(Inserted && "Pair was already in CallSiteToFunctionMap"); 157 FunctionToCallSitesMap[F].insert(CallSite); 158 } 159 160 void EraseAllCallSitesForPrelocked(Function *F); 161 162 // Erases _all_ call sites regardless of their function. This is used to 163 // unregister the stub addresses from the StubToResolverMap in 164 // ~JITResolver(). 165 void EraseAllCallSitesPrelocked(); 166 }; 167 168 /// JITResolver - Keep track of, and resolve, call sites for functions that 169 /// have not yet been compiled. 170 class JITResolver { 171 typedef JITResolverState::FunctionToLazyStubMapTy FunctionToLazyStubMapTy; 172 typedef JITResolverState::CallSiteToFunctionMapTy CallSiteToFunctionMapTy; 173 typedef JITResolverState::GlobalToIndirectSymMapTy GlobalToIndirectSymMapTy; 174 175 /// LazyResolverFn - The target lazy resolver function that we actually 176 /// rewrite instructions to use. 177 TargetJITInfo::LazyResolverFn LazyResolverFn; 178 179 JITResolverState state; 180 181 /// ExternalFnToStubMap - This is the equivalent of FunctionToLazyStubMap 182 /// for external functions. TODO: Of course, external functions don't need 183 /// a lazy stub. It's actually here to make it more likely that far calls 184 /// succeed, but no single stub can guarantee that. I'll remove this in a 185 /// subsequent checkin when I actually fix far calls. 186 std::map<void*, void*> ExternalFnToStubMap; 187 188 /// revGOTMap - map addresses to indexes in the GOT 189 std::map<void*, unsigned> revGOTMap; 190 unsigned nextGOTIndex; 191 192 JITEmitter &JE; 193 194 /// Instance of JIT corresponding to this Resolver. 195 JIT *TheJIT; 196 197 public: 198 explicit JITResolver(JIT &jit, JITEmitter &je) 199 : state(&jit), nextGOTIndex(0), JE(je), TheJIT(&jit) { 200 LazyResolverFn = jit.getJITInfo().getLazyResolverFunction(JITCompilerFn); 201 } 202 203 ~JITResolver(); 204 205 /// getLazyFunctionStubIfAvailable - This returns a pointer to a function's 206 /// lazy-compilation stub if it has already been created. 207 void *getLazyFunctionStubIfAvailable(Function *F); 208 209 /// getLazyFunctionStub - This returns a pointer to a function's 210 /// lazy-compilation stub, creating one on demand as needed. 211 void *getLazyFunctionStub(Function *F); 212 213 /// getExternalFunctionStub - Return a stub for the function at the 214 /// specified address, created lazily on demand. 215 void *getExternalFunctionStub(void *FnAddr); 216 217 /// getGlobalValueIndirectSym - Return an indirect symbol containing the 218 /// specified GV address. 219 void *getGlobalValueIndirectSym(GlobalValue *V, void *GVAddress); 220 221 /// getGOTIndexForAddress - Return a new or existing index in the GOT for 222 /// an address. This function only manages slots, it does not manage the 223 /// contents of the slots or the memory associated with the GOT. 224 unsigned getGOTIndexForAddr(void *addr); 225 226 /// JITCompilerFn - This function is called to resolve a stub to a compiled 227 /// address. If the LLVM Function corresponding to the stub has not yet 228 /// been compiled, this function compiles it first. 229 static void *JITCompilerFn(void *Stub); 230 }; 231 232 class StubToResolverMapTy { 233 /// Map a stub address to a specific instance of a JITResolver so that 234 /// lazily-compiled functions can find the right resolver to use. 235 /// 236 /// Guarded by Lock. 237 std::map<void*, JITResolver*> Map; 238 239 /// Guards Map from concurrent accesses. 240 mutable sys::Mutex Lock; 241 242 public: 243 /// Registers a Stub to be resolved by Resolver. 244 void RegisterStubResolver(void *Stub, JITResolver *Resolver) { 245 MutexGuard guard(Lock); 246 Map.insert(std::make_pair(Stub, Resolver)); 247 } 248 /// Unregisters the Stub when it's invalidated. 249 void UnregisterStubResolver(void *Stub) { 250 MutexGuard guard(Lock); 251 Map.erase(Stub); 252 } 253 /// Returns the JITResolver instance that owns the Stub. 254 JITResolver *getResolverFromStub(void *Stub) const { 255 MutexGuard guard(Lock); 256 // The address given to us for the stub may not be exactly right, it might 257 // be a little bit after the stub. As such, use upper_bound to find it. 258 // This is the same trick as in LookupFunctionFromCallSite from 259 // JITResolverState. 260 std::map<void*, JITResolver*>::const_iterator I = Map.upper_bound(Stub); 261 assert(I != Map.begin() && "This is not a known stub!"); 262 --I; 263 return I->second; 264 } 265 /// True if any stubs refer to the given resolver. Only used in an assert(). 266 /// O(N) 267 bool ResolverHasStubs(JITResolver* Resolver) const { 268 MutexGuard guard(Lock); 269 for (std::map<void*, JITResolver*>::const_iterator I = Map.begin(), 270 E = Map.end(); I != E; ++I) { 271 if (I->second == Resolver) 272 return true; 273 } 274 return false; 275 } 276 }; 277 /// This needs to be static so that a lazy call stub can access it with no 278 /// context except the address of the stub. 279 ManagedStatic<StubToResolverMapTy> StubToResolverMap; 280 281 /// JITEmitter - The JIT implementation of the MachineCodeEmitter, which is 282 /// used to output functions to memory for execution. 283 class JITEmitter : public JITCodeEmitter { 284 JITMemoryManager *MemMgr; 285 286 // When outputting a function stub in the context of some other function, we 287 // save BufferBegin/BufferEnd/CurBufferPtr here. 288 uint8_t *SavedBufferBegin, *SavedBufferEnd, *SavedCurBufferPtr; 289 290 // When reattempting to JIT a function after running out of space, we store 291 // the estimated size of the function we're trying to JIT here, so we can 292 // ask the memory manager for at least this much space. When we 293 // successfully emit the function, we reset this back to zero. 294 uintptr_t SizeEstimate; 295 296 /// Relocations - These are the relocations that the function needs, as 297 /// emitted. 298 std::vector<MachineRelocation> Relocations; 299 300 /// MBBLocations - This vector is a mapping from MBB ID's to their address. 301 /// It is filled in by the StartMachineBasicBlock callback and queried by 302 /// the getMachineBasicBlockAddress callback. 303 std::vector<uintptr_t> MBBLocations; 304 305 /// ConstantPool - The constant pool for the current function. 306 /// 307 MachineConstantPool *ConstantPool; 308 309 /// ConstantPoolBase - A pointer to the first entry in the constant pool. 310 /// 311 void *ConstantPoolBase; 312 313 /// ConstPoolAddresses - Addresses of individual constant pool entries. 314 /// 315 SmallVector<uintptr_t, 8> ConstPoolAddresses; 316 317 /// JumpTable - The jump tables for the current function. 318 /// 319 MachineJumpTableInfo *JumpTable; 320 321 /// JumpTableBase - A pointer to the first entry in the jump table. 322 /// 323 void *JumpTableBase; 324 325 /// Resolver - This contains info about the currently resolved functions. 326 JITResolver Resolver; 327 328 /// DE - The dwarf emitter for the jit. 329 OwningPtr<JITDwarfEmitter> DE; 330 331 /// LabelLocations - This vector is a mapping from Label ID's to their 332 /// address. 333 DenseMap<MCSymbol*, uintptr_t> LabelLocations; 334 335 /// MMI - Machine module info for exception informations 336 MachineModuleInfo* MMI; 337 338 // CurFn - The llvm function being emitted. Only valid during 339 // finishFunction(). 340 const Function *CurFn; 341 342 /// Information about emitted code, which is passed to the 343 /// JITEventListeners. This is reset in startFunction and used in 344 /// finishFunction. 345 JITEvent_EmittedFunctionDetails EmissionDetails; 346 347 struct EmittedCode { 348 void *FunctionBody; // Beginning of the function's allocation. 349 void *Code; // The address the function's code actually starts at. 350 void *ExceptionTable; 351 EmittedCode() : FunctionBody(0), Code(0), ExceptionTable(0) {} 352 }; 353 struct EmittedFunctionConfig : public ValueMapConfig<const Function*> { 354 typedef JITEmitter *ExtraData; 355 static void onDelete(JITEmitter *, const Function*); 356 static void onRAUW(JITEmitter *, const Function*, const Function*); 357 }; 358 ValueMap<const Function *, EmittedCode, 359 EmittedFunctionConfig> EmittedFunctions; 360 361 DebugLoc PrevDL; 362 363 /// Instance of the JIT 364 JIT *TheJIT; 365 366 bool JITExceptionHandling; 367 368 public: 369 JITEmitter(JIT &jit, JITMemoryManager *JMM, TargetMachine &TM) 370 : SizeEstimate(0), Resolver(jit, *this), MMI(0), CurFn(0), 371 EmittedFunctions(this), TheJIT(&jit), 372 JITExceptionHandling(TM.Options.JITExceptionHandling) { 373 MemMgr = JMM ? JMM : JITMemoryManager::CreateDefaultMemManager(); 374 if (jit.getJITInfo().needsGOT()) { 375 MemMgr->AllocateGOT(); 376 DEBUG(dbgs() << "JIT is managing a GOT\n"); 377 } 378 379 if (JITExceptionHandling) { 380 DE.reset(new JITDwarfEmitter(jit)); 381 } 382 } 383 ~JITEmitter() { 384 delete MemMgr; 385 } 386 387 /// classof - Methods for support type inquiry through isa, cast, and 388 /// dyn_cast: 389 /// 390 static inline bool classof(const MachineCodeEmitter*) { return true; } 391 392 JITResolver &getJITResolver() { return Resolver; } 393 394 virtual void startFunction(MachineFunction &F); 395 virtual bool finishFunction(MachineFunction &F); 396 397 void emitConstantPool(MachineConstantPool *MCP); 398 void initJumpTableInfo(MachineJumpTableInfo *MJTI); 399 void emitJumpTableInfo(MachineJumpTableInfo *MJTI); 400 401 void startGVStub(const GlobalValue* GV, 402 unsigned StubSize, unsigned Alignment = 1); 403 void startGVStub(void *Buffer, unsigned StubSize); 404 void finishGVStub(); 405 virtual void *allocIndirectGV(const GlobalValue *GV, 406 const uint8_t *Buffer, size_t Size, 407 unsigned Alignment); 408 409 /// allocateSpace - Reserves space in the current block if any, or 410 /// allocate a new one of the given size. 411 virtual void *allocateSpace(uintptr_t Size, unsigned Alignment); 412 413 /// allocateGlobal - Allocate memory for a global. Unlike allocateSpace, 414 /// this method does not allocate memory in the current output buffer, 415 /// because a global may live longer than the current function. 416 virtual void *allocateGlobal(uintptr_t Size, unsigned Alignment); 417 418 virtual void addRelocation(const MachineRelocation &MR) { 419 Relocations.push_back(MR); 420 } 421 422 virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) { 423 if (MBBLocations.size() <= (unsigned)MBB->getNumber()) 424 MBBLocations.resize((MBB->getNumber()+1)*2); 425 MBBLocations[MBB->getNumber()] = getCurrentPCValue(); 426 if (MBB->hasAddressTaken()) 427 TheJIT->addPointerToBasicBlock(MBB->getBasicBlock(), 428 (void*)getCurrentPCValue()); 429 DEBUG(dbgs() << "JIT: Emitting BB" << MBB->getNumber() << " at [" 430 << (void*) getCurrentPCValue() << "]\n"); 431 } 432 433 virtual uintptr_t getConstantPoolEntryAddress(unsigned Entry) const; 434 virtual uintptr_t getJumpTableEntryAddress(unsigned Entry) const; 435 436 virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const{ 437 assert(MBBLocations.size() > (unsigned)MBB->getNumber() && 438 MBBLocations[MBB->getNumber()] && "MBB not emitted!"); 439 return MBBLocations[MBB->getNumber()]; 440 } 441 442 /// retryWithMoreMemory - Log a retry and deallocate all memory for the 443 /// given function. Increase the minimum allocation size so that we get 444 /// more memory next time. 445 void retryWithMoreMemory(MachineFunction &F); 446 447 /// deallocateMemForFunction - Deallocate all memory for the specified 448 /// function body. 449 void deallocateMemForFunction(const Function *F); 450 451 virtual void processDebugLoc(DebugLoc DL, bool BeforePrintingInsn); 452 453 virtual void emitLabel(MCSymbol *Label) { 454 LabelLocations[Label] = getCurrentPCValue(); 455 } 456 457 virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() { 458 return &LabelLocations; 459 } 460 461 virtual uintptr_t getLabelAddress(MCSymbol *Label) const { 462 assert(LabelLocations.count(Label) && "Label not emitted!"); 463 return LabelLocations.find(Label)->second; 464 } 465 466 virtual void setModuleInfo(MachineModuleInfo* Info) { 467 MMI = Info; 468 if (DE.get()) DE->setModuleInfo(Info); 469 } 470 471 private: 472 void *getPointerToGlobal(GlobalValue *GV, void *Reference, 473 bool MayNeedFarStub); 474 void *getPointerToGVIndirectSym(GlobalValue *V, void *Reference); 475 }; 476 } 477 478 void CallSiteValueMapConfig::onDelete(JITResolverState *JRS, Function *F) { 479 JRS->EraseAllCallSitesForPrelocked(F); 480 } 481 482 void JITResolverState::EraseAllCallSitesForPrelocked(Function *F) { 483 FunctionToCallSitesMapTy::iterator F2C = FunctionToCallSitesMap.find(F); 484 if (F2C == FunctionToCallSitesMap.end()) 485 return; 486 StubToResolverMapTy &S2RMap = *StubToResolverMap; 487 for (SmallPtrSet<void*, 1>::const_iterator I = F2C->second.begin(), 488 E = F2C->second.end(); I != E; ++I) { 489 S2RMap.UnregisterStubResolver(*I); 490 bool Erased = CallSiteToFunctionMap.erase(*I); 491 (void)Erased; 492 assert(Erased && "Missing call site->function mapping"); 493 } 494 FunctionToCallSitesMap.erase(F2C); 495 } 496 497 void JITResolverState::EraseAllCallSitesPrelocked() { 498 StubToResolverMapTy &S2RMap = *StubToResolverMap; 499 for (CallSiteToFunctionMapTy::const_iterator 500 I = CallSiteToFunctionMap.begin(), 501 E = CallSiteToFunctionMap.end(); I != E; ++I) { 502 S2RMap.UnregisterStubResolver(I->first); 503 } 504 CallSiteToFunctionMap.clear(); 505 FunctionToCallSitesMap.clear(); 506 } 507 508 JITResolver::~JITResolver() { 509 // No need to lock because we're in the destructor, and state isn't shared. 510 state.EraseAllCallSitesPrelocked(); 511 assert(!StubToResolverMap->ResolverHasStubs(this) && 512 "Resolver destroyed with stubs still alive."); 513 } 514 515 /// getLazyFunctionStubIfAvailable - This returns a pointer to a function stub 516 /// if it has already been created. 517 void *JITResolver::getLazyFunctionStubIfAvailable(Function *F) { 518 MutexGuard locked(TheJIT->lock); 519 520 // If we already have a stub for this function, recycle it. 521 return state.getFunctionToLazyStubMap(locked).lookup(F); 522 } 523 524 /// getFunctionStub - This returns a pointer to a function stub, creating 525 /// one on demand as needed. 526 void *JITResolver::getLazyFunctionStub(Function *F) { 527 MutexGuard locked(TheJIT->lock); 528 529 // If we already have a lazy stub for this function, recycle it. 530 void *&Stub = state.getFunctionToLazyStubMap(locked)[F]; 531 if (Stub) return Stub; 532 533 // Call the lazy resolver function if we are JIT'ing lazily. Otherwise we 534 // must resolve the symbol now. 535 void *Actual = TheJIT->isCompilingLazily() 536 ? (void *)(intptr_t)LazyResolverFn : (void *)0; 537 538 // If this is an external declaration, attempt to resolve the address now 539 // to place in the stub. 540 if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) { 541 Actual = TheJIT->getPointerToFunction(F); 542 543 // If we resolved the symbol to a null address (eg. a weak external) 544 // don't emit a stub. Return a null pointer to the application. 545 if (!Actual) return 0; 546 } 547 548 TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 549 JE.startGVStub(F, SL.Size, SL.Alignment); 550 // Codegen a new stub, calling the lazy resolver or the actual address of the 551 // external function, if it was resolved. 552 Stub = TheJIT->getJITInfo().emitFunctionStub(F, Actual, JE); 553 JE.finishGVStub(); 554 555 if (Actual != (void*)(intptr_t)LazyResolverFn) { 556 // If we are getting the stub for an external function, we really want the 557 // address of the stub in the GlobalAddressMap for the JIT, not the address 558 // of the external function. 559 TheJIT->updateGlobalMapping(F, Stub); 560 } 561 562 DEBUG(dbgs() << "JIT: Lazy stub emitted at [" << Stub << "] for function '" 563 << F->getName() << "'\n"); 564 565 if (TheJIT->isCompilingLazily()) { 566 // Register this JITResolver as the one corresponding to this call site so 567 // JITCompilerFn will be able to find it. 568 StubToResolverMap->RegisterStubResolver(Stub, this); 569 570 // Finally, keep track of the stub-to-Function mapping so that the 571 // JITCompilerFn knows which function to compile! 572 state.AddCallSite(locked, Stub, F); 573 } else if (!Actual) { 574 // If we are JIT'ing non-lazily but need to call a function that does not 575 // exist yet, add it to the JIT's work list so that we can fill in the 576 // stub address later. 577 assert(!isNonGhostDeclaration(F) && !F->hasAvailableExternallyLinkage() && 578 "'Actual' should have been set above."); 579 TheJIT->addPendingFunction(F); 580 } 581 582 return Stub; 583 } 584 585 /// getGlobalValueIndirectSym - Return a lazy pointer containing the specified 586 /// GV address. 587 void *JITResolver::getGlobalValueIndirectSym(GlobalValue *GV, void *GVAddress) { 588 MutexGuard locked(TheJIT->lock); 589 590 // If we already have a stub for this global variable, recycle it. 591 void *&IndirectSym = state.getGlobalToIndirectSymMap(locked)[GV]; 592 if (IndirectSym) return IndirectSym; 593 594 // Otherwise, codegen a new indirect symbol. 595 IndirectSym = TheJIT->getJITInfo().emitGlobalValueIndirectSym(GV, GVAddress, 596 JE); 597 598 DEBUG(dbgs() << "JIT: Indirect symbol emitted at [" << IndirectSym 599 << "] for GV '" << GV->getName() << "'\n"); 600 601 return IndirectSym; 602 } 603 604 /// getExternalFunctionStub - Return a stub for the function at the 605 /// specified address, created lazily on demand. 606 void *JITResolver::getExternalFunctionStub(void *FnAddr) { 607 // If we already have a stub for this function, recycle it. 608 void *&Stub = ExternalFnToStubMap[FnAddr]; 609 if (Stub) return Stub; 610 611 TargetJITInfo::StubLayout SL = TheJIT->getJITInfo().getStubLayout(); 612 JE.startGVStub(0, SL.Size, SL.Alignment); 613 Stub = TheJIT->getJITInfo().emitFunctionStub(0, FnAddr, JE); 614 JE.finishGVStub(); 615 616 DEBUG(dbgs() << "JIT: Stub emitted at [" << Stub 617 << "] for external function at '" << FnAddr << "'\n"); 618 return Stub; 619 } 620 621 unsigned JITResolver::getGOTIndexForAddr(void* addr) { 622 unsigned idx = revGOTMap[addr]; 623 if (!idx) { 624 idx = ++nextGOTIndex; 625 revGOTMap[addr] = idx; 626 DEBUG(dbgs() << "JIT: Adding GOT entry " << idx << " for addr [" 627 << addr << "]\n"); 628 } 629 return idx; 630 } 631 632 /// JITCompilerFn - This function is called when a lazy compilation stub has 633 /// been entered. It looks up which function this stub corresponds to, compiles 634 /// it if necessary, then returns the resultant function pointer. 635 void *JITResolver::JITCompilerFn(void *Stub) { 636 JITResolver *JR = StubToResolverMap->getResolverFromStub(Stub); 637 assert(JR && "Unable to find the corresponding JITResolver to the call site"); 638 639 Function* F = 0; 640 void* ActualPtr = 0; 641 642 { 643 // Only lock for getting the Function. The call getPointerToFunction made 644 // in this function might trigger function materializing, which requires 645 // JIT lock to be unlocked. 646 MutexGuard locked(JR->TheJIT->lock); 647 648 // The address given to us for the stub may not be exactly right, it might 649 // be a little bit after the stub. As such, use upper_bound to find it. 650 std::pair<void*, Function*> I = 651 JR->state.LookupFunctionFromCallSite(locked, Stub); 652 F = I.second; 653 ActualPtr = I.first; 654 } 655 656 // If we have already code generated the function, just return the address. 657 void *Result = JR->TheJIT->getPointerToGlobalIfAvailable(F); 658 659 if (!Result) { 660 // Otherwise we don't have it, do lazy compilation now. 661 662 // If lazy compilation is disabled, emit a useful error message and abort. 663 if (!JR->TheJIT->isCompilingLazily()) { 664 report_fatal_error("LLVM JIT requested to do lazy compilation of" 665 " function '" 666 + F->getName() + "' when lazy compiles are disabled!"); 667 } 668 669 DEBUG(dbgs() << "JIT: Lazily resolving function '" << F->getName() 670 << "' In stub ptr = " << Stub << " actual ptr = " 671 << ActualPtr << "\n"); 672 (void)ActualPtr; 673 674 Result = JR->TheJIT->getPointerToFunction(F); 675 } 676 677 // Reacquire the lock to update the GOT map. 678 MutexGuard locked(JR->TheJIT->lock); 679 680 // We might like to remove the call site from the CallSiteToFunction map, but 681 // we can't do that! Multiple threads could be stuck, waiting to acquire the 682 // lock above. As soon as the 1st function finishes compiling the function, 683 // the next one will be released, and needs to be able to find the function it 684 // needs to call. 685 686 // FIXME: We could rewrite all references to this stub if we knew them. 687 688 // What we will do is set the compiled function address to map to the 689 // same GOT entry as the stub so that later clients may update the GOT 690 // if they see it still using the stub address. 691 // Note: this is done so the Resolver doesn't have to manage GOT memory 692 // Do this without allocating map space if the target isn't using a GOT 693 if(JR->revGOTMap.find(Stub) != JR->revGOTMap.end()) 694 JR->revGOTMap[Result] = JR->revGOTMap[Stub]; 695 696 return Result; 697 } 698 699 //===----------------------------------------------------------------------===// 700 // JITEmitter code. 701 // 702 void *JITEmitter::getPointerToGlobal(GlobalValue *V, void *Reference, 703 bool MayNeedFarStub) { 704 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V)) 705 return TheJIT->getOrEmitGlobalVariable(GV); 706 707 if (GlobalAlias *GA = dyn_cast<GlobalAlias>(V)) 708 return TheJIT->getPointerToGlobal(GA->resolveAliasedGlobal(false)); 709 710 // If we have already compiled the function, return a pointer to its body. 711 Function *F = cast<Function>(V); 712 713 void *FnStub = Resolver.getLazyFunctionStubIfAvailable(F); 714 if (FnStub) { 715 // Return the function stub if it's already created. We do this first so 716 // that we're returning the same address for the function as any previous 717 // call. TODO: Yes, this is wrong. The lazy stub isn't guaranteed to be 718 // close enough to call. 719 return FnStub; 720 } 721 722 // If we know the target can handle arbitrary-distance calls, try to 723 // return a direct pointer. 724 if (!MayNeedFarStub) { 725 // If we have code, go ahead and return that. 726 void *ResultPtr = TheJIT->getPointerToGlobalIfAvailable(F); 727 if (ResultPtr) return ResultPtr; 728 729 // If this is an external function pointer, we can force the JIT to 730 // 'compile' it, which really just adds it to the map. 731 if (isNonGhostDeclaration(F) || F->hasAvailableExternallyLinkage()) 732 return TheJIT->getPointerToFunction(F); 733 } 734 735 // Otherwise, we may need a to emit a stub, and, conservatively, we always do 736 // so. Note that it's possible to return null from getLazyFunctionStub in the 737 // case of a weak extern that fails to resolve. 738 return Resolver.getLazyFunctionStub(F); 739 } 740 741 void *JITEmitter::getPointerToGVIndirectSym(GlobalValue *V, void *Reference) { 742 // Make sure GV is emitted first, and create a stub containing the fully 743 // resolved address. 744 void *GVAddress = getPointerToGlobal(V, Reference, false); 745 void *StubAddr = Resolver.getGlobalValueIndirectSym(V, GVAddress); 746 return StubAddr; 747 } 748 749 void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) { 750 if (DL.isUnknown()) return; 751 if (!BeforePrintingInsn) return; 752 753 const LLVMContext &Context = EmissionDetails.MF->getFunction()->getContext(); 754 755 if (DL.getScope(Context) != 0 && PrevDL != DL) { 756 JITEvent_EmittedFunctionDetails::LineStart NextLine; 757 NextLine.Address = getCurrentPCValue(); 758 NextLine.Loc = DL; 759 EmissionDetails.LineStarts.push_back(NextLine); 760 } 761 762 PrevDL = DL; 763 } 764 765 static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP, 766 const TargetData *TD) { 767 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 768 if (Constants.empty()) return 0; 769 770 unsigned Size = 0; 771 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 772 MachineConstantPoolEntry CPE = Constants[i]; 773 unsigned AlignMask = CPE.getAlignment() - 1; 774 Size = (Size + AlignMask) & ~AlignMask; 775 Type *Ty = CPE.getType(); 776 Size += TD->getTypeAllocSize(Ty); 777 } 778 return Size; 779 } 780 781 void JITEmitter::startFunction(MachineFunction &F) { 782 DEBUG(dbgs() << "JIT: Starting CodeGen of Function " 783 << F.getName() << "\n"); 784 785 uintptr_t ActualSize = 0; 786 // Set the memory writable, if it's not already 787 MemMgr->setMemoryWritable(); 788 789 if (SizeEstimate > 0) { 790 // SizeEstimate will be non-zero on reallocation attempts. 791 ActualSize = SizeEstimate; 792 } 793 794 BufferBegin = CurBufferPtr = MemMgr->startFunctionBody(F.getFunction(), 795 ActualSize); 796 BufferEnd = BufferBegin+ActualSize; 797 EmittedFunctions[F.getFunction()].FunctionBody = BufferBegin; 798 799 // Ensure the constant pool/jump table info is at least 4-byte aligned. 800 emitAlignment(16); 801 802 emitConstantPool(F.getConstantPool()); 803 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 804 initJumpTableInfo(MJTI); 805 806 // About to start emitting the machine code for the function. 807 emitAlignment(std::max(F.getFunction()->getAlignment(), 8U)); 808 TheJIT->updateGlobalMapping(F.getFunction(), CurBufferPtr); 809 EmittedFunctions[F.getFunction()].Code = CurBufferPtr; 810 811 MBBLocations.clear(); 812 813 EmissionDetails.MF = &F; 814 EmissionDetails.LineStarts.clear(); 815 } 816 817 bool JITEmitter::finishFunction(MachineFunction &F) { 818 if (CurBufferPtr == BufferEnd) { 819 // We must call endFunctionBody before retrying, because 820 // deallocateMemForFunction requires it. 821 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 822 retryWithMoreMemory(F); 823 return true; 824 } 825 826 if (MachineJumpTableInfo *MJTI = F.getJumpTableInfo()) 827 emitJumpTableInfo(MJTI); 828 829 // FnStart is the start of the text, not the start of the constant pool and 830 // other per-function data. 831 uint8_t *FnStart = 832 (uint8_t *)TheJIT->getPointerToGlobalIfAvailable(F.getFunction()); 833 834 // FnEnd is the end of the function's machine code. 835 uint8_t *FnEnd = CurBufferPtr; 836 837 if (!Relocations.empty()) { 838 CurFn = F.getFunction(); 839 NumRelos += Relocations.size(); 840 841 // Resolve the relocations to concrete pointers. 842 for (unsigned i = 0, e = Relocations.size(); i != e; ++i) { 843 MachineRelocation &MR = Relocations[i]; 844 void *ResultPtr = 0; 845 if (!MR.letTargetResolve()) { 846 if (MR.isExternalSymbol()) { 847 ResultPtr = TheJIT->getPointerToNamedFunction(MR.getExternalSymbol(), 848 false); 849 DEBUG(dbgs() << "JIT: Map \'" << MR.getExternalSymbol() << "\' to [" 850 << ResultPtr << "]\n"); 851 852 // If the target REALLY wants a stub for this function, emit it now. 853 if (MR.mayNeedFarStub()) { 854 ResultPtr = Resolver.getExternalFunctionStub(ResultPtr); 855 } 856 } else if (MR.isGlobalValue()) { 857 ResultPtr = getPointerToGlobal(MR.getGlobalValue(), 858 BufferBegin+MR.getMachineCodeOffset(), 859 MR.mayNeedFarStub()); 860 } else if (MR.isIndirectSymbol()) { 861 ResultPtr = getPointerToGVIndirectSym( 862 MR.getGlobalValue(), BufferBegin+MR.getMachineCodeOffset()); 863 } else if (MR.isBasicBlock()) { 864 ResultPtr = (void*)getMachineBasicBlockAddress(MR.getBasicBlock()); 865 } else if (MR.isConstantPoolIndex()) { 866 ResultPtr = 867 (void*)getConstantPoolEntryAddress(MR.getConstantPoolIndex()); 868 } else { 869 assert(MR.isJumpTableIndex()); 870 ResultPtr=(void*)getJumpTableEntryAddress(MR.getJumpTableIndex()); 871 } 872 873 MR.setResultPointer(ResultPtr); 874 } 875 876 // if we are managing the GOT and the relocation wants an index, 877 // give it one 878 if (MR.isGOTRelative() && MemMgr->isManagingGOT()) { 879 unsigned idx = Resolver.getGOTIndexForAddr(ResultPtr); 880 MR.setGOTIndex(idx); 881 if (((void**)MemMgr->getGOTBase())[idx] != ResultPtr) { 882 DEBUG(dbgs() << "JIT: GOT was out of date for " << ResultPtr 883 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 884 << "\n"); 885 ((void**)MemMgr->getGOTBase())[idx] = ResultPtr; 886 } 887 } 888 } 889 890 CurFn = 0; 891 TheJIT->getJITInfo().relocate(BufferBegin, &Relocations[0], 892 Relocations.size(), MemMgr->getGOTBase()); 893 } 894 895 // Update the GOT entry for F to point to the new code. 896 if (MemMgr->isManagingGOT()) { 897 unsigned idx = Resolver.getGOTIndexForAddr((void*)BufferBegin); 898 if (((void**)MemMgr->getGOTBase())[idx] != (void*)BufferBegin) { 899 DEBUG(dbgs() << "JIT: GOT was out of date for " << (void*)BufferBegin 900 << " pointing at " << ((void**)MemMgr->getGOTBase())[idx] 901 << "\n"); 902 ((void**)MemMgr->getGOTBase())[idx] = (void*)BufferBegin; 903 } 904 } 905 906 // CurBufferPtr may have moved beyond FnEnd, due to memory allocation for 907 // global variables that were referenced in the relocations. 908 MemMgr->endFunctionBody(F.getFunction(), BufferBegin, CurBufferPtr); 909 910 if (CurBufferPtr == BufferEnd) { 911 retryWithMoreMemory(F); 912 return true; 913 } else { 914 // Now that we've succeeded in emitting the function, reset the 915 // SizeEstimate back down to zero. 916 SizeEstimate = 0; 917 } 918 919 BufferBegin = CurBufferPtr = 0; 920 NumBytes += FnEnd-FnStart; 921 922 // Invalidate the icache if necessary. 923 sys::Memory::InvalidateInstructionCache(FnStart, FnEnd-FnStart); 924 925 TheJIT->NotifyFunctionEmitted(*F.getFunction(), FnStart, FnEnd-FnStart, 926 EmissionDetails); 927 928 // Reset the previous debug location. 929 PrevDL = DebugLoc(); 930 931 DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart 932 << "] Function: " << F.getName() 933 << ": " << (FnEnd-FnStart) << " bytes of text, " 934 << Relocations.size() << " relocations\n"); 935 936 Relocations.clear(); 937 ConstPoolAddresses.clear(); 938 939 // Mark code region readable and executable if it's not so already. 940 MemMgr->setMemoryExecutable(); 941 942 DEBUG({ 943 if (sys::hasDisassembler()) { 944 dbgs() << "JIT: Disassembled code:\n"; 945 dbgs() << sys::disassembleBuffer(FnStart, FnEnd-FnStart, 946 (uintptr_t)FnStart); 947 } else { 948 dbgs() << "JIT: Binary code:\n"; 949 uint8_t* q = FnStart; 950 for (int i = 0; q < FnEnd; q += 4, ++i) { 951 if (i == 4) 952 i = 0; 953 if (i == 0) 954 dbgs() << "JIT: " << (long)(q - FnStart) << ": "; 955 bool Done = false; 956 for (int j = 3; j >= 0; --j) { 957 if (q + j >= FnEnd) 958 Done = true; 959 else 960 dbgs() << (unsigned short)q[j]; 961 } 962 if (Done) 963 break; 964 dbgs() << ' '; 965 if (i == 3) 966 dbgs() << '\n'; 967 } 968 dbgs()<< '\n'; 969 } 970 }); 971 972 if (JITExceptionHandling) { 973 uintptr_t ActualSize = 0; 974 SavedBufferBegin = BufferBegin; 975 SavedBufferEnd = BufferEnd; 976 SavedCurBufferPtr = CurBufferPtr; 977 978 BufferBegin = CurBufferPtr = MemMgr->startExceptionTable(F.getFunction(), 979 ActualSize); 980 BufferEnd = BufferBegin+ActualSize; 981 EmittedFunctions[F.getFunction()].ExceptionTable = BufferBegin; 982 uint8_t *EhStart; 983 uint8_t *FrameRegister = DE->EmitDwarfTable(F, *this, FnStart, FnEnd, 984 EhStart); 985 MemMgr->endExceptionTable(F.getFunction(), BufferBegin, CurBufferPtr, 986 FrameRegister); 987 BufferBegin = SavedBufferBegin; 988 BufferEnd = SavedBufferEnd; 989 CurBufferPtr = SavedCurBufferPtr; 990 991 if (JITExceptionHandling) { 992 TheJIT->RegisterTable(F.getFunction(), FrameRegister); 993 } 994 } 995 996 if (MMI) 997 MMI->EndFunction(); 998 999 return false; 1000 } 1001 1002 void JITEmitter::retryWithMoreMemory(MachineFunction &F) { 1003 DEBUG(dbgs() << "JIT: Ran out of space for native code. Reattempting.\n"); 1004 Relocations.clear(); // Clear the old relocations or we'll reapply them. 1005 ConstPoolAddresses.clear(); 1006 ++NumRetries; 1007 deallocateMemForFunction(F.getFunction()); 1008 // Try again with at least twice as much free space. 1009 SizeEstimate = (uintptr_t)(2 * (BufferEnd - BufferBegin)); 1010 1011 for (MachineFunction::iterator MBB = F.begin(), E = F.end(); MBB != E; ++MBB){ 1012 if (MBB->hasAddressTaken()) 1013 TheJIT->clearPointerToBasicBlock(MBB->getBasicBlock()); 1014 } 1015 } 1016 1017 /// deallocateMemForFunction - Deallocate all memory for the specified 1018 /// function body. Also drop any references the function has to stubs. 1019 /// May be called while the Function is being destroyed inside ~Value(). 1020 void JITEmitter::deallocateMemForFunction(const Function *F) { 1021 ValueMap<const Function *, EmittedCode, EmittedFunctionConfig>::iterator 1022 Emitted = EmittedFunctions.find(F); 1023 if (Emitted != EmittedFunctions.end()) { 1024 MemMgr->deallocateFunctionBody(Emitted->second.FunctionBody); 1025 MemMgr->deallocateExceptionTable(Emitted->second.ExceptionTable); 1026 TheJIT->NotifyFreeingMachineCode(Emitted->second.Code); 1027 1028 EmittedFunctions.erase(Emitted); 1029 } 1030 1031 if (JITExceptionHandling) { 1032 TheJIT->DeregisterTable(F); 1033 } 1034 } 1035 1036 1037 void *JITEmitter::allocateSpace(uintptr_t Size, unsigned Alignment) { 1038 if (BufferBegin) 1039 return JITCodeEmitter::allocateSpace(Size, Alignment); 1040 1041 // create a new memory block if there is no active one. 1042 // care must be taken so that BufferBegin is invalidated when a 1043 // block is trimmed 1044 BufferBegin = CurBufferPtr = MemMgr->allocateSpace(Size, Alignment); 1045 BufferEnd = BufferBegin+Size; 1046 return CurBufferPtr; 1047 } 1048 1049 void *JITEmitter::allocateGlobal(uintptr_t Size, unsigned Alignment) { 1050 // Delegate this call through the memory manager. 1051 return MemMgr->allocateGlobal(Size, Alignment); 1052 } 1053 1054 void JITEmitter::emitConstantPool(MachineConstantPool *MCP) { 1055 if (TheJIT->getJITInfo().hasCustomConstantPool()) 1056 return; 1057 1058 const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants(); 1059 if (Constants.empty()) return; 1060 1061 unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData()); 1062 unsigned Align = MCP->getConstantPoolAlignment(); 1063 ConstantPoolBase = allocateSpace(Size, Align); 1064 ConstantPool = MCP; 1065 1066 if (ConstantPoolBase == 0) return; // Buffer overflow. 1067 1068 DEBUG(dbgs() << "JIT: Emitted constant pool at [" << ConstantPoolBase 1069 << "] (size: " << Size << ", alignment: " << Align << ")\n"); 1070 1071 // Initialize the memory for all of the constant pool entries. 1072 unsigned Offset = 0; 1073 for (unsigned i = 0, e = Constants.size(); i != e; ++i) { 1074 MachineConstantPoolEntry CPE = Constants[i]; 1075 unsigned AlignMask = CPE.getAlignment() - 1; 1076 Offset = (Offset + AlignMask) & ~AlignMask; 1077 1078 uintptr_t CAddr = (uintptr_t)ConstantPoolBase + Offset; 1079 ConstPoolAddresses.push_back(CAddr); 1080 if (CPE.isMachineConstantPoolEntry()) { 1081 // FIXME: add support to lower machine constant pool values into bytes! 1082 report_fatal_error("Initialize memory with machine specific constant pool" 1083 "entry has not been implemented!"); 1084 } 1085 TheJIT->InitializeMemory(CPE.Val.ConstVal, (void*)CAddr); 1086 DEBUG(dbgs() << "JIT: CP" << i << " at [0x"; 1087 dbgs().write_hex(CAddr) << "]\n"); 1088 1089 Type *Ty = CPE.Val.ConstVal->getType(); 1090 Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty); 1091 } 1092 } 1093 1094 void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) { 1095 if (TheJIT->getJITInfo().hasCustomJumpTables()) 1096 return; 1097 if (MJTI->getEntryKind() == MachineJumpTableInfo::EK_Inline) 1098 return; 1099 1100 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1101 if (JT.empty()) return; 1102 1103 unsigned NumEntries = 0; 1104 for (unsigned i = 0, e = JT.size(); i != e; ++i) 1105 NumEntries += JT[i].MBBs.size(); 1106 1107 unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getTargetData()); 1108 1109 // Just allocate space for all the jump tables now. We will fix up the actual 1110 // MBB entries in the tables after we emit the code for each block, since then 1111 // we will know the final locations of the MBBs in memory. 1112 JumpTable = MJTI; 1113 JumpTableBase = allocateSpace(NumEntries * EntrySize, 1114 MJTI->getEntryAlignment(*TheJIT->getTargetData())); 1115 } 1116 1117 void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) { 1118 if (TheJIT->getJITInfo().hasCustomJumpTables()) 1119 return; 1120 1121 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables(); 1122 if (JT.empty() || JumpTableBase == 0) return; 1123 1124 1125 switch (MJTI->getEntryKind()) { 1126 case MachineJumpTableInfo::EK_Inline: 1127 return; 1128 case MachineJumpTableInfo::EK_BlockAddress: { 1129 // EK_BlockAddress - Each entry is a plain address of block, e.g.: 1130 // .word LBB123 1131 assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == sizeof(void*) && 1132 "Cross JIT'ing?"); 1133 1134 // For each jump table, map each target in the jump table to the address of 1135 // an emitted MachineBasicBlock. 1136 intptr_t *SlotPtr = (intptr_t*)JumpTableBase; 1137 1138 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1139 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1140 // Store the address of the basic block for this jump table slot in the 1141 // memory we allocated for the jump table in 'initJumpTableInfo' 1142 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) 1143 *SlotPtr++ = getMachineBasicBlockAddress(MBBs[mi]); 1144 } 1145 break; 1146 } 1147 1148 case MachineJumpTableInfo::EK_Custom32: 1149 case MachineJumpTableInfo::EK_GPRel32BlockAddress: 1150 case MachineJumpTableInfo::EK_LabelDifference32: { 1151 assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == 4&&"Cross JIT'ing?"); 1152 // For each jump table, place the offset from the beginning of the table 1153 // to the target address. 1154 int *SlotPtr = (int*)JumpTableBase; 1155 1156 for (unsigned i = 0, e = JT.size(); i != e; ++i) { 1157 const std::vector<MachineBasicBlock*> &MBBs = JT[i].MBBs; 1158 // Store the offset of the basic block for this jump table slot in the 1159 // memory we allocated for the jump table in 'initJumpTableInfo' 1160 uintptr_t Base = (uintptr_t)SlotPtr; 1161 for (unsigned mi = 0, me = MBBs.size(); mi != me; ++mi) { 1162 uintptr_t MBBAddr = getMachineBasicBlockAddress(MBBs[mi]); 1163 /// FIXME: USe EntryKind instead of magic "getPICJumpTableEntry" hook. 1164 *SlotPtr++ = TheJIT->getJITInfo().getPICJumpTableEntry(MBBAddr, Base); 1165 } 1166 } 1167 break; 1168 } 1169 case MachineJumpTableInfo::EK_GPRel64BlockAddress: 1170 llvm_unreachable( 1171 "JT Info emission not implemented for GPRel64BlockAddress yet."); 1172 } 1173 } 1174 1175 void JITEmitter::startGVStub(const GlobalValue* GV, 1176 unsigned StubSize, unsigned Alignment) { 1177 SavedBufferBegin = BufferBegin; 1178 SavedBufferEnd = BufferEnd; 1179 SavedCurBufferPtr = CurBufferPtr; 1180 1181 BufferBegin = CurBufferPtr = MemMgr->allocateStub(GV, StubSize, Alignment); 1182 BufferEnd = BufferBegin+StubSize+1; 1183 } 1184 1185 void JITEmitter::startGVStub(void *Buffer, unsigned StubSize) { 1186 SavedBufferBegin = BufferBegin; 1187 SavedBufferEnd = BufferEnd; 1188 SavedCurBufferPtr = CurBufferPtr; 1189 1190 BufferBegin = CurBufferPtr = (uint8_t *)Buffer; 1191 BufferEnd = BufferBegin+StubSize+1; 1192 } 1193 1194 void JITEmitter::finishGVStub() { 1195 assert(CurBufferPtr != BufferEnd && "Stub overflowed allocated space."); 1196 NumBytes += getCurrentPCOffset(); 1197 BufferBegin = SavedBufferBegin; 1198 BufferEnd = SavedBufferEnd; 1199 CurBufferPtr = SavedCurBufferPtr; 1200 } 1201 1202 void *JITEmitter::allocIndirectGV(const GlobalValue *GV, 1203 const uint8_t *Buffer, size_t Size, 1204 unsigned Alignment) { 1205 uint8_t *IndGV = MemMgr->allocateStub(GV, Size, Alignment); 1206 memcpy(IndGV, Buffer, Size); 1207 return IndGV; 1208 } 1209 1210 // getConstantPoolEntryAddress - Return the address of the 'ConstantNum' entry 1211 // in the constant pool that was last emitted with the 'emitConstantPool' 1212 // method. 1213 // 1214 uintptr_t JITEmitter::getConstantPoolEntryAddress(unsigned ConstantNum) const { 1215 assert(ConstantNum < ConstantPool->getConstants().size() && 1216 "Invalid ConstantPoolIndex!"); 1217 return ConstPoolAddresses[ConstantNum]; 1218 } 1219 1220 // getJumpTableEntryAddress - Return the address of the JumpTable with index 1221 // 'Index' in the jumpp table that was last initialized with 'initJumpTableInfo' 1222 // 1223 uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const { 1224 const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables(); 1225 assert(Index < JT.size() && "Invalid jump table index!"); 1226 1227 unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getTargetData()); 1228 1229 unsigned Offset = 0; 1230 for (unsigned i = 0; i < Index; ++i) 1231 Offset += JT[i].MBBs.size(); 1232 1233 Offset *= EntrySize; 1234 1235 return (uintptr_t)((char *)JumpTableBase + Offset); 1236 } 1237 1238 void JITEmitter::EmittedFunctionConfig::onDelete( 1239 JITEmitter *Emitter, const Function *F) { 1240 Emitter->deallocateMemForFunction(F); 1241 } 1242 void JITEmitter::EmittedFunctionConfig::onRAUW( 1243 JITEmitter *, const Function*, const Function*) { 1244 llvm_unreachable("The JIT doesn't know how to handle a" 1245 " RAUW on a value it has emitted."); 1246 } 1247 1248 1249 //===----------------------------------------------------------------------===// 1250 // Public interface to this file 1251 //===----------------------------------------------------------------------===// 1252 1253 JITCodeEmitter *JIT::createEmitter(JIT &jit, JITMemoryManager *JMM, 1254 TargetMachine &tm) { 1255 return new JITEmitter(jit, JMM, tm); 1256 } 1257 1258 // getPointerToFunctionOrStub - If the specified function has been 1259 // code-gen'd, return a pointer to the function. If not, compile it, or use 1260 // a stub to implement lazy compilation if available. 1261 // 1262 void *JIT::getPointerToFunctionOrStub(Function *F) { 1263 // If we have already code generated the function, just return the address. 1264 if (void *Addr = getPointerToGlobalIfAvailable(F)) 1265 return Addr; 1266 1267 // Get a stub if the target supports it. 1268 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1269 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1270 return JE->getJITResolver().getLazyFunctionStub(F); 1271 } 1272 1273 void JIT::updateFunctionStub(Function *F) { 1274 // Get the empty stub we generated earlier. 1275 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1276 JITEmitter *JE = cast<JITEmitter>(getCodeEmitter()); 1277 void *Stub = JE->getJITResolver().getLazyFunctionStub(F); 1278 void *Addr = getPointerToGlobalIfAvailable(F); 1279 assert(Addr != Stub && "Function must have non-stub address to be updated."); 1280 1281 // Tell the target jit info to rewrite the stub at the specified address, 1282 // rather than creating a new one. 1283 TargetJITInfo::StubLayout layout = getJITInfo().getStubLayout(); 1284 JE->startGVStub(Stub, layout.Size); 1285 getJITInfo().emitFunctionStub(F, Addr, *getCodeEmitter()); 1286 JE->finishGVStub(); 1287 } 1288 1289 /// freeMachineCodeForFunction - release machine code memory for given Function. 1290 /// 1291 void JIT::freeMachineCodeForFunction(Function *F) { 1292 // Delete translation for this from the ExecutionEngine, so it will get 1293 // retranslated next time it is used. 1294 updateGlobalMapping(F, 0); 1295 1296 // Free the actual memory for the function body and related stuff. 1297 assert(isa<JITEmitter>(JCE) && "Unexpected MCE?"); 1298 cast<JITEmitter>(JCE)->deallocateMemForFunction(F); 1299 } 1300