Home | History | Annotate | Download | only in gallivm
      1 /**************************************************************************
      2  *
      3  * Copyright 2009 VMware, Inc.
      4  * All Rights Reserved.
      5  *
      6  * Permission is hereby granted, free of charge, to any person obtaining a
      7  * copy of this software and associated documentation files (the
      8  * "Software"), to deal in the Software without restriction, including
      9  * without limitation the rights to use, copy, modify, merge, publish,
     10  * distribute, sub license, and/or sell copies of the Software, and to
     11  * permit persons to whom the Software is furnished to do so, subject to
     12  * the following conditions:
     13  *
     14  * The above copyright notice and this permission notice (including the
     15  * next paragraph) shall be included in all copies or substantial portions
     16  * of the Software.
     17  *
     18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
     19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
     21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
     22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
     23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
     24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
     25  *
     26  **************************************************************************/
     27 
     28 
     29 #include "pipe/p_config.h"
     30 #include "pipe/p_compiler.h"
     31 #include "util/u_cpu_detect.h"
     32 #include "util/u_debug.h"
     33 #include "util/u_memory.h"
     34 #include "util/simple_list.h"
     35 #include "os/os_time.h"
     36 #include "lp_bld.h"
     37 #include "lp_bld_debug.h"
     38 #include "lp_bld_misc.h"
     39 #include "lp_bld_init.h"
     40 
     41 #include <llvm-c/Analysis.h>
     42 #include <llvm-c/Transforms/Scalar.h>
     43 #include <llvm-c/BitWriter.h>
     44 
     45 
     46 /* Only MCJIT is available as of LLVM SVN r216982 */
     47 #if HAVE_LLVM >= 0x0306
     48 #  define USE_MCJIT 1
     49 #elif defined(PIPE_ARCH_PPC_64) || defined(PIPE_ARCH_S390) || defined(PIPE_ARCH_ARM) || defined(PIPE_ARCH_AARCH64)
     50 #  define USE_MCJIT 1
     51 #endif
     52 
     53 #if defined(USE_MCJIT)
     54 static const bool use_mcjit = USE_MCJIT;
     55 #else
     56 static bool use_mcjit = FALSE;
     57 #endif
     58 
     59 
     60 #ifdef DEBUG
     61 unsigned gallivm_debug = 0;
     62 
     63 static const struct debug_named_value lp_bld_debug_flags[] = {
     64    { "tgsi",   GALLIVM_DEBUG_TGSI, NULL },
     65    { "ir",     GALLIVM_DEBUG_IR, NULL },
     66    { "asm",    GALLIVM_DEBUG_ASM, NULL },
     67    { "nopt",   GALLIVM_DEBUG_NO_OPT, NULL },
     68    { "perf",   GALLIVM_DEBUG_PERF, NULL },
     69    { "no_brilinear", GALLIVM_DEBUG_NO_BRILINEAR, NULL },
     70    { "no_rho_approx", GALLIVM_DEBUG_NO_RHO_APPROX, NULL },
     71    { "no_quad_lod", GALLIVM_DEBUG_NO_QUAD_LOD, NULL },
     72    { "gc",     GALLIVM_DEBUG_GC, NULL },
     73    { "dumpbc", GALLIVM_DEBUG_DUMP_BC, NULL },
     74    DEBUG_NAMED_VALUE_END
     75 };
     76 
     77 DEBUG_GET_ONCE_FLAGS_OPTION(gallivm_debug, "GALLIVM_DEBUG", lp_bld_debug_flags, 0)
     78 #endif
     79 
     80 
     81 static boolean gallivm_initialized = FALSE;
     82 
     83 unsigned lp_native_vector_width;
     84 
     85 
     86 /*
     87  * Optimization values are:
     88  * - 0: None (-O0)
     89  * - 1: Less (-O1)
     90  * - 2: Default (-O2, -Os)
     91  * - 3: Aggressive (-O3)
     92  *
     93  * See also CodeGenOpt::Level in llvm/Target/TargetMachine.h
     94  */
     95 enum LLVM_CodeGenOpt_Level {
     96    None,        // -O0
     97    Less,        // -O1
     98    Default,     // -O2, -Os
     99    Aggressive   // -O3
    100 };
    101 
    102 
    103 /**
    104  * Create the LLVM (optimization) pass manager and install
    105  * relevant optimization passes.
    106  * \return  TRUE for success, FALSE for failure
    107  */
    108 static boolean
    109 create_pass_manager(struct gallivm_state *gallivm)
    110 {
    111    assert(!gallivm->passmgr);
    112    assert(gallivm->target);
    113 
    114    gallivm->passmgr = LLVMCreateFunctionPassManagerForModule(gallivm->module);
    115    if (!gallivm->passmgr)
    116       return FALSE;
    117    /*
    118     * TODO: some per module pass manager with IPO passes might be helpful -
    119     * the generated texture functions may benefit from inlining if they are
    120     * simple, or constant propagation into them, etc.
    121     */
    122 
    123 #if HAVE_LLVM < 0x0309
    124    // Old versions of LLVM get the DataLayout from the pass manager.
    125    LLVMAddTargetData(gallivm->target, gallivm->passmgr);
    126 #endif
    127 
    128    /* Setting the module's DataLayout to an empty string will cause the
    129     * ExecutionEngine to copy to the DataLayout string from its target
    130     * machine to the module.  As of LLVM 3.8 the module and the execution
    131     * engine are required to have the same DataLayout.
    132     *
    133     * TODO: This is just a temporary work-around.  The correct solution is
    134     * for gallivm_init_state() to create a TargetMachine and pull the
    135     * DataLayout from there.  Currently, the TargetMachine used by llvmpipe
    136     * is being implicitly created by the EngineBuilder in
    137     * lp_build_create_jit_compiler_for_module()
    138     */
    139 
    140 #if HAVE_LLVM < 0x0308
    141    {
    142       char *td_str;
    143       // New ones from the Module.
    144       td_str = LLVMCopyStringRepOfTargetData(gallivm->target);
    145       LLVMSetDataLayout(gallivm->module, td_str);
    146       free(td_str);
    147    }
    148 #else
    149    LLVMSetDataLayout(gallivm->module, "");
    150 #endif
    151 
    152    if ((gallivm_debug & GALLIVM_DEBUG_NO_OPT) == 0) {
    153       /* These are the passes currently listed in llvm-c/Transforms/Scalar.h,
    154        * but there are more on SVN.
    155        * TODO: Add more passes.
    156        */
    157       LLVMAddScalarReplAggregatesPass(gallivm->passmgr);
    158       LLVMAddLICMPass(gallivm->passmgr);
    159       LLVMAddCFGSimplificationPass(gallivm->passmgr);
    160       LLVMAddReassociatePass(gallivm->passmgr);
    161       LLVMAddPromoteMemoryToRegisterPass(gallivm->passmgr);
    162       LLVMAddConstantPropagationPass(gallivm->passmgr);
    163       LLVMAddInstructionCombiningPass(gallivm->passmgr);
    164       LLVMAddGVNPass(gallivm->passmgr);
    165    }
    166    else {
    167       /* We need at least this pass to prevent the backends to fail in
    168        * unexpected ways.
    169        */
    170       LLVMAddPromoteMemoryToRegisterPass(gallivm->passmgr);
    171    }
    172 
    173    return TRUE;
    174 }
    175 
    176 
    177 /**
    178  * Free gallivm object's LLVM allocations, but not any generated code
    179  * nor the gallivm object itself.
    180  */
    181 void
    182 gallivm_free_ir(struct gallivm_state *gallivm)
    183 {
    184    if (gallivm->passmgr) {
    185       LLVMDisposePassManager(gallivm->passmgr);
    186    }
    187 
    188    if (gallivm->engine) {
    189       /* This will already destroy any associated module */
    190       LLVMDisposeExecutionEngine(gallivm->engine);
    191    } else if (gallivm->module) {
    192       LLVMDisposeModule(gallivm->module);
    193    }
    194 
    195    FREE(gallivm->module_name);
    196 
    197    if (!use_mcjit) {
    198       /* Don't free the TargetData, it's owned by the exec engine */
    199    } else {
    200       if (gallivm->target) {
    201          LLVMDisposeTargetData(gallivm->target);
    202       }
    203    }
    204 
    205    if (gallivm->builder)
    206       LLVMDisposeBuilder(gallivm->builder);
    207 
    208    /* The LLVMContext should be owned by the parent of gallivm. */
    209 
    210    gallivm->engine = NULL;
    211    gallivm->target = NULL;
    212    gallivm->module = NULL;
    213    gallivm->module_name = NULL;
    214    gallivm->passmgr = NULL;
    215    gallivm->context = NULL;
    216    gallivm->builder = NULL;
    217 }
    218 
    219 
    220 /**
    221  * Free LLVM-generated code.  Should be done AFTER gallivm_free_ir().
    222  */
    223 static void
    224 gallivm_free_code(struct gallivm_state *gallivm)
    225 {
    226    assert(!gallivm->module);
    227    assert(!gallivm->engine);
    228    lp_free_generated_code(gallivm->code);
    229    gallivm->code = NULL;
    230    lp_free_memory_manager(gallivm->memorymgr);
    231    gallivm->memorymgr = NULL;
    232 }
    233 
    234 
    235 static boolean
    236 init_gallivm_engine(struct gallivm_state *gallivm)
    237 {
    238    if (1) {
    239       enum LLVM_CodeGenOpt_Level optlevel;
    240       char *error = NULL;
    241       int ret;
    242 
    243       if (gallivm_debug & GALLIVM_DEBUG_NO_OPT) {
    244          optlevel = None;
    245       }
    246       else {
    247          optlevel = Default;
    248       }
    249 
    250       ret = lp_build_create_jit_compiler_for_module(&gallivm->engine,
    251                                                     &gallivm->code,
    252                                                     gallivm->module,
    253                                                     gallivm->memorymgr,
    254                                                     (unsigned) optlevel,
    255                                                     use_mcjit,
    256                                                     &error);
    257       if (ret) {
    258          _debug_printf("%s\n", error);
    259          LLVMDisposeMessage(error);
    260          goto fail;
    261       }
    262    }
    263 
    264    if (!use_mcjit) {
    265       gallivm->target = LLVMGetExecutionEngineTargetData(gallivm->engine);
    266       if (!gallivm->target)
    267          goto fail;
    268    } else {
    269       if (0) {
    270           /*
    271            * Dump the data layout strings.
    272            */
    273 
    274           LLVMTargetDataRef target = LLVMGetExecutionEngineTargetData(gallivm->engine);
    275           char *data_layout;
    276           char *engine_data_layout;
    277 
    278           data_layout = LLVMCopyStringRepOfTargetData(gallivm->target);
    279           engine_data_layout = LLVMCopyStringRepOfTargetData(target);
    280 
    281           if (1) {
    282              debug_printf("module target data = %s\n", data_layout);
    283              debug_printf("engine target data = %s\n", engine_data_layout);
    284           }
    285 
    286           free(data_layout);
    287           free(engine_data_layout);
    288       }
    289    }
    290 
    291    return TRUE;
    292 
    293 fail:
    294    return FALSE;
    295 }
    296 
    297 
    298 /**
    299  * Allocate gallivm LLVM objects.
    300  * \return  TRUE for success, FALSE for failure
    301  */
    302 static boolean
    303 init_gallivm_state(struct gallivm_state *gallivm, const char *name,
    304                    LLVMContextRef context)
    305 {
    306    assert(!gallivm->context);
    307    assert(!gallivm->module);
    308 
    309    if (!lp_build_init())
    310       return FALSE;
    311 
    312    gallivm->context = context;
    313 
    314    if (!gallivm->context)
    315       goto fail;
    316 
    317    gallivm->module_name = NULL;
    318    if (name) {
    319       size_t size = strlen(name) + 1;
    320       gallivm->module_name = MALLOC(size);
    321       if (gallivm->module_name) {
    322          memcpy(gallivm->module_name, name, size);
    323       }
    324    }
    325 
    326    gallivm->module = LLVMModuleCreateWithNameInContext(name,
    327                                                        gallivm->context);
    328    if (!gallivm->module)
    329       goto fail;
    330 
    331    gallivm->builder = LLVMCreateBuilderInContext(gallivm->context);
    332    if (!gallivm->builder)
    333       goto fail;
    334 
    335    gallivm->memorymgr = lp_get_default_memory_manager();
    336    if (!gallivm->memorymgr)
    337       goto fail;
    338 
    339    /* FIXME: MC-JIT only allows compiling one module at a time, and it must be
    340     * complete when MC-JIT is created. So defer the MC-JIT engine creation for
    341     * now.
    342     */
    343    if (!use_mcjit) {
    344       if (!init_gallivm_engine(gallivm)) {
    345          goto fail;
    346       }
    347    } else {
    348       /*
    349        * MC-JIT engine compiles the module immediately on creation, so we can't
    350        * obtain the target data from it.  Instead we create a target data layout
    351        * from a string.
    352        *
    353        * The produced layout strings are not precisely the same, but should make
    354        * no difference for the kind of optimization passes we run.
    355        *
    356        * For reference this is the layout string on x64:
    357        *
    358        *   e-p:64:64:64-S128-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f16:16:16-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-f128:128:128-n8:16:32:64
    359        *
    360        * See also:
    361        * - http://llvm.org/docs/LangRef.html#datalayout
    362        */
    363 
    364       {
    365          const unsigned pointer_size = 8 * sizeof(void *);
    366          char layout[512];
    367          util_snprintf(layout, sizeof layout, "%c-p:%u:%u:%u-i64:64:64-a0:0:%u-s0:%u:%u",
    368 #ifdef PIPE_ARCH_LITTLE_ENDIAN
    369                        'e', // little endian
    370 #else
    371                        'E', // big endian
    372 #endif
    373                        pointer_size, pointer_size, pointer_size, // pointer size, abi alignment, preferred alignment
    374                        pointer_size, // aggregate preferred alignment
    375                        pointer_size, pointer_size); // stack objects abi alignment, preferred alignment
    376 
    377          gallivm->target = LLVMCreateTargetData(layout);
    378          if (!gallivm->target) {
    379             return FALSE;
    380          }
    381       }
    382    }
    383 
    384    if (!create_pass_manager(gallivm))
    385       goto fail;
    386 
    387    return TRUE;
    388 
    389 fail:
    390    gallivm_free_ir(gallivm);
    391    gallivm_free_code(gallivm);
    392    return FALSE;
    393 }
    394 
    395 
    396 boolean
    397 lp_build_init(void)
    398 {
    399    if (gallivm_initialized)
    400       return TRUE;
    401 
    402 
    403    /* LLVMLinkIn* are no-ops at runtime.  They just ensure the respective
    404     * component is linked at buildtime, which is sufficient for its static
    405     * constructors to be called at load time.
    406     */
    407 #if defined(USE_MCJIT)
    408 #  if USE_MCJIT
    409       LLVMLinkInMCJIT();
    410 #  else
    411       LLVMLinkInJIT();
    412 #  endif
    413 #else
    414    use_mcjit = debug_get_bool_option("GALLIVM_MCJIT", FALSE);
    415    LLVMLinkInJIT();
    416    LLVMLinkInMCJIT();
    417 #endif
    418 
    419 #ifdef DEBUG
    420    gallivm_debug = debug_get_option_gallivm_debug();
    421 #endif
    422 
    423    lp_set_target_options();
    424 
    425    util_cpu_detect();
    426 
    427    /* For simulating less capable machines */
    428 #ifdef DEBUG
    429    if (debug_get_bool_option("LP_FORCE_SSE2", FALSE)) {
    430       assert(util_cpu_caps.has_sse2);
    431       util_cpu_caps.has_sse3 = 0;
    432       util_cpu_caps.has_ssse3 = 0;
    433       util_cpu_caps.has_sse4_1 = 0;
    434       util_cpu_caps.has_sse4_2 = 0;
    435       util_cpu_caps.has_avx = 0;
    436       util_cpu_caps.has_avx2 = 0;
    437       util_cpu_caps.has_f16c = 0;
    438       util_cpu_caps.has_fma = 0;
    439    }
    440 #endif
    441 
    442    /* AMD Bulldozer AVX's throughput is the same as SSE2; and because using
    443     * 8-wide vector needs more floating ops than 4-wide (due to padding), it is
    444     * actually more efficient to use 4-wide vectors on this processor.
    445     *
    446     * See also:
    447     * - http://www.anandtech.com/show/4955/the-bulldozer-review-amd-fx8150-tested/2
    448     */
    449    if (util_cpu_caps.has_avx &&
    450        util_cpu_caps.has_intel) {
    451       lp_native_vector_width = 256;
    452    } else {
    453       /* Leave it at 128, even when no SIMD extensions are available.
    454        * Really needs to be a multiple of 128 so can fit 4 floats.
    455        */
    456       lp_native_vector_width = 128;
    457    }
    458 
    459    lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH",
    460                                                  lp_native_vector_width);
    461 
    462    if (lp_native_vector_width <= 128) {
    463       /* Hide AVX support, as often LLVM AVX intrinsics are only guarded by
    464        * "util_cpu_caps.has_avx" predicate, and lack the
    465        * "lp_native_vector_width > 128" predicate. And also to ensure a more
    466        * consistent behavior, allowing one to test SSE2 on AVX machines.
    467        * XXX: should not play games with util_cpu_caps directly as it might
    468        * get used for other things outside llvm too.
    469        */
    470       util_cpu_caps.has_avx = 0;
    471       util_cpu_caps.has_avx2 = 0;
    472       util_cpu_caps.has_f16c = 0;
    473       util_cpu_caps.has_fma = 0;
    474    }
    475    if (HAVE_LLVM < 0x0304 || !use_mcjit) {
    476       /* AVX2 support has only been tested with LLVM 3.4, and it requires
    477        * MCJIT. */
    478       util_cpu_caps.has_avx2 = 0;
    479    }
    480 
    481 #ifdef PIPE_ARCH_PPC_64
    482    /* Set the NJ bit in VSCR to 0 so denormalized values are handled as
    483     * specified by IEEE standard (PowerISA 2.06 - Section 6.3). This guarantees
    484     * that some rounding and half-float to float handling does not round
    485     * incorrectly to 0.
    486     * XXX: should eventually follow same logic on all platforms.
    487     * Right now denorms get explicitly disabled (but elsewhere) for x86,
    488     * whereas ppc64 explicitly enables them...
    489     */
    490    if (util_cpu_caps.has_altivec) {
    491       unsigned short mask[] = { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF,
    492                                 0xFFFF, 0xFFFF, 0xFFFE, 0xFFFF };
    493       __asm (
    494         "mfvscr %%v1\n"
    495         "vand   %0,%%v1,%0\n"
    496         "mtvscr %0"
    497         :
    498         : "r" (*mask)
    499       );
    500    }
    501 #endif
    502 
    503    gallivm_initialized = TRUE;
    504 
    505    return TRUE;
    506 }
    507 
    508 
    509 
    510 /**
    511  * Create a new gallivm_state object.
    512  */
    513 struct gallivm_state *
    514 gallivm_create(const char *name, LLVMContextRef context)
    515 {
    516    struct gallivm_state *gallivm;
    517 
    518    gallivm = CALLOC_STRUCT(gallivm_state);
    519    if (gallivm) {
    520       if (!init_gallivm_state(gallivm, name, context)) {
    521          FREE(gallivm);
    522          gallivm = NULL;
    523       }
    524    }
    525 
    526    return gallivm;
    527 }
    528 
    529 
    530 /**
    531  * Destroy a gallivm_state object.
    532  */
    533 void
    534 gallivm_destroy(struct gallivm_state *gallivm)
    535 {
    536    gallivm_free_ir(gallivm);
    537    gallivm_free_code(gallivm);
    538    FREE(gallivm);
    539 }
    540 
    541 
    542 /**
    543  * Validate a function.
    544  * Verification is only done with debug builds.
    545  */
    546 void
    547 gallivm_verify_function(struct gallivm_state *gallivm,
    548                         LLVMValueRef func)
    549 {
    550    /* Verify the LLVM IR.  If invalid, dump and abort */
    551 #ifdef DEBUG
    552    if (LLVMVerifyFunction(func, LLVMPrintMessageAction)) {
    553       lp_debug_dump_value(func);
    554       assert(0);
    555       return;
    556    }
    557 #endif
    558 
    559    if (gallivm_debug & GALLIVM_DEBUG_IR) {
    560       /* Print the LLVM IR to stderr */
    561       lp_debug_dump_value(func);
    562       debug_printf("\n");
    563    }
    564 }
    565 
    566 
    567 /**
    568  * Compile a module.
    569  * This does IR optimization on all functions in the module.
    570  */
    571 void
    572 gallivm_compile_module(struct gallivm_state *gallivm)
    573 {
    574    LLVMValueRef func;
    575    int64_t time_begin = 0;
    576 
    577    assert(!gallivm->compiled);
    578 
    579    if (gallivm->builder) {
    580       LLVMDisposeBuilder(gallivm->builder);
    581       gallivm->builder = NULL;
    582    }
    583 
    584    if (gallivm_debug & GALLIVM_DEBUG_PERF)
    585       time_begin = os_time_get();
    586 
    587    /* Run optimization passes */
    588    LLVMInitializeFunctionPassManager(gallivm->passmgr);
    589    func = LLVMGetFirstFunction(gallivm->module);
    590    while (func) {
    591       if (0) {
    592          debug_printf("optimizing func %s...\n", LLVMGetValueName(func));
    593       }
    594 
    595    /* Disable frame pointer omission on debug/profile builds */
    596    /* XXX: And workaround http://llvm.org/PR21435 */
    597 #if HAVE_LLVM >= 0x0307 && \
    598     (defined(DEBUG) || defined(PROFILE) || \
    599      defined(PIPE_ARCH_X86) || defined(PIPE_ARCH_X86_64))
    600       LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim", "true");
    601       LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim-non-leaf", "true");
    602 #endif
    603 
    604       LLVMRunFunctionPassManager(gallivm->passmgr, func);
    605       func = LLVMGetNextFunction(func);
    606    }
    607    LLVMFinalizeFunctionPassManager(gallivm->passmgr);
    608 
    609    if (gallivm_debug & GALLIVM_DEBUG_PERF) {
    610       int64_t time_end = os_time_get();
    611       int time_msec = (int)(time_end - time_begin) / 1000;
    612       assert(gallivm->module_name);
    613       debug_printf("optimizing module %s took %d msec\n",
    614                    gallivm->module_name, time_msec);
    615    }
    616 
    617    /* Dump byte code to a file */
    618    if (gallivm_debug & GALLIVM_DEBUG_DUMP_BC) {
    619       char filename[256];
    620       assert(gallivm->module_name);
    621       util_snprintf(filename, sizeof(filename), "ir_%s.bc", gallivm->module_name);
    622       LLVMWriteBitcodeToFile(gallivm->module, filename);
    623       debug_printf("%s written\n", filename);
    624       debug_printf("Invoke as \"llc %s%s -o - %s\"\n",
    625                    (HAVE_LLVM >= 0x0305) ? "[-mcpu=<-mcpu option] " : "",
    626                    "[-mattr=<-mattr option(s)>]",
    627                    filename);
    628    }
    629 
    630    if (use_mcjit) {
    631       assert(!gallivm->engine);
    632       if (!init_gallivm_engine(gallivm)) {
    633          assert(0);
    634       }
    635    }
    636    assert(gallivm->engine);
    637 
    638    ++gallivm->compiled;
    639 
    640    if (gallivm_debug & GALLIVM_DEBUG_ASM) {
    641       LLVMValueRef llvm_func = LLVMGetFirstFunction(gallivm->module);
    642 
    643       while (llvm_func) {
    644          /*
    645           * Need to filter out functions which don't have an implementation,
    646           * such as the intrinsics. May not be sufficient in case of IPO?
    647           * LLVMGetPointerToGlobal() will abort otherwise.
    648           */
    649          if (!LLVMIsDeclaration(llvm_func)) {
    650             void *func_code = LLVMGetPointerToGlobal(gallivm->engine, llvm_func);
    651             lp_disassemble(llvm_func, func_code);
    652          }
    653          llvm_func = LLVMGetNextFunction(llvm_func);
    654       }
    655    }
    656 
    657 #if defined(PROFILE)
    658    {
    659       LLVMValueRef llvm_func = LLVMGetFirstFunction(gallivm->module);
    660 
    661       while (llvm_func) {
    662          if (!LLVMIsDeclaration(llvm_func)) {
    663             void *func_code = LLVMGetPointerToGlobal(gallivm->engine, llvm_func);
    664             lp_profile(llvm_func, func_code);
    665          }
    666          llvm_func = LLVMGetNextFunction(llvm_func);
    667       }
    668    }
    669 #endif
    670 }
    671 
    672 
    673 
    674 func_pointer
    675 gallivm_jit_function(struct gallivm_state *gallivm,
    676                      LLVMValueRef func)
    677 {
    678    void *code;
    679    func_pointer jit_func;
    680    int64_t time_begin = 0;
    681 
    682    assert(gallivm->compiled);
    683    assert(gallivm->engine);
    684 
    685    if (gallivm_debug & GALLIVM_DEBUG_PERF)
    686       time_begin = os_time_get();
    687 
    688    code = LLVMGetPointerToGlobal(gallivm->engine, func);
    689    assert(code);
    690    jit_func = pointer_to_func(code);
    691 
    692    if (gallivm_debug & GALLIVM_DEBUG_PERF) {
    693       int64_t time_end = os_time_get();
    694       int time_msec = (int)(time_end - time_begin) / 1000;
    695       debug_printf("   jitting func %s took %d msec\n",
    696                    LLVMGetValueName(func), time_msec);
    697    }
    698 
    699    return jit_func;
    700 }
    701