util.py 6.68 KB
Newer Older
John Zhang's avatar
John Zhang committed
1 2 3 4
import subprocess as subp
import os, sys
import ctypes
import py
5
from multiprocessing import Process
John Zhang's avatar
John Zhang committed
6 7 8 9 10

CC = os.environ.get('CC', 'clang')
proj_dir = py.path.local(__file__).join('..', '..', '..')
test_jit_dir = proj_dir.join('tests', 'test_jit')
testsuite_dir = test_jit_dir.join('suite')
11 12 13
# testsuite_dir = py.path.local('/Users/johnz/Documents/Work/mu-client-pypy/rpython/translator/mu/test_impl')
bin_dir = py.path.local('/tmp')

14 15 16 17 18 19 20
if sys.platform.startswith('darwin'):
    libext = '.dylib'
elif sys.platform.startswith('linux'):
    libext = '.so'
else:
    libext = '.dll'
libmu_path = proj_dir.join('target', 'debug', 'libmu' + libext)
John Zhang's avatar
John Zhang committed
21 22


23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41
def mu_instance_via_ctyeps():
    libmu = preload_libmu()
    class MuVM(ctypes.Structure):
        pass
    MuVM._fields_ = [
            ('header', ctypes.c_voidp),
            ('new_context', ctypes.c_voidp),    # function pointers should have the same size as c_voidp
            ('id_of', ctypes.c_voidp),
            ('name_of', ctypes.c_voidp),
            ('set_trap_handler', ctypes.c_voidp),
            ('compile_to_sharedlib', ctypes.c_voidp),
            ('current_thread_as_mu_thread', ctypes.CFUNCTYPE(None, ctypes.POINTER(MuVM), ctypes.c_voidp)),
        ]
    libmu.mu_fastimpl_new.restype = ctypes.POINTER(MuVM)
    mu = libmu.mu_fastimpl_new()
    mu.contents.current_thread_as_mu_thread(mu, None)
    return mu


John Zhang's avatar
John Zhang committed
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
def compile_c_script(c_src_name):
    testname = c_src_name[:-2]
    src_c = testsuite_dir.join(c_src_name)
    bin_path = bin_dir.join(testname)
    CFLAGS = [
        "-std=c11",
        "-I%(proj_dir)s/src/vm/api" % globals(),
        "-L" + libmu_path.dirname,
        "-lmu",
    ]
    cmd = [CC] + CFLAGS + ['-o', bin_path.strpath] + [src_c.strpath]

    # compile
    p = subp.Popen(cmd, stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
    out, err = p.communicate()
    if p.returncode != 0:  # failed
        sys.stdout.write(out + '\n')
        sys.stderr.write(err + '\n')
        raise subp.CalledProcessError(p.returncode, cmd)

    os.environ['LD_LIBRARY_PATH'] = "%s:%s" % ("%(proj_dir)s/target/debug" % globals(),
                                               os.environ['LD_LIBRARY_PATH'] if 'LD_LIBRARY_PATH' in os.environ else "")
    # run
65
    p = subp.Popen([bin_path.strpath], stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
John Zhang's avatar
John Zhang committed
66 67 68 69 70 71
    out, err = p.communicate()
    if p.returncode != 0:  # failed
        sys.stdout.write(out + '\n')
        sys.stderr.write(err + '\n')
        raise subp.CalledProcessError(p.returncode, bin_path)

72
    return py.path.local('emit').join('lib%(testname)s' % locals() + libext)
John Zhang's avatar
John Zhang committed
73 74


75 76
def ctypes_fncptr_from_lib(libpath, fnc_name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_LOCAL):
    lib = ctypes.CDLL(libpath.strpath, mode)
John Zhang's avatar
John Zhang committed
77 78 79
    fnp = getattr(lib, fnc_name)
    fnp.argtypes = argtypes
    fnp.restype = restype
80 81 82
    return fnp, lib


83
def rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, restype, mode=ctypes.RTLD_LOCAL):
84
    from rpython.rtyper.lltypesystem import rffi
85 86 87 88 89
    from rpython.translator.platform import platform
    if platform.name.startswith('linux'):
        link_extra = ['-Wl,-R' + libpath.dirpath().strpath]
    else:
        link_extra = []
90
    libname = libpath.basename[3:libpath.basename.index(libext)]
91

92 93 94
    if mode == ctypes.RTLD_GLOBAL:
        lib = ctypes.CDLL(libpath.strpath, mode)    # preload lib using RTLD_GLOBAL

95 96
    return rffi.llexternal(fnc_name, llargtypes, restype,
                           compilation_info=rffi.ExternalCompilationInfo(
97
                               libraries=[libname],
98 99
                               library_dirs=[libpath.dirpath().strpath],
                               link_extra=link_extra
100 101
                           ),
                           _nowrapper=True)
John Zhang's avatar
John Zhang committed
102

103

104
def fncptr_from_c_script(c_src_name, name, argtypes=[], restype=ctypes.c_ulonglong, mode=ctypes.RTLD_LOCAL):
105
    libpath = compile_c_script(c_src_name)
106
    return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode)
107 108 109 110


def is_ctypes(t):
    return isinstance(t, type(ctypes.c_longlong))
John Zhang's avatar
John Zhang committed
111

112

113
def fncptr_from_py_script(py_fnc, heapinit_fnc, name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_LOCAL, **kwargs):
114
    import os
John Zhang's avatar
John Zhang committed
115
    # NOTE: requires mu-client-pypy
116
    from rpython.rlib.rmu import zebu as rmu
John Zhang's avatar
John Zhang committed
117

118
    # load libmu before rffi so to load it with RTLD_GLOBAL
119
    libmu = preload_libmu()
120

121
    loglvl = os.environ.get('MU_LOG_LEVEL', 'none')
122
    emit_dir = kwargs.get('muemitdir', os.environ.get('MU_EMIT_DIR', 'emit'))
123
    mu = rmu.MuVM("--log-level=%(loglvl)s --aot-emit-dir=%(emit_dir)s" % locals())
John Zhang's avatar
John Zhang committed
124 125 126 127 128
    ctx = mu.new_context()
    bldr = ctx.new_ir_builder()

    id_dict = py_fnc(bldr, rmu)
    bldr.load()
129 130
    if heapinit_fnc:
        heapinit_fnc(ctx, id_dict, rmu)
131
    libpath = py.path.local(emit_dir).join('lib%(name)s' % locals() + libext)
132
    mu.compile_to_sharedlib(libpath.strpath, [])
John Zhang's avatar
John Zhang committed
133

134
    if (len(argtypes) > 0 and is_ctypes(argtypes[0])) or is_ctypes(restype):
135
        return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
136
    else:
137
        return rffi_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
John Zhang's avatar
John Zhang committed
138

139

140 141 142
def preload_libmu():
    # load libmu before rffi so to load it with RTLD_GLOBAL
    return ctypes.CDLL(libmu_path.strpath, ctypes.RTLD_GLOBAL)
143

144

145
spawn_proc = bool(int(os.environ.get('SPAWN_PROC', '1')))
146 147 148 149 150 151 152
def may_spawn_proc(test_fnc):
    def wrapper():
        if spawn_proc:
            p = Process(target=test_fnc, args=tuple())
            p.start()
            p.join()
            assert p.exitcode == 0
153
        else:
154 155
            test_fnc()
    return wrapper
156 157


158
def fncptr_from_rpy_func(rpy_fnc, llargtypes, llrestype, mode=ctypes.RTLD_LOCAL, **kwargs):
John Zhang's avatar
John Zhang committed
159 160 161
    # NOTE: requires mu-client-pypy
    from rpython.rtyper.lltypesystem import rffi
    from rpython.translator.interactive import Translation
162
    from rpython.config.translationoption import set_opt_level
John Zhang's avatar
John Zhang committed
163

164
    preload_libmu()
165
    emit_dir = os.environ.get('MU_EMIT_DIR', 'emit')
John Zhang's avatar
John Zhang committed
166
    kwargs.setdefault('backend', 'mu')
167 168 169 170
    kwargs.setdefault('impl', 'zebu')
    kwargs.setdefault('codegen', 'api')
    kwargs.setdefault('testjit', True)
    kwargs.setdefault('vmargs', "--aot-emit-dir=" + emit_dir)
171
    kwargs.setdefault('no_ovf', True)
John Zhang's avatar
John Zhang committed
172 173

    t = Translation(rpy_fnc, llargtypes, **kwargs)
174
    set_opt_level(t.config, '3')
John Zhang's avatar
John Zhang committed
175 176
    if kwargs['backend'] == 'mu':
        db, bdlgen, fnc_name = t.compile_mu()
177
        emit_dir = py.path.local(emit_dir)
178
        libpath = emit_dir.join('lib%(fnc_name)s' % locals() + libext)
179
        bdlgen.mu.compile_to_sharedlib(libpath.strpath, [])
John Zhang's avatar
John Zhang committed
180 181 182 183 184
        extras = (db, bdlgen)
    else:
        libpath = t.compile_c()
        fnc_name = 'pypy_g_' + rpy_fnc.__name__
        extras = None
185
    return rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, llrestype, mode), extras