WARNING! Access to this system is limited to authorised users only.
Unauthorised users may be subject to prosecution.
Unauthorised access to this system is a criminal offence under Australian law (Federal Crimes Act 1914 Part VIA)
It is a criminal offence to:
(1) Obtain access to data without authority. -Penalty 2 years imprisonment.
(2) Damage, delete, alter or insert data without authority. -Penalty 10 years imprisonment.
User activity is monitored and recorded. Anyone using this system expressly consents to such monitoring and recording.

util.py 6.87 KB
Newer Older
John Zhang's avatar
John Zhang committed
1 2 3 4
import subprocess as subp
import os, sys
import ctypes
import py
5
from multiprocessing import Process
John Zhang's avatar
John Zhang committed
6 7 8 9 10

CC = os.environ.get('CC', 'clang')
proj_dir = py.path.local(__file__).join('..', '..', '..')
test_jit_dir = proj_dir.join('tests', 'test_jit')
testsuite_dir = test_jit_dir.join('suite')
11 12 13
# testsuite_dir = py.path.local('/Users/johnz/Documents/Work/mu-client-pypy/rpython/translator/mu/test_impl')
bin_dir = py.path.local('/tmp')

14 15 16 17 18 19
if sys.platform.startswith('darwin'):
    libext = '.dylib'
elif sys.platform.startswith('linux'):
    libext = '.so'
else:
    libext = '.dll'
qinsoon's avatar
qinsoon committed
20 21 22 23 24 25

libmu_build = os.environ.get('ZEBU_BUILD', 'debug')

libmu_dir_path       = proj_dir.join('target', libmu_build)
libmu_dylib_path     = proj_dir.join('target', libmu_build, 'libmu' + libext)
libmu_staticlib_path = proj_dir.join('target', libmu_build, 'libmu.a')
John Zhang's avatar
John Zhang committed
26 27


28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
def mu_instance_via_ctyeps():
    libmu = preload_libmu()
    class MuVM(ctypes.Structure):
        pass
    MuVM._fields_ = [
            ('header', ctypes.c_voidp),
            ('new_context', ctypes.c_voidp),    # function pointers should have the same size as c_voidp
            ('id_of', ctypes.c_voidp),
            ('name_of', ctypes.c_voidp),
            ('set_trap_handler', ctypes.c_voidp),
            ('compile_to_sharedlib', ctypes.c_voidp),
            ('current_thread_as_mu_thread', ctypes.CFUNCTYPE(None, ctypes.POINTER(MuVM), ctypes.c_voidp)),
        ]
    libmu.mu_fastimpl_new.restype = ctypes.POINTER(MuVM)
    mu = libmu.mu_fastimpl_new()
    mu.contents.current_thread_as_mu_thread(mu, None)
    return mu


John Zhang's avatar
John Zhang committed
47 48 49 50 51 52 53
def compile_c_script(c_src_name):
    testname = c_src_name[:-2]
    src_c = testsuite_dir.join(c_src_name)
    bin_path = bin_dir.join(testname)
    CFLAGS = [
        "-std=c11",
        "-I%(proj_dir)s/src/vm/api" % globals(),
qinsoon's avatar
qinsoon committed
54
        "-L" + libmu_dir_path.strpath,
55
        "-lmu"
John Zhang's avatar
John Zhang committed
56
    ]
57
    cmd = [CC] + CFLAGS + ['-o', bin_path.strpath] + [src_c.strpath]
John Zhang's avatar
John Zhang committed
58 59 60 61 62 63 64 65 66

    # compile
    p = subp.Popen(cmd, stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
    out, err = p.communicate()
    if p.returncode != 0:  # failed
        sys.stdout.write(out + '\n')
        sys.stderr.write(err + '\n')
        raise subp.CalledProcessError(p.returncode, cmd)

qinsoon's avatar
qinsoon committed
67
    os.environ['LD_LIBRARY_PATH'] = "%s:%s" % (libmu_dir_path.strpath,
John Zhang's avatar
John Zhang committed
68 69
                                               os.environ['LD_LIBRARY_PATH'] if 'LD_LIBRARY_PATH' in os.environ else "")
    # run
70
    p = subp.Popen([bin_path.strpath], stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
John Zhang's avatar
John Zhang committed
71 72 73 74 75 76
    out, err = p.communicate()
    if p.returncode != 0:  # failed
        sys.stdout.write(out + '\n')
        sys.stderr.write(err + '\n')
        raise subp.CalledProcessError(p.returncode, bin_path)

77
    return py.path.local('emit').join('lib%(testname)s' % locals() + libext)
John Zhang's avatar
John Zhang committed
78 79


80
def ctypes_fncptr_from_lib(libpath, fnc_name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_GLOBAL):
81
    lib = ctypes.CDLL(libpath.strpath, mode)
John Zhang's avatar
John Zhang committed
82 83 84
    fnp = getattr(lib, fnc_name)
    fnp.argtypes = argtypes
    fnp.restype = restype
85 86 87
    return fnp, lib


88
def rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, restype, mode=ctypes.RTLD_GLOBAL):
89
    from rpython.rtyper.lltypesystem import rffi
90 91 92 93 94
    from rpython.translator.platform import platform
    if platform.name.startswith('linux'):
        link_extra = ['-Wl,-R' + libpath.dirpath().strpath]
    else:
        link_extra = []
95
    libname = libpath.basename[3:libpath.basename.index(libext)]
96

97 98 99
    if mode == ctypes.RTLD_GLOBAL:
        lib = ctypes.CDLL(libpath.strpath, mode)    # preload lib using RTLD_GLOBAL

100 101
    return rffi.llexternal(fnc_name, llargtypes, restype,
                           compilation_info=rffi.ExternalCompilationInfo(
102
                               libraries=[libname],
103 104
                               library_dirs=[libpath.dirpath().strpath],
                               link_extra=link_extra
105 106
                           ),
                           _nowrapper=True)
John Zhang's avatar
John Zhang committed
107

108

109
def fncptr_from_c_script(c_src_name, name, argtypes=[], restype=ctypes.c_ulonglong, mode=ctypes.RTLD_GLOBAL):
110
    libpath = compile_c_script(c_src_name)
111
    return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode)
112 113 114 115


def is_ctypes(t):
    return isinstance(t, type(ctypes.c_longlong))
John Zhang's avatar
John Zhang committed
116

117

118
def fncptr_from_py_script(py_fnc, heapinit_fnc, name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_GLOBAL, **kwargs):
119
    import os
John Zhang's avatar
John Zhang committed
120
    # NOTE: requires mu-client-pypy
121
    from rpython.rlib.rmu import zebu as rmu
John Zhang's avatar
John Zhang committed
122

123
    # load libmu before rffi so to load it with RTLD_GLOBAL
124
    libmu = preload_libmu()
125

126
    loglvl = os.environ.get('MU_LOG_LEVEL', 'none')
127
    emit_dir = kwargs.get('muemitdir', os.environ.get('MU_EMIT_DIR', 'emit'))
128
    mu = rmu.MuVM("--log-level=%(loglvl)s --aot-emit-dir=%(emit_dir)s" % locals())
John Zhang's avatar
John Zhang committed
129 130 131 132 133
    ctx = mu.new_context()
    bldr = ctx.new_ir_builder()

    id_dict = py_fnc(bldr, rmu)
    bldr.load()
134 135
    if heapinit_fnc:
        heapinit_fnc(ctx, id_dict, rmu)
136
    libpath = py.path.local(emit_dir).join('lib%(name)s' % locals() + libext)
137
    mu.compile_to_sharedlib(libpath.strpath, [])
John Zhang's avatar
John Zhang committed
138

139
    if (len(argtypes) > 0 and is_ctypes(argtypes[0])) or is_ctypes(restype):
140
        return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
141
    else:
142
        return rffi_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
John Zhang's avatar
John Zhang committed
143

144

145 146
def preload_libmu():
    # load libmu before rffi so to load it with RTLD_GLOBAL
qinsoon's avatar
qinsoon committed
147
    return ctypes.CDLL(libmu_dylib_path.strpath, ctypes.RTLD_GLOBAL)
148

149

John Zhang's avatar
John Zhang committed
150
spawn_proc = bool(int(os.environ.get('SPAWN_PROC', '1')))
151 152 153 154 155 156 157
def may_spawn_proc(test_fnc):
    def wrapper():
        if spawn_proc:
            p = Process(target=test_fnc, args=tuple())
            p.start()
            p.join()
            assert p.exitcode == 0
John Zhang's avatar
John Zhang committed
158
        else:
159 160
            test_fnc()
    return wrapper
161 162


163
def fncptr_from_rpy_func(rpy_fnc, llargtypes, llrestype, mode=ctypes.RTLD_GLOBAL, **kwargs):
John Zhang's avatar
John Zhang committed
164 165 166
    # NOTE: requires mu-client-pypy
    from rpython.rtyper.lltypesystem import rffi
    from rpython.translator.interactive import Translation
167
    from rpython.config.translationoption import set_opt_level
John Zhang's avatar
John Zhang committed
168

169
    preload_libmu()
170
    emit_dir = os.environ.get('MU_EMIT_DIR', 'emit')
John Zhang's avatar
John Zhang committed
171
    kwargs.setdefault('backend', 'mu')
172 173 174 175
    kwargs.setdefault('impl', 'zebu')
    kwargs.setdefault('codegen', 'api')
    kwargs.setdefault('testjit', True)
    kwargs.setdefault('vmargs', "--aot-emit-dir=" + emit_dir)
176
    kwargs.setdefault('no_ovf', True)
John Zhang's avatar
John Zhang committed
177 178

    t = Translation(rpy_fnc, llargtypes, **kwargs)
179
    set_opt_level(t.config, '3')
John Zhang's avatar
John Zhang committed
180 181
    if kwargs['backend'] == 'mu':
        db, bdlgen, fnc_name = t.compile_mu()
182
        emit_dir = py.path.local(emit_dir)
183
        libpath = emit_dir.join('lib%(fnc_name)s' % locals() + libext)
184
        bdlgen.mu.compile_to_sharedlib(libpath.strpath, [])
John Zhang's avatar
John Zhang committed
185 186 187 188 189
        extras = (db, bdlgen)
    else:
        libpath = t.compile_c()
        fnc_name = 'pypy_g_' + rpy_fnc.__name__
        extras = None
190
    return rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, llrestype, mode), extras