GitLab will be upgraded to the 12.10.14-ce.0 on 28 Sept 2020 at 2.00pm (AEDT) to 2.30pm (AEDT). During the update, GitLab and Mattermost services will not be available. If you have any concerns with this, please talk to us at N110 (b) CSIT building.

util.py 7.68 KB
Newer Older
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
1
# Copyright 2017 The Australian National University
John Zhang's avatar
John Zhang committed
2
#
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
John Zhang's avatar
John Zhang committed
6
#
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
7
#     http://www.apache.org/licenses/LICENSE-2.0
John Zhang's avatar
John Zhang committed
8
#
Isaac Oscar Gariano's avatar
Isaac Oscar Gariano committed
9 10 11 12 13 14
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

John Zhang's avatar
John Zhang committed
15 16 17 18
import subprocess as subp
import os, sys
import ctypes
import py
19
from multiprocessing import Process
John Zhang's avatar
John Zhang committed
20 21

CC = os.environ.get('CC', 'clang')
22 23 24
# if $MU_ZEBU is defined, use that as the project directory, else define from file.
MU_ZEBU = os.environ.get('MU_ZEBU', '')
proj_dir = py.path.local(MU_ZEBU) if MU_ZEBU else py.path.local(__file__).join('..', '..', '..')
John Zhang's avatar
John Zhang committed
25 26
test_jit_dir = proj_dir.join('tests', 'test_jit')
testsuite_dir = test_jit_dir.join('suite')
27
# testsuite_dir = py.path.local('/Users/johnz/Documents/Work/mu-client-pypy/rpython/translator/mu/test_impl')
John Zhang's avatar
John Zhang committed
28 29 30
bin_dir = py.path.local('emit')     # put everything under emit
if not bin_dir.exists():
    bin_dir.mkdir()
31

32 33 34 35 36 37
if sys.platform.startswith('darwin'):
    libext = '.dylib'
elif sys.platform.startswith('linux'):
    libext = '.so'
else:
    libext = '.dll'
qinsoon's avatar
qinsoon committed
38

39
libmu_build = os.environ.get('ZEBU_BUILD', 'debug') + '/deps'
qinsoon's avatar
qinsoon committed
40 41 42 43

libmu_dir_path       = proj_dir.join('target', libmu_build)
libmu_dylib_path     = proj_dir.join('target', libmu_build, 'libmu' + libext)
libmu_staticlib_path = proj_dir.join('target', libmu_build, 'libmu.a')
John Zhang's avatar
John Zhang committed
44 45


46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
def mu_instance_via_ctyeps():
    libmu = preload_libmu()
    class MuVM(ctypes.Structure):
        pass
    MuVM._fields_ = [
            ('header', ctypes.c_voidp),
            ('new_context', ctypes.c_voidp),    # function pointers should have the same size as c_voidp
            ('id_of', ctypes.c_voidp),
            ('name_of', ctypes.c_voidp),
            ('set_trap_handler', ctypes.c_voidp),
            ('compile_to_sharedlib', ctypes.c_voidp),
            ('current_thread_as_mu_thread', ctypes.CFUNCTYPE(None, ctypes.POINTER(MuVM), ctypes.c_voidp)),
        ]
    libmu.mu_fastimpl_new.restype = ctypes.POINTER(MuVM)
    mu = libmu.mu_fastimpl_new()
    mu.contents.current_thread_as_mu_thread(mu, None)
    return mu


John Zhang's avatar
John Zhang committed
65 66 67 68 69 70 71
def compile_c_script(c_src_name):
    testname = c_src_name[:-2]
    src_c = testsuite_dir.join(c_src_name)
    bin_path = bin_dir.join(testname)
    CFLAGS = [
        "-std=c11",
        "-I%(proj_dir)s/src/vm/api" % globals(),
qinsoon's avatar
qinsoon committed
72
        "-L" + libmu_dir_path.strpath,
73
        "-lmu"
John Zhang's avatar
John Zhang committed
74
    ]
75
    cmd = [CC] + CFLAGS + ['-o', bin_path.strpath] + [src_c.strpath]
John Zhang's avatar
John Zhang committed
76 77 78 79 80 81 82 83 84

    # compile
    p = subp.Popen(cmd, stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
    out, err = p.communicate()
    if p.returncode != 0:  # failed
        sys.stdout.write(out + '\n')
        sys.stderr.write(err + '\n')
        raise subp.CalledProcessError(p.returncode, cmd)

qinsoon's avatar
qinsoon committed
85
    os.environ['LD_LIBRARY_PATH'] = "%s:%s" % (libmu_dir_path.strpath,
John Zhang's avatar
John Zhang committed
86 87
                                               os.environ['LD_LIBRARY_PATH'] if 'LD_LIBRARY_PATH' in os.environ else "")
    # run
88
    p = subp.Popen([bin_path.strpath], stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
John Zhang's avatar
John Zhang committed
89 90 91 92 93 94
    out, err = p.communicate()
    if p.returncode != 0:  # failed
        sys.stdout.write(out + '\n')
        sys.stderr.write(err + '\n')
        raise subp.CalledProcessError(p.returncode, bin_path)

95
    return py.path.local('emit').join('lib%(testname)s' % locals() + libext)
John Zhang's avatar
John Zhang committed
96 97


98
def ctypes_fncptr_from_lib(libpath, fnc_name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_GLOBAL):
99
    lib = ctypes.CDLL(libpath.strpath, mode)
John Zhang's avatar
John Zhang committed
100 101 102
    fnp = getattr(lib, fnc_name)
    fnp.argtypes = argtypes
    fnp.restype = restype
103 104 105
    return fnp, lib


106
def rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, restype, mode=ctypes.RTLD_GLOBAL):
107
    from rpython.rtyper.lltypesystem import rffi
108 109 110 111 112
    from rpython.translator.platform import platform
    if platform.name.startswith('linux'):
        link_extra = ['-Wl,-R' + libpath.dirpath().strpath]
    else:
        link_extra = []
113
    libname = libpath.basename[3:libpath.basename.index(libext)]
114

115 116 117
    if mode == ctypes.RTLD_GLOBAL:
        lib = ctypes.CDLL(libpath.strpath, mode)    # preload lib using RTLD_GLOBAL

118 119
    return rffi.llexternal(fnc_name, llargtypes, restype,
                           compilation_info=rffi.ExternalCompilationInfo(
120
                               libraries=[libname],
121 122
                               library_dirs=[libpath.dirpath().strpath],
                               link_extra=link_extra
123 124
                           ),
                           _nowrapper=True)
John Zhang's avatar
John Zhang committed
125

126

127
def fncptr_from_c_script(c_src_name, name, argtypes=[], restype=ctypes.c_ulonglong, mode=ctypes.RTLD_GLOBAL):
128
    libpath = compile_c_script(c_src_name)
129
    return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode)
130 131 132 133


def is_ctypes(t):
    return isinstance(t, type(ctypes.c_longlong))
John Zhang's avatar
John Zhang committed
134

135

136
def fncptr_from_py_script(py_fnc, heapinit_fnc, name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_GLOBAL, **kwargs):
137
    import os
John Zhang's avatar
John Zhang committed
138
    # NOTE: requires mu-client-pypy
139
    from rpython.rlib.rmu import zebu as rmu
John Zhang's avatar
John Zhang committed
140

141
    # load libmu before rffi so to load it with RTLD_GLOBAL
142
    libmu = preload_libmu()
143

144
    emit_dir = kwargs.get('muemitdir', os.environ.get('MU_EMIT_DIR', 'emit'))
145
    mu = rmu.MuVM("--aot-emit-dir=%(emit_dir)s" % locals())
John Zhang's avatar
John Zhang committed
146 147 148 149 150
    ctx = mu.new_context()
    bldr = ctx.new_ir_builder()

    id_dict = py_fnc(bldr, rmu)
    bldr.load()
151 152
    if heapinit_fnc:
        heapinit_fnc(ctx, id_dict, rmu)
153
    libpath = py.path.local(emit_dir).join('lib%(name)s' % locals() + libext)
154
    mu.compile_to_sharedlib(libpath.strpath, [])
John Zhang's avatar
John Zhang committed
155

156
    if (len(argtypes) > 0 and is_ctypes(argtypes[0])) or is_ctypes(restype):
157
        return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
158
    else:
159
        return rffi_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
John Zhang's avatar
John Zhang committed
160

161

162 163
def preload_libmu():
    # load libmu before rffi so to load it with RTLD_GLOBAL
qinsoon's avatar
qinsoon committed
164
    return ctypes.CDLL(libmu_dylib_path.strpath, ctypes.RTLD_GLOBAL)
165

166

167
spawn_proc = bool(int(os.environ.get('SPAWN_PROC', '1')))
168 169 170 171 172 173 174
def may_spawn_proc(test_fnc):
    def wrapper():
        if spawn_proc:
            p = Process(target=test_fnc, args=tuple())
            p.start()
            p.join()
            assert p.exitcode == 0
175
        else:
176 177
            test_fnc()
    return wrapper
178 179


180
def fncptr_from_rpy_func(rpy_fnc, llargtypes, llrestype, mode=ctypes.RTLD_GLOBAL, **kwargs):
John Zhang's avatar
John Zhang committed
181 182 183
    # NOTE: requires mu-client-pypy
    from rpython.rtyper.lltypesystem import rffi
    from rpython.translator.interactive import Translation
184
    from rpython.config.translationoption import set_opt_level
John Zhang's avatar
John Zhang committed
185

186
    preload_libmu()
187
    emit_dir = os.environ.get('MU_EMIT_DIR', str(bin_dir))
John Zhang's avatar
John Zhang committed
188
    kwargs.setdefault('backend', 'mu')
189 190 191 192
    kwargs.setdefault('impl', 'zebu')
    kwargs.setdefault('codegen', 'api')
    kwargs.setdefault('testjit', True)
    kwargs.setdefault('vmargs', "--aot-emit-dir=" + emit_dir)
193
    kwargs.setdefault('suplibdir', str(bin_dir))
194
    kwargs.setdefault('no_ovf', True)
John Zhang's avatar
John Zhang committed
195 196

    t = Translation(rpy_fnc, llargtypes, **kwargs)
197
    set_opt_level(t.config, '3')
John Zhang's avatar
John Zhang committed
198 199
    if kwargs['backend'] == 'mu':
        db, bdlgen, fnc_name = t.compile_mu()
200
        emit_dir = py.path.local(emit_dir)
201
        libpath = emit_dir.join('lib%(fnc_name)s' % locals() + libext)
202
        bdlgen.mu.compile_to_sharedlib(libpath.strpath, [])
John Zhang's avatar
John Zhang committed
203 204 205 206 207
        extras = (db, bdlgen)
    else:
        libpath = t.compile_c()
        fnc_name = 'pypy_g_' + rpy_fnc.__name__
        extras = None
208
    return rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, llrestype, mode), extras