To protect your data, the CISO officer has suggested users to enable GitLab 2FA as soon as possible.

Commit db98df6f authored by Javad Ebrahimian Amiri's avatar Javad Ebrahimian Amiri
Browse files

added some RTMu stuff, missing on previous commits

parent 177d90a2
from rpython.rlib.rrtmu import rtzebu
def alloc_region(size):
return rtzebu.new_region(size)
alloc_region._dont_inline_ = True
def tester():
s = 24
res = alloc_region(s)
return res
from rpython.rlib.rthread import *
import time
counter = 0
def thread_entry_point():
global counter
while counter < 1000000:
id = get_ident()
print('thread id (((%d))) counter (((%d))) \n' %(id, counter))
# print('counter = %d \n' %counter)
counter = counter+1
def run_rthreads():
for i in range(10):
start_new_thread(thread_entry_point, ())
time.sleep(10)
\ No newline at end of file
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess as subp
import os, sys
import ctypes
import py
from multiprocessing import Process
CC = os.environ.get('CC', 'clang')
# if $MU_ZEBU is defined, use that as the project directory, else define from file.
MU_ZEBU = os.environ.get('MU_ZEBU', '')
proj_dir = py.path.local(MU_ZEBU) if MU_ZEBU else py.path.local(__file__).join('..', '..', '..')
test_jit_dir = proj_dir.join('tests', 'test_jit')
testsuite_dir = test_jit_dir.join('suite')
# testsuite_dir = py.path.local('/Users/johnz/Documents/Work/mu-client-pypy/rpython/translator/mu/test_impl')
bin_dir = py.path.local('emit') # put everything under emit
if not bin_dir.exists():
bin_dir.mkdir()
if sys.platform.startswith('darwin'):
libext = '.dylib'
elif sys.platform.startswith('linux'):
libext = '.so'
else:
libext = '.dll'
libmu_build = os.environ.get('ZEBU_BUILD', 'debug') + '/deps'
libmu_dir_path = proj_dir.join('target', libmu_build)
libmu_dylib_path = proj_dir.join('target', libmu_build, 'libmu' + libext)
libmu_staticlib_path = proj_dir.join('target', libmu_build, 'libmu.a')
def mu_instance_via_ctyeps():
libmu = preload_libmu()
class MuVM(ctypes.Structure):
pass
MuVM._fields_ = [
('header', ctypes.c_voidp),
('new_context', ctypes.c_voidp), # function pointers should have the same size as c_voidp
('id_of', ctypes.c_voidp),
('name_of', ctypes.c_voidp),
('set_trap_handler', ctypes.c_voidp),
('compile_to_sharedlib', ctypes.c_voidp),
('current_thread_as_mu_thread', ctypes.CFUNCTYPE(None, ctypes.POINTER(MuVM), ctypes.c_voidp)),
]
libmu.mu_fastimpl_new.restype = ctypes.POINTER(MuVM)
mu = libmu.mu_fastimpl_new()
mu.contents.current_thread_as_mu_thread(mu, None)
return mu
def compile_c_script(c_src_name):
testname = c_src_name[:-2]
src_c = testsuite_dir.join(c_src_name)
bin_path = bin_dir.join(testname)
CFLAGS = [
"-std=c11",
"-I%(proj_dir)s/src/vm/api" % globals(),
"-L" + libmu_dir_path.strpath,
"-lmu"
]
cmd = [CC] + CFLAGS + ['-o', bin_path.strpath] + [src_c.strpath]
# compile
p = subp.Popen(cmd, stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
out, err = p.communicate()
if p.returncode != 0: # failed
sys.stdout.write(out + '\n')
sys.stderr.write(err + '\n')
raise subp.CalledProcessError(p.returncode, cmd)
os.environ['LD_LIBRARY_PATH'] = "%s:%s" % (libmu_dir_path.strpath,
os.environ['LD_LIBRARY_PATH'] if 'LD_LIBRARY_PATH' in os.environ else "")
# run
p = subp.Popen([bin_path.strpath], stdout=subp.PIPE, stderr=subp.PIPE, env=os.environ)
out, err = p.communicate()
if p.returncode != 0: # failed
sys.stdout.write(out + '\n')
sys.stderr.write(err + '\n')
raise subp.CalledProcessError(p.returncode, bin_path)
return py.path.local('emit').join('lib%(testname)s' % locals() + libext)
def ctypes_fncptr_from_lib(libpath, fnc_name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_GLOBAL):
lib = ctypes.CDLL(libpath.strpath, mode)
fnp = getattr(lib, fnc_name)
fnp.argtypes = argtypes
fnp.restype = restype
return fnp, lib
def rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, restype, mode=ctypes.RTLD_GLOBAL):
from rpython.rtyper.lltypesystem import rffi
from rpython.translator.platform import platform
if platform.name.startswith('linux'):
link_extra = ['-Wl,-R' + libpath.dirpath().strpath]
else:
link_extra = []
libname = libpath.basename[3:libpath.basename.index(libext)]
if mode == ctypes.RTLD_GLOBAL:
lib = ctypes.CDLL(libpath.strpath, mode) # preload lib using RTLD_GLOBAL
return rffi.llexternal(fnc_name, llargtypes, restype,
compilation_info=rffi.ExternalCompilationInfo(
libraries=[libname],
library_dirs=[libpath.dirpath().strpath],
link_extra=link_extra
),
_nowrapper=True)
def fncptr_from_c_script(c_src_name, name, argtypes=[], restype=ctypes.c_ulonglong, mode=ctypes.RTLD_GLOBAL):
libpath = compile_c_script(c_src_name)
return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode)
def is_ctypes(t):
return isinstance(t, type(ctypes.c_longlong))
def fncptr_from_py_script(py_fnc, heapinit_fnc, name, argtypes=[], restype=ctypes.c_longlong, mode=ctypes.RTLD_GLOBAL, **kwargs):
import os
# NOTE: requires mu-client-pypy
from rpython.rlib.rmu import zebu as rmu
# load libmu before rffi so to load it with RTLD_GLOBAL
libmu = preload_libmu()
emit_dir = kwargs.get('muemitdir', os.environ.get('MU_EMIT_DIR', 'emit'))
mu = rmu.MuVM("--aot-emit-dir=%(emit_dir)s" % locals())
ctx = mu.new_context()
bldr = ctx.new_ir_builder()
id_dict = py_fnc(bldr, rmu)
bldr.load()
if heapinit_fnc:
heapinit_fnc(ctx, id_dict, rmu)
libpath = py.path.local(emit_dir).join('lib%(name)s' % locals() + libext)
mu.compile_to_sharedlib(libpath.strpath, [])
if (len(argtypes) > 0 and is_ctypes(argtypes[0])) or is_ctypes(restype):
return ctypes_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
else:
return rffi_fncptr_from_lib(libpath, name, argtypes, restype, mode), (mu, ctx, bldr)
def preload_libmu():
# load libmu before rffi so to load it with RTLD_GLOBAL
return ctypes.CDLL(libmu_dylib_path.strpath, ctypes.RTLD_GLOBAL)
# return ctypes.CDLL(libmu_staticlib_path.strpath, ctypes.RTLD_GLOBAL)
spawn_proc = bool(int(os.environ.get('SPAWN_PROC', '1')))
def may_spawn_proc(test_fnc):
def wrapper():
if spawn_proc:
p = Process(target=test_fnc, args=tuple())
p.start()
p.join()
assert p.exitcode == 0
else:
test_fnc()
return wrapper
def fncptr_from_rpy_func(rpy_fnc, llargtypes, llrestype, mode=ctypes.RTLD_GLOBAL, **kwargs):
# NOTE: requires mu-client-pypy
from rpython.rtyper.lltypesystem import rffi
from rpython.translator.interactive import Translation
from rpython.config.translationoption import set_opt_level
preload_libmu()
emit_dir = os.environ.get('MU_EMIT_DIR', str(bin_dir))
kwargs.setdefault('backend', 'mu')
kwargs.setdefault('impl', 'zebu')
kwargs.setdefault('codegen', 'api')
kwargs.setdefault('testjit', True)
kwargs.setdefault('vmargs', "--aot-emit-dir=" + emit_dir)
kwargs.setdefault('suplibdir', str(bin_dir))
kwargs.setdefault('no_ovf', True)
t = Translation(rpy_fnc, llargtypes, **kwargs)
set_opt_level(t.config, '3')
if kwargs['backend'] == 'mu':
db, bdlgen, fnc_name = t.compile_mu()
print("compile_mu finished")
emit_dir = py.path.local(emit_dir)
libpath = emit_dir.join('lib%(fnc_name)s' % locals() + libext)
print("libpath => %s" % libpath)
bdlgen.mu.compile_to_sharedlib(libpath.strpath, [])
print("compiled to shared lib")
extras = (db, bdlgen)
else:
libpath = t.compile_c()
fnc_name = 'pypy_g_' + rpy_fnc.__name__
extras = None
return rffi_fncptr_from_lib(libpath, fnc_name, llargtypes, llrestype, mode), extras
from rpython.rlib.rmu.zebu import *
from rpython.rtyper.lltypesystem import lltype, rffi
c_exit = rffi.llexternal('exit', [rffi.INT], lltype.Void, _nowrapper=True)
RTMU_OP_NAMES = {
"new_region",
"delete_region",
"enter_region",
"exit_region",
"enter_emm",
"exit_emm",
# "start_new_rt_thread",
"sleep",
"get_time_ns",
"new_timer",
"delete_timer",
"arm_timer",
"disarm_timer",
"new_attr",
"delete_attr",
"set_attr_priority",
"get_attr_priority",
"set_attr_cpu",
"isset_attr_cpu",
"clear_attr_cpu",
"zero_attr_cpu",
"new_futex",
"wait_futex",
"wake_futex"
}
RTMU_NOP_NAMES = {
"Region.__init__",
"Thread.__init__",
"Thread___init__",
"Attr.__init__",
"Attr___init__",
"PIFutex.__init__",
"PIFutex___init__",
# "_fake_entry",
"do_something_realy_non_trivial",
"do_something_non_trivial"
}
RTMU_TRD_NAMES = {
"start_new_rt_thread",
"_fake_entry"
}
RTMU_TYPE_NAMES = {
"MU_REGIONREF",
"MU_FUTEXREF",
"MU_ATTRREF",
"MU_TIMERREF",
"MU_THREADREF",
"MU_STACKREF"
}
RTMU_METHOD_NAMES = {
"delete"
}
NO_COPY_LIST = {
"memo_",
"copy_string_to_raw",
"scoped_nonmovingbuffer",
"free_nonmovingbuffer",
"write",
"validate_fd",
"get_nonmovingbuffer",
"_errno_",
"setraw",
"getraw",
"rpython_print_newline",
"rpython_print_item",
"get_saved_errno",
"dtoa",
"format_number",
"str2charp",
"time",
}
#########################################
# new RT_RPYTHON types
# on top of RTMu
#########################################
class RTMuObject(object):
def delete(self):
s = do_something_non_trivial("enter_emm")
t = do_something_non_trivial(s)
u = do_something_non_trivial(t)
class Region:
def __init__(self, size):
self.__size = size
def get_self(self):
return self
class PIFutex:
__slots__ = ('id',)
def __init__(self):
self.id = self
class Timer:
__slots__ = ('id',)
def __init__(self):
self.id = self
class Attr:
__slots__ = ('id',)
def __init__(self):
self.id = self
class Thread:
__slots__ = ('entry', 'args', 'attr')
def __init__(self, entry=None, attr=None, args=None):
self.entry = entry if entry else None
self.attr = attr if attr else None
self.args = args if args else None
class MutableGlobals:
class Created:
pass
instance = None
def __init__(self):
if not MutableGlobals.instance:
MutableGlobals.instance = MutableGlobals.Created()
mutable_globals = MutableGlobals()
#########################################
# Implementation sugars
#########################################
__fake_region_ptr = Region(1)
__fake_futex_ptr = PIFutex()
__fake_timer_ptr = Timer()
__fake_attr_ptr = Attr()
# __fake_thread_ptr = Thread()
#########################################
# helpers to fill the gap between RPython
# and RT_RPython
#########################################
HEAP = 0
EMM = 1
REGIONS = 2
CURRENT_AREA = HEAP
CURRENT_REGION = __fake_region_ptr
def get_current_region():
global CURRENT_REGION
return CURRENT_REGION.get_self()
def set_current_region(some_reg):
global CURRENT_REGION
CURRENT_REGION = some_reg
def unset_current_region():
global CURRENT_REGION, __fake_region_ptr
CURRENT_REGION = __fake_region_ptr
def is_current_region_set():
global CURRENT_REGION, __fake_region_ptr
return CURRENT_REGION == __fake_region_ptr
def is_rtmu_reserved_method(fn):
short_name = fn.split('.')[-1]
# logger.info('is_rtmu_reserved_method: %s' % short_name)
return short_name in RTMU_METHOD_NAMES
def do_something_non_trivial(func):
fn = func
return fn + "_non_trivial"
def do_something_realy_non_trivial(func):
s = do_something_non_trivial(func)
t = do_something_non_trivial(s)
u = do_something_non_trivial(t)
return u
def is_rtmu_untraced(graph):
return graph.name[-16:-1] == '_rtmu_untraced_'
#########################################
# new RT_RPYTHON instructions
# on top of RTMu
#########################################
def os_exit(res):
c_exit(rffi.cast(rffi.INT, res))
def delete_self():
res = do_something_realy_non_trivial('delete_self')
def new_region(size):
return Region(size)
# return __fake_region_ptr.get_self()
def delete_region(reg):
res = do_something_realy_non_trivial('delete_region')
def enter_emm():
res = do_something_realy_non_trivial("enter_emm")
def exit_emm():
res = do_something_realy_non_trivial('exit_emm')
def enter_region(reg):
res = do_something_realy_non_trivial('enter_region')
def exit_region():
res = do_something_realy_non_trivial('exit_region')
class GlFlag:
_flag = False
_fake_thread_initialized = GlFlag()
_fake_timer_initialized = GlFlag()
def start_new_rt_thread(entry, attr, *args):
# res = do_something_realy_non_trivial('start_rt_thread')
# assert callable(entry)
entry(*args)
global _fake_thread_initialized
if not _fake_thread_initialized._flag:
def _fake_entry(*args):
pass
_fake_thread_initialized._flag = True
start_new_rt_thread(_fake_entry, attr, *args)
entry(*args)
return Thread()
def get_my_thread_id():
return Thread()
def terminate_thread(t):
res = do_something_realy_non_trivial('terminate_thread')
def yield_turn():
res = do_something_realy_non_trivial('yield')
def set_thread_priority(t, p):
r = do_something_realy_non_trivial("set_thread_priority")
def get_thread_priority(t):
import random
return random.randint(0, 255)
def set_thread_cpu(t, c):
r = do_something_realy_non_trivial("set_thread_cpu")
def isset_thread_cpu(t, c):
import random
return True if (random.randint(0, 2) == 0) else False
def clear_thread_cpu(t, c):
r = do_something_realy_non_trivial("clear_thread_cpu")
def new_futex():
return PIFutex()
def delete_futex(f):
r = do_something_realy_non_trivial('delete_futex')
def wait_futex(f):
r = do_something_realy_non_trivial('wait_futex')
def wake_futex(f):
r = do_something_realy_non_trivial('wake_futex')
def get_time_ns():
# import random
# return random.randint(0, 0xFFFF)
r = do_something_realy_non_trivial('get_time_ns')
return len(r)
def set_time_ns(tm):
r = do_something_realy_non_trivial("set_time")
def new_timer():
return Timer()
def delete_timer(tmr):
r = do_something_realy_non_trivial("delete_timer")
def arm_timer(tmr, entry, dur, prd, *args):
global _fake_timer_initialized
if not _fake_timer_initialized._flag:
def _fake_entry(*_args):
pass
_fake_timer_initialized._flag = True
arm_timer(tmr, _fake_entry, dur, prd, *args)
entry(*args)
def disarm_timer(tmr):
r = do_something_realy_non_trivial("arm_timer")
def new_attr():
return Attr()
def delete_attr(a):
r = do_something_realy_non_trivial("delete_attr")
def set_thread_attr(t, a):