wip: storing an exposed Mu function and CCALL it from within Mu.

Most of the work has been done. test_genmu.test_fakeaddress_fncptr gives
a possible use case.
The back-end now can store a ufuncptr as a result of statically exposing
a Mu function; when it loads it back at run time, calling it is done
through CCALL.
The test fails because reference implementation still doesn't support
this.

rpython/translator/mu/database.py

@@ -29,6 +29,9 @@ class MuDatabase:
         self.nman = None
         self.heap_NULL_constant_map = {}
 
+        # other
+        self.default_cookie_const = Constant(mutype.mu_int64(0), mutype.MU_INT64)  # default cookie for exposed funcs
+
     def build_database(self):
         """
         Tasks to be done at database stage:
@@ -53,6 +56,10 @@ class MuDatabase:
         self.assign_mu_name()
 
     def collect_global_defs(self):
+        # predefined stuff
+        predef_consts = {self.default_cookie_const}
+        self.consts.update(predef_consts)
+
         # collect global definitions in graphs
         for graph in self.graphs:
             self._add_type(graph.sig)
@@ -242,6 +249,10 @@ class MuDatabase:
         for c in self.gcells:
             self.mu_name_map[c] = man.assign(c)
 
+        # exposed functions
+        for g, c in self.objtracer.exposed_func_graph_to_const.items():
+            self.mu_name_map[c] = '@exp_' + g.name
+
         # graphs
         for g in self.tlc.graphs:
             graph_name = '@' + g.name
@@ -271,6 +282,7 @@ class HeapObjectTracer:
         self.fixed_objs = set()  # objects pointed to by uptr, needs relocation support
         self.nullref_ts = set()
         self.types = set()
+        self.exposed_func_graph_to_const = {}
 
     def trace(self, obj):
         MuT = mutype.mutypeOf(obj)
@@ -281,6 +293,9 @@ class HeapObjectTracer:
             self.nullref_ts.add(mutype.mutypeOf(obj))
             return
 
+        if isinstance(MuT, mutype.MuUFuncPtr) and isinstance(obj, mutype._muexposedfunc):
+            self.exposed_func_graph_to_const[obj.graph] = Constant(obj, MuT)    # make into constant to be able to hash
+
         if isinstance(MuT, mutype.MuObjectRef):
             self.check_reference_assumptions(obj)
 

rpython/translator/mu/genmu.py

@@ -281,6 +281,10 @@ class MuBundleGen:
         for c in self.db.extern_fncs:
             self.bdr.new_const_extern(self._id_of(c), self._id_of(c.concretetype), c.value._name)
 
+        for g, c in self.db.objtracer.exposed_func_graph_to_const.items():
+            self.bdr.new_exp_func(self._id_of(c), self._id_of(g), self.rmu.MuCallConv.DEFAULT,
+                                  self._id_of(self.db.default_cookie_const))
+
     def gen_gcells(self):
         self.log.gen_gcells("declaring %d global cells..." % (len(self.db.gcells)))
         for gcl in self.db.gcells:
@@ -644,6 +648,9 @@ class MuBundleGen:
             elif isinstance(MuT, mutype.MuFuncRef):
                 return self.ctx.handle_from_func(self._id_of(obj.graph))
 
+            elif isinstance(MuT, mutype.MuUFuncPtr) and isinstance(obj, mutype._muexposedfunc):
+                return self.ctx.handle_from_expose(self._id_of(self.db.objtracer.exposed_func_graph_to_const[obj.graph]))
+
         raise NotImplementedError("Don't know how to initialise heap object %s of type %s" % (obj, MuT))
 
     def _init_heap_struct_like(self, obj, hiref, names=None):

rpython/translator/mu/ll2mu.py

@@ -371,8 +371,18 @@ class LL2MuMapper:
             else:
                 raise NotImplementedError("Unanticipted %s" % llv)
         elif isinstance(llv, llmemory.fakeaddress):
-            assert not llv  # must be NULL
-            return MuT._null()
+            if not llv:     # NULL
+                return MuT._null()
+            ptr = llv.ptr
+            LLT = lltype.typeOf(ptr)
+            MuT = self.map_type(LLT)
+            if isinstance(LLT.TO, lltype.FuncType):     # address of function pointer
+                MuT = mutype.MuUFuncPtr(MuT.Sig)
+                fnc = ptr._obj
+                if hasattr(fnc, 'graph'):   # exposed Mu function
+                    return mutype._muexposedfunc(MuT, graph=fnc.graph, _name=getattr(fnc, '_name', ''))
+                else:   # external C function
+                    return mutype._muexternfunc(MuT, _name=fnc._name, eci=fnc.compilation_info, _llfnctype=LLT.TO)
 
         return MuT._val_type(llv)
 
@@ -525,7 +535,7 @@ class LL2MuMapper:
             Sig = MuT.Sig
             MuT = mutype.MuUFuncPtr(Sig)
             c_name = fnc._name
-            return mutype._muufuncptr(MuT, _name=c_name, eci=fnc.compilation_info, _llfnctype=LLT.TO)
+            return mutype._muexternfunc(MuT, _name=c_name, eci=fnc.compilation_info, _llfnctype=LLT.TO)
 
     def map_value_adrofs(self, llv):
         def rec(llv):
@@ -654,7 +664,7 @@ class LL2MuMapper:
     def map_op_direct_call(self, llop):
         fr = llop.args[0].value
         args, ops = self._prepare_call_args(llop.args[1:], mutype.mutypeOf(fr).Sig)
-        mtd = self.gen_mu_ccall if isinstance(fr, mutype._muufuncptr) else self.gen_mu_call
+        mtd = self.gen_mu_ccall if isinstance(fr, (mutype._muexternfunc, mutype._muexposedfunc)) else self.gen_mu_call
         ops.append(mtd(llop.args[0], args, llop.result))
         return ops
 
@@ -665,7 +675,8 @@ class LL2MuMapper:
         else:
             args = llop.args[1:]
         call_args, ops = self._prepare_call_args(args, llop.args[0].concretetype.Sig)
-        ops.append(self.gen_mu_call(llop.args[0], call_args, llop.result))
+        mtd = self.gen_mu_call if isinstance(llop.args[0].concretetype, mutype.MuFuncRef) else self.gen_mu_ccall
+        ops.append(mtd(llop.args[0], call_args, llop.result))
         return ops
 
     # ----------------
@@ -1264,11 +1275,16 @@ class LL2MuMapper:
             else:
                 optr = 'REFCAST'
             return [self.gen_mu_convop(optr, DST, var, llop.result)]
-        else:
-            assert isinstance(SRC, mutype.MuRef) and isinstance(DST, mutype.MuUPtr)
+        elif isinstance(SRC, mutype.MuUPtr) and isinstance(DST, mutype.MuUFuncPtr):
+            assert SRC.TO in (mutype.MU_VOID, mutype.MU_INT8)
+            # allow casting uptr<void/int<8>> -> ufuncptr<sig>
+            return [self.gen_mu_convop('PTRCAST', DST, var, llop.result)]
+        elif isinstance(SRC, mutype.MuRef) and isinstance(DST, mutype.MuUPtr):
             # allowing ref -> uptr through pinning
             ops = self.map_op_cast_ptr_to_adr(SpaceOperation('cast_ptr_to_adr', [var], llop.result))
             return ops
+        else:
+            raise TypeError('Unanticipated cast: %(SRC)s -> %(DST)s' % locals())
 
     def map_op_cast_opaque_ptr(self, llop):
         llop.__init__('cast_pointer', [llop.args[0]], llop.result)
@@ -1357,8 +1373,12 @@ class LL2MuMapper:
 
         elif SRC == RES:
             return self._rename_to_same_as(llop)
+        elif SRC == self.map_type(llmemory.Address) and isinstance(RES, mutype.MuFuncRef):
+            # HACK
+            llop.result.concretetype = mutype.MuUFuncPtr(RES.Sig)
+            return self.map_op_cast_pointer(llop)
         else:
-            raise NotImplementedError("forcecast: %s -> %s" % (SRC, RES))
+            raise NotImplementedError("force_cast: %s -> %s" % (SRC, RES))
 
     map_op_cast_primitive = map_op_force_cast
 
@@ -1674,7 +1694,7 @@ class LL2MuMapper:
                               res if res else varof(mutype.MU_VOID))
 
     def gen_mu_call(self, callee, args, res=None, keepalive=None, excclause=None):
-        assert isinstance(callee.concretetype, mutype.MuFuncRef)
+        assert isinstance(callee.concretetype, (mutype.MuFuncRef, mutype.MuUFuncPtr))   # allow calling ufuncptr
         Sig = callee.concretetype.Sig
         assert len(args) == len(Sig.ARGS)
         for i, arg in enumerate(args):

rpython/translator/mu/mutype.py

@@ -437,7 +437,7 @@ class _mumemarray(_muparentable):
             self._setparentstructure(parent, parentindex)
 
     def setitem(self, index, value):
-        assert mutypeOf(value) == self._TYPE.OF
+        # assert mutypeOf(value) == self._TYPE.OF   # NOTE: commented out to relax; may need to give way to reference types
         self.items[index] = value
 
     def __getitem__(self, idx):
@@ -833,6 +833,14 @@ class _muufuncptr(_mufunction_reference):
 _setup_consistent_methods(_muufuncptr)
 
 
+class _muexternfunc(_muufuncptr):
+    pass
+
+
+class _muexposedfunc(_mufunction_reference):
+    pass
+
+
 # ----------------------------------------------------------
 class MuObjectRef(MuReferenceType):
     _val_type = property(lambda self: _muobject_reference)

rpython/translator/mu/mutyper.py

@@ -257,34 +257,42 @@ def graph_closure(g_entry):
                 refnt = refnt._normalizedcontainer()
 
             pending_objects.append(refnt)
-        else:
-            if isinstance(obj, lltype._struct):
-                if obj in visited_obj:
-                    return
-                visited_obj.add(obj)
-                fld_dic = lltype.typeOf(obj)._flds
-                for fld in fld_dic:
-                    _find_funcrefs(obj._getattr(fld))
-
-            elif isinstance(obj, lltype._array):
-                if obj in visited_obj:
-                    return
-                visited_obj.add(obj)
-                if isinstance(lltype.typeOf(obj).OF, (lltype.ContainerType, lltype.Ptr)):
-                    for i in range(len(obj.items)):
-                        itm = obj.getitem(i)
-                        _find_funcrefs(itm)
-
-            elif isinstance(obj, lltype._opaque):
-                if hasattr(obj, 'container'):
-                    _find_funcrefs(obj._normalizedcontainer())
-
-            elif isinstance(obj, llmemory._wref):
-                _find_funcrefs(obj._dereference())
-
-            elif isinstance(obj, lltype._func):
-                if hasattr(obj, 'graph'):
-                    pending_graphs.append(obj.graph)
+        elif isinstance(obj, lltype._struct):
+            if obj in visited_obj:
+                return
+            visited_obj.add(obj)
+            fld_dic = lltype.typeOf(obj)._flds
+            for fld in fld_dic:
+                _find_funcrefs(obj._getattr(fld))
+
+        elif isinstance(obj, lltype._array):
+            if obj in visited_obj:
+                return
+            visited_obj.add(obj)
+            T = lltype.typeOf(obj).OF
+            if isinstance(T, (lltype.ContainerType, lltype.Ptr)) or T == llmemory.Address:
+                for i in range(len(obj.items)):
+                    itm = obj.getitem(i)
+                    _find_funcrefs(itm)
+
+        elif isinstance(obj, lltype._opaque):
+            if hasattr(obj, 'container'):
+                _find_funcrefs(obj._normalizedcontainer())
+
+        elif isinstance(obj, llmemory._wref):
+            _find_funcrefs(obj._dereference())
+
+        elif isinstance(obj, lltype._func):
+            if hasattr(obj, 'graph'):
+                pending_graphs.append(obj.graph)
+
+        elif isinstance(obj, llmemory.fakeaddress):
+            ptr = obj.ptr
+            LLT = lltype.typeOf(ptr)
+            if isinstance(LLT.TO, lltype.FuncType):     # address of function pointer
+                fnc = ptr._obj
+                if hasattr(fnc, 'graph'):   # function address
+                    pending_graphs.append(fnc.graph)
 
     def visit(graph):
         if graph in graph_closure:

rpython/translator/mu/test/test_database.py

@@ -191,3 +191,48 @@ def test_assign_mu_name():
     n = db.mu_name_map.get(g.startblock.inputargs[0])
     assert n
     assert n.startswith('@fac_0.blk0.n')
+
+
+def test_fakeaddress_fncptr():
+    from rpython.rtyper.lltypesystem import llmemory
+    from test_mutyper import graph_of
+    from rpython.translator.platform import platform
+    def f(a, b):
+        return a + b
+    def g(a, b):
+        return a - b
+
+
+    t = Translation(f, [int, int])
+    t.rtype()
+    graph_f = graph_of(f, t)
+    t = Translation(g, [int, int])
+    t.rtype()
+    graph_g = graph_of(g, t)
+
+    fp_f = lltype.getfunctionptr(graph_f)
+    fp_g = lltype.getfunctionptr(graph_g)
+    FUNCPTR = lltype.typeOf(fp_f)
+
+    fncadrs = map(llmemory.cast_ptr_to_adr, [fp_f, fp_g])
+    def main(argv):
+        fncadr = fncadrs[int(argv[1])]
+        fncptr = rffi.cast(FUNCPTR, fncadr)
+        print fncptr(int(argv[2]), int(argv[3]))
+        return 0
+
+    t = Translation(main, None, backend='c')
+    exe = t.compile_c()
+    res = platform.execute(exe, ['0', '3', '4'])
+    assert res.out == '7\n'
+    res = platform.execute(exe, ['1', '3', '4'])
+    assert res.out == '-1\n'
+
+    # t = Translation(main, None, backend='mu')
+    # t.mutype()
+    #
+    # g_main = graph_of(main, t)
+    #
+    # db = MuDatabase(t.context)
+    # db.collect_global_defs()
+    # assert len(db.exposed_fncs) == 2

rpython/translator/mu/test/test_genmu.py

@@ -311,3 +311,61 @@ def test_tlref():
 
     assert res.returncode == 1, res.err
     # assert res.out == 'key = %d\n%s' % (k, hex(k)) + '\n'
+
+
+def test_fakeaddress_fncptr():
+    from rpython.rtyper.lltypesystem import llmemory
+    from test_mutyper import graph_of
+    from rpython.rtyper.lltypesystem import rffi
+
+    def f(a, b):
+        return a + b
+    def g(a, b):
+        return a - b
+
+
+    t = Translation(f, [int, int])
+    t.rtype()
+    graph_f = graph_of(f, t)
+    t = Translation(g, [int, int])
+    t.rtype()
+    graph_g = graph_of(g, t)
+
+    fp_f = lltype.getfunctionptr(graph_f)
+    fp_g = lltype.getfunctionptr(graph_g)
+    FUNCPTR = lltype.typeOf(fp_f)
+
+    fncadrs = map(llmemory.cast_ptr_to_adr, [fp_f, fp_g])
+    def main(argv):
+        fncadr = fncadrs[int(argv[1])]
+        fncptr = rffi.cast(FUNCPTR, fncadr)
+        print fncptr(int(argv[2]), int(argv[3]))
+        return 0
+
+    # C backend
+    # t = Translation(main, None, backend='c')
+    # exe = t.compile_c()
+    # res = platform.execute(exe, ['0', '3', '4'])
+    # assert res.out == '7\n'
+    # res = platform.execute(exe, ['1', '3', '4'])
+    # assert res.out == '-1\n'
+
+    # Mu backend
+    config = {
+        'backend': 'mu',
+        'impl': 'holstein',
+        'codegen': 'c'
+    }
+
+    t = Translation(main, None, **config)
+    t.driver.exe_name = 'test_fakeaddress_fncptr-%(backend)s'
+    t.rtype()
+    # g_main = graph_of(main, t)
+    # g_main.view()
+    t.mutype()
+    t.compile_mu()
+
+    res = holstein_run(t.driver.exe_name % config, ['0', '3', '4'])
+    assert res.out == '7\n'
+    res = holstein_run(t.driver.exe_name % config, ['-1', '3', '4'])
+    assert res.out == '-1\n'

rpython/translator/mu/test/test_ll2mu.py

@@ -727,3 +727,25 @@ def test_map_op_mu_meta_lst2carr():
     muops = ll2mu.map_op(llop)
     assert [op.opname for op in muops] == ['mu_getiref', 'mu_getfieldiref', 'mu_load',
                                            'mu_getiref', 'mu_getvarpartiref', 'mu_shiftiref']
+
+
+def test_fakeaddress_fncptr():
+    from rpython.translator.interactive import Translation
+    from test_mutyper import graph_of
+
+    def f(a, b):
+        return a + b
+    t = Translation(f, [int, int])
+    t.rtype()
+    graph_f = graph_of(f, t)
+    fncptr = lltype.getfunctionptr(graph_f)
+    fncadr = llmemory.cast_ptr_to_adr(fncptr)
+    assert isinstance(fncadr, llmemory.fakeaddress)
+
+    ll2mu = LL2MuMapper()
+    mu_fncadr = ll2mu.map_value(fncadr)
+    ll2mu.resolve_ptr_types()
+    ll2mu.resolve_ptr_values()
+    assert isinstance(mu_fncadr, mutype._muexposedfunc)
+    assert mu_fncadr.graph is graph_f
+    assert isinstance(mutype.mutypeOf(mu_fncadr), mutype.MuUFuncPtr)