database.py 17.8 KB
Newer Older
John Zhang's avatar
John Zhang committed
1
from rpython.flowspace.model import Constant
2
from rpython.rtyper.lltypesystem import lltype, rffi
John Zhang's avatar
John Zhang committed
3
from rpython.translator.mu import mutype
4 5
from rpython.translator.platform import platform
from rpython.tool.udir import udir
6 7
from rpython.config.translationoption import get_translation_config

John Zhang's avatar
John Zhang committed
8
import ctypes, ctypes.util
9
import os, sys, py, re
John Zhang's avatar
John Zhang committed
10 11 12

from rpython.tool.ansi_mandelbrot import Driver
from rpython.tool.ansi_print import AnsiLogger
13
log = AnsiLogger("MuDatabase")
John Zhang's avatar
John Zhang committed
14 15 16 17 18 19 20
mdb = Driver()


class MuDatabase:
    def __init__(self, tlc):
        # type: (rpython.translator.translator.TranslationContext) -> None
        self.tlc = tlc
21
        self.graphs = tlc.graphs
John Zhang's avatar
John Zhang committed
22 23 24 25 26 27
        self.types = set()
        self.consts = set()
        self.funcref_consts = set()
        self.gcells = set()
        self.extern_fncs = set()
        self.objtracer = None
28
        self.libsupport_path = None
29
        self.merged_eci = None
John Zhang's avatar
John Zhang committed
30
        self.mu_name_map = {}
31 32
        self.nman = None
        self.heap_NULL_constant_map = {}
John Zhang's avatar
John Zhang committed
33 34 35 36 37 38 39 40 41 42 43 44 45

    def build_database(self):
        """
        Tasks to be done at database stage:
        - lower the debug operations to mu_ccalls
        - collect all global definitions
            - types
            - constants
            - external functions
            - global cells
            - graphs & function references
        - assign a Mu name to each global entity and local variable
        - process external C functions
John Zhang's avatar
John Zhang committed
46 47 48 49
            - compiling C function macros,
                        C function declared in post_include_bits,
                        and C functions defined in PyPy C backend into one shared library
            - update the ecis, and rename function names
John Zhang's avatar
John Zhang committed
50 51
        - trace heap objects
        """
52
        # TODO: lower debug operations
John Zhang's avatar
John Zhang committed
53
        self.collect_global_defs()
John Zhang's avatar
John Zhang committed
54
        self.compile_pypy_c_extern_funcs()
John Zhang's avatar
John Zhang committed
55
        self.assign_mu_name()
John Zhang's avatar
John Zhang committed
56 57

    def collect_global_defs(self):
John Zhang's avatar
John Zhang committed
58
        # collect global definitions in graphs
59 60
        for graph in self.graphs:
            self._add_type(graph.sig)
John Zhang's avatar
John Zhang committed
61 62 63 64

            for blk in graph.iterblocks():
                for a in blk.inputargs:
                    self._add_type(a.concretetype)
65 66 67
                if blk.mu_excparam:
                    self._add_type(blk.mu_excparam.concretetype)

John Zhang's avatar
John Zhang committed
68 69 70 71 72 73 74 75 76 77 78 79 80 81
                for op in blk.operations:
                    for a in op.args:
                        self._add_type(a.concretetype)
                        if isinstance(a, Constant):
                            self._collect_constant(a)
                    self._add_type(op.result.concretetype)
                for lnk in blk.exits:
                    for a in lnk.args:
                        self._add_type(a.concretetype)
                        if isinstance(a, Constant):
                            self._collect_constant(a)
                    if isinstance(lnk.exitcase, Constant):
                        self._collect_constant(lnk.exitcase)

John Zhang's avatar
John Zhang committed
82 83 84 85 86 87 88 89 90 91
        # trace heap objects
        self.objtracer = HeapObjectTracer()

        for gcl in self.gcells:
            self.objtracer.trace(gcl.value._load())

        # add types in heap to global type definitions
        for t in self.objtracer.types_in_heap():
            self._add_type(t)

92 93 94 95 96
        # create constants for NULLs found in heap objects
        for T in self.objtracer.nullref_ts:
            c = Constant(T._null(), T)
            self.consts.add(c)
            self.heap_NULL_constant_map[T] = c
97

John Zhang's avatar
John Zhang committed
98 99 100 101 102
    def _collect_constant(self, c):
        if isinstance(c.concretetype, mutype.MuNumber):
            self.consts.add(c)
        elif isinstance(c.concretetype, mutype.MuGlobalCell):
            self.gcells.add(c)
John Zhang's avatar
John Zhang committed
103 104
        elif isinstance(c.concretetype, mutype.MuReferenceType) and c.value._is_null():
            self.consts.add(c)
John Zhang's avatar
John Zhang committed
105 106 107 108 109 110 111 112 113 114 115
        elif isinstance(c.concretetype, mutype.MuUFuncPtr):
            self.extern_fncs.add(c)
        elif isinstance(c.concretetype, mutype.MuFuncRef):
            self.funcref_consts.add(c)

    def _add_type(self, T):
        assert isinstance(T, mutype.MuType)

        if isinstance(T, mutype.MuGlobalCell):
            T = T.TO

116 117 118 119
        if T in self.types:
            return

        self.types.add(T)
John Zhang's avatar
John Zhang committed
120 121 122 123 124 125 126 127 128 129 130 131

        if isinstance(T, mutype.MuStruct):
            for FLD in tuple(getattr(T, fld) for fld in T._names):
                self._add_type(FLD)
        elif isinstance(T, mutype.MuHybrid):
            for FLD in tuple(getattr(T, fld) for fld in T._names[:-1]):
                self._add_type(FLD)
            self._add_type(T._vartype.OF)
        elif isinstance(T, mutype.MuArray):
            self._add_type(T.OF)
        elif isinstance(T, mutype.MuObjectRef):
            self._add_type(T.TO)
132 133
            if T == mutype.MU_WEAKREF_VOID:
                self._add_type(mutype.MuRef(mutype.MU_VOID))    # add ref<void> so help to declare NULL
John Zhang's avatar
John Zhang committed
134 135 136 137 138 139
        elif isinstance(T, mutype.MuGeneralFunctionReference):
            self._add_type(T.Sig)
        elif isinstance(T, mutype.MuFuncSig):
            ts = T.ARGS + T.RESULTS
            for t in ts:
                self._add_type(t)
140 141 142 143

    def compile_pypy_c_extern_funcs(self):
        all_ecis = []
        replace_ecis = []
144
        header_file_name = 'common_header.h'
145 146
        header_file_dir_path = udir.strpath

147
        macro_wrapper_regex = re.compile(r'RPY_EXTERN (?P<ret_t>.*) '
148
                                         r'\*?pypy_macro_wrapper_(?P<macro_name>[\w_]*)'
149
                                         r'\((?P<arg_list>.*)\) '
150
                                         r'\{ *(return)? *(?P=macro_name)\(.*\); \}')
151

152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
        # step 1: identify macros, functions defined in post_include_bits
        for c in self.extern_fncs:
            fnp = c.value
            eci = fnp.eci

            if eci.post_include_bits:
                if any(fnp._name in s for s in eci.post_include_bits):      # C function declaration
                    # wrap in a macro (renaming will be done below)
                    # rely on clang -O3 optimisation to inline
                    _macro_fnp = rffi.llexternal(fnp._name, fnp._llfnctype.ARGS, fnp._llfnctype.RESULT,
                                                 compilation_info=eci,
                                                 macro=True, _nowrapper=True)
                    eci = _macro_fnp._obj.compilation_info

            if hasattr(eci, '_with_ctypes'):
                # function in the same module with macros (same eci)
                eci = eci._with_ctypes
                fnp.eci = eci   # reassign the eci
                for src_str in eci.separate_module_sources:
171 172 173 174 175 176 177
                    m = re.match(macro_wrapper_regex, src_str)
                    if m:
                        d = m.groupdict()
                        if fnp._name == d['macro_name']:    # is a macro
                            fnp._name = 'pypy_macro_wrapper_' + fnp._name
                            break
                    else:
178 179
                        # print src_str
                        pass
180 181 182 183 184 185

            if eci.post_include_bits or eci.separate_module_sources or eci.separate_module_files:
                replace_ecis.append(fnp)

            all_ecis.append(eci)

186 187 188 189 190 191
        # HACK: define LL_stack_set_length_fraction to be an empty function
        hack_eci = rffi.ExternalCompilationInfo(
            separate_module_sources=['RPY_EXTERN void LL_stack_set_length_fraction(double fraction) {}']
        )
        all_ecis.append(hack_eci)

192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208
        pypy_include_dir = py.path.local(__file__).join('..', '..', 'c')
        eci = rffi.ExternalCompilationInfo(include_dirs=[pypy_include_dir.strpath])
        eci = eci.merge(*all_ecis).convert_sources_to_files()

        # step 2: convert all separate module sources to files
        header_file = udir.join(header_file_name)
        with header_file.open('w') as fp:
            fp.write('#ifndef _PY_COMMON_HEADER_H\n#define _PY_COMMON_HEADER_H\n')
            eci.write_c_header(fp)
            fp.write('#include "src/g_prerequisite.h"\n')
            fp.write('#endif /* _PY_COMMON_HEADER_H*/\n')

        # add common header to eci
        eci.post_include_bits = ()  # should have been written to mu_common_header.h
        eci = eci.merge(rffi.ExternalCompilationInfo(includes=[header_file_name], include_dirs=[header_file_dir_path]))

        # step 3: compile these files into shared library
John Zhang's avatar
John Zhang committed
209
        eci = eci.compile_shared_lib(platform.so_prefixes[0] + 'pypy_mu_support',
210 211 212 213 214 215 216
                                     debug_mode=False,      # no '-g -O0'
                                     defines=['RPY_EXTERN=RPY_EXPORTED'])

        # step 4: update eci to include the compiled shared library
        for fnp in replace_ecis:
            fnp.eci = eci

217
        if eci.libraries:
218 219
            # NOTE: move compiled library to be under current dir
            libpath = py.path.local(eci.libraries[-1])
220 221
            suplibdir = get_translation_config().translation.mu.suplibdir
            targetpath = py.path.local(suplibdir).join(libpath.basename)
222 223 224
            libpath.move(targetpath)
            self.libsupport_path = targetpath
            eci.libraries = eci.libraries[:-1] + (targetpath.strpath, )
225

226
        self.merged_eci = eci
227
        self.extern_fncs = set(list(self.extern_fncs))  # rehash all
228
        return eci
John Zhang's avatar
John Zhang committed
229

John Zhang's avatar
John Zhang committed
230 231
    def assign_mu_name(self):
        man = MuNameManager()
232
        self.nman = man
John Zhang's avatar
John Zhang committed
233 234 235 236 237 238 239 240 241
        # types
        for T in self.types:
            self.mu_name_map[T] = man.assign(T)

        # constants
        for c in self.consts:
            self.mu_name_map[c] = man.assign(c)

        for c in self.extern_fncs:
242
            self.mu_name_map[c] = man.assign(c)
John Zhang's avatar
John Zhang committed
243 244 245 246 247 248 249

        # global cells
        for c in self.gcells:
            self.mu_name_map[c] = man.assign(c)

        # graphs
        for g in self.tlc.graphs:
John Zhang's avatar
John Zhang committed
250 251
            g.name = man.get_graph_name(g)
            graph_name = g.name
John Zhang's avatar
John Zhang committed
252 253 254 255 256 257 258
            self.mu_name_map[g] = graph_name
            for i, blk in enumerate(g.iterblocks()):
                blk_name = '%(graph_name)s.blk%(i)d' % locals()
                self.mu_name_map[blk] = blk_name

                for v in blk.inputargs:
                    self.mu_name_map[v] = '%(blk_name)s.%(v)s' % locals()
259 260 261
                if blk.mu_excparam:
                    ep = blk.mu_excparam
                    self.mu_name_map[ep] = '%(blk_name)s.%(ep)s' % locals()
John Zhang's avatar
John Zhang committed
262 263 264
                for op in blk.operations:
                    res = op.result
                    self.mu_name_map[res] = '%(blk_name)s.%(res)s' % locals()
265 266
                    if op.opname == 'mu_binop':
                        metainfo = op.args[-1].value
267
                        if 'status' in metainfo:
268 269
                            for v in metainfo['status'][1]:
                                self.mu_name_map[v] = '%(blk_name)s.%(v)s' % locals()
John Zhang's avatar
John Zhang committed
270

John Zhang's avatar
John Zhang committed
271 272 273

class HeapObjectTracer:
    def __init__(self):
274 275
        self.heap_objs = set()
        self.fixed_objs = set()  # objects pointed to by uptr, needs relocation support
John Zhang's avatar
John Zhang committed
276 277 278 279 280 281 282 283
        self.nullref_ts = set()
        self.types = set()

    def trace(self, obj):
        MuT = mutype.mutypeOf(obj)
        if not isinstance(MuT, mutype._MuMemArray):
            self.types.add(MuT)

284
        if isinstance(MuT, mutype.MuReferenceType) and obj._is_null():
285 286
            self.nullref_ts.add(mutype.mutypeOf(obj))
            return
John Zhang's avatar
John Zhang committed
287

288
        if isinstance(MuT, mutype.MuObjectRef):
289 290
            self.check_reference_assumptions(obj)

John Zhang's avatar
John Zhang committed
291
            refnt = obj._obj() if isinstance(MuT, mutype.MuWeakRef) else obj._obj
292
            if isinstance(mutype.mutypeOf(refnt), mutype.MuStruct):
John Zhang's avatar
John Zhang committed
293 294
                refnt = refnt._normalizedcontainer()

295 296 297 298
            obj_set = self.fixed_objs if isinstance(MuT, mutype.MuUPtr) else self.heap_objs
            if refnt not in obj_set:
                obj_set.add(refnt)
                self.trace(refnt)
John Zhang's avatar
John Zhang committed
299

300 301
        elif isinstance(MuT, (mutype.MuStruct, mutype.MuHybrid)):
            for fld in MuT._flds:
John Zhang's avatar
John Zhang committed
302 303
                self.trace(getattr(obj, fld))

304 305
        elif isinstance(MuT, (mutype._MuMemArray, mutype.MuArray)):
            if isinstance(MuT.OF, (mutype.MuContainerType, mutype.MuObjectRef)):
John Zhang's avatar
John Zhang committed
306 307 308 309
                for i in range(len(obj.items)):
                    itm = obj[i]
                    self.trace(itm)

310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
    def check_reference_assumptions(self, ref):
        obj = ref._obj
        norm_obj = obj
        if isinstance(obj, mutype._mustruct):
            norm_obj = obj._normalizedcontainer()

        # assumption 1: all references should be to the beginning
        # of containers, not in the guts
        if isinstance(norm_obj, mutype._muparentable):
            assert norm_obj._parentstructure() is None

        # assumption 2: if ref is uptr,
        # then the outer container must not be pointed by a ref
        if isinstance(ref, mutype._muuptr):
            # TODO: how to check this?
            pass

John Zhang's avatar
John Zhang committed
327
    def types_in_heap(self):
John Zhang's avatar
John Zhang committed
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
        return self.types


class MuNameManager:
    def __init__(self):
        self.name_map = {
            mutype.MU_FLOAT: '@flt',
            mutype.MU_DOUBLE: '@dbl',
            mutype.MU_VOID: '@void',
        }
        self._counter = {
            'stt': 0,
            'hyb': 0,
            'arr': 0,
            'gcl': 0,
        }
344
        self._assigned_names = set()
345
        self._conflict_ctr = {}
John Zhang's avatar
John Zhang committed
346
        self.graph_name_dic = {}
John Zhang's avatar
John Zhang committed
347

John Zhang's avatar
John Zhang committed
348 349 350 351
        # verbose struct name?
        from rpython.config.translationoption import get_translation_config
        config = get_translation_config()
        if config:
352
            self.verbose_type_name = config.translation.mu.verbtypename
John Zhang's avatar
John Zhang committed
353

John Zhang's avatar
John Zhang committed
354 355
    def assign(self, obj):
        if isinstance(obj, mutype.MuType):
356 357 358 359 360
            name = self.get_type_name(obj)
        else:
            assert isinstance(obj, Constant)
            name = self.get_const_name(obj)

361 362 363
            if name in self._assigned_names:
                log.WARNING('name conflict: already assigned %(name)s' % locals())
                n = self._conflict_ctr.get(name, 2)
364
                self._conflict_ctr[name] = n + 1
365 366 367
                name = "%(name)s_%(n)d" % locals()
                log.WARNING('rename to %(name)s.' % locals())

368 369 370
        self._assigned_names.add(name)

        return name
John Zhang's avatar
John Zhang committed
371 372 373 374 375 376 377 378 379

    def get_type_name(self, MuT):
        if MuT in self.name_map:
            return self.name_map[MuT]

        if isinstance(MuT, mutype.MuIntType):
            name = 'i%d' % MuT.BITS

        if isinstance(MuT, mutype.MuStruct):
380
            if getattr(self, 'verbose_type_name', False):
John Zhang's avatar
John Zhang committed
381 382 383
                name = 'stt' + MuT._name
                if name in self._counter:
                    self._counter[name] += 1
John Zhang's avatar
John Zhang committed
384
                    name = name + str(self._counter[name])
John Zhang's avatar
John Zhang committed
385
                else:
John Zhang's avatar
John Zhang committed
386
                    self._counter[name] = 1
John Zhang's avatar
John Zhang committed
387 388 389
            else:
                name = 'stt%d' % self._counter['stt']
                self._counter['stt'] += 1
John Zhang's avatar
John Zhang committed
390 391

        if isinstance(MuT, mutype.MuHybrid):
392 393 394 395
            if getattr(self, 'verbose_type_name', False):
                name = 'hyb' + MuT._name
                if name in self._counter:
                    self._counter[name] += 1
John Zhang's avatar
John Zhang committed
396
                    name = name + str(self._counter[name])
397
                else:
John Zhang's avatar
John Zhang committed
398
                    self._counter[name] = 1
399 400 401
            else:
                name = 'hyb%d' % self._counter['hyb']
                self._counter['hyb'] += 1
John Zhang's avatar
John Zhang committed
402 403

        if isinstance(MuT, mutype.MuArray):
404 405 406 407
            if getattr(self, 'verbose_type_name', False):
                name = 'arr%d%s' % (MuT.length, self.get_type_name(MuT.OF)[1:])
                if name in self._counter:
                    self._counter[name] += 1
John Zhang's avatar
John Zhang committed
408
                    name = name + str(self._counter[name])
409
                else:
John Zhang's avatar
John Zhang committed
410
                    self._counter[name] = 1
411 412 413
            else:
                name = 'arr%d' % self._counter['arr']
                self._counter['arr'] += 1
John Zhang's avatar
John Zhang committed
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449

        if isinstance(MuT, mutype.MuReferenceType):
            prefix_map = {
                mutype.MuRef: 'ref',
                mutype.MuIRef: 'irf',
                mutype.MuUPtr: 'ptr',
                mutype.MuWeakRef: 'wrf',
                mutype.MuFuncRef: 'fnr',
                mutype.MuUFuncPtr: 'fnp',
                mutype.MuOpaqueRef: 'opqr'
            }
            prefix = prefix_map[type(MuT)]
            if isinstance(MuT, mutype.MuObjectRef):
                refnt = self.get_type_name(MuT.TO)[1:]
                name = prefix + refnt
            elif isinstance(MuT, mutype.MuGeneralFunctionReference):
                sig = self.get_type_name(MuT.Sig)[1:]
                name = prefix + sig
            elif isinstance(MuT, mutype.MuOpaqueRef):
                name = prefix + MuT.obj_name

        if isinstance(MuT, mutype.MuFuncSig):
            name = 'sig_%(args)s_%(rets)s' % {
                'args': ''.join([self.get_type_name(T)[1:] for T in MuT.ARGS]),
                'rets': ''.join([self.get_type_name(T)[1:] for T in MuT.RESULTS])
            }

        name = '@' + name
        self.name_map[MuT] = name
        return name

    def get_const_name(self, const):
        if isinstance(const.concretetype, mutype.MuGlobalCell):
            name = 'gcl%d' % self._counter['gcl']
            self._counter['gcl'] += 1
        elif isinstance(const.concretetype, mutype.MuReferenceType) and const.value._is_null():
450
            name = 'NULL_%s' % self.get_type_name(const.concretetype)[1:]
John Zhang's avatar
John Zhang committed
451 452 453
        elif isinstance(const.concretetype, mutype.MuNumber):
            name = '%(hex)s_%(type)s' % {'hex': mutype.hex_repr(const.value),
                                         'type': self.get_type_name(const.concretetype)[1:]}
454 455 456
        elif isinstance(const.concretetype, mutype.MuUFuncPtr):
            name = 'extfnc_' + const.value._name

John Zhang's avatar
John Zhang committed
457
        return '@' + name
John Zhang's avatar
John Zhang committed
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473

    def get_graph_name(self, g):
        name = g.name if '.' in g.name else g.name.split('__')[0]
        name = re.sub(r'[^0-9a-zA-Z_-]', '_', name)     # replace illegal characters in names
        name_dic = self.graph_name_dic

        if name not in name_dic:
            ctr = 0
            name_dic[name] = ([g], ctr)
        else:
            gs, ctr = name_dic[name]
            if g not in gs:
                gs.append(g)
                ctr += 1
                name_dic[name] = (gs, ctr)
        return "@%s_%d" % (name, ctr)