perfcmp.py 9.73 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14
# Copyright 2017 The Australian National University
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15 16 17 18 19
"""
Performance comparison
"""
from time import time
from tempfile import mkdtemp
20
import py, os, sys
21 22
import subprocess as subp
import ctypes
23
import math
24

25
from rpython.rtyper.lltypesystem import lltype, rffi
26
from util import libext, preload_libmu, fncptr_from_py_script, fncptr_from_rpy_func
27

28 29 30
perf_target_dir = py.path.local(__file__).dirpath().join('perftarget')


31 32 33
CPYTHON = os.environ.get('CPYTHON', 'python')
PYPY = os.environ.get('PYPY', 'pypy')
RPYTHON = os.environ.get('RPYTHON', None)
John Zhang's avatar
John Zhang committed
34
CC = os.environ.get('CC', 'clang')
35

36

37
def run(cmd):
38
    print ' '.join(cmd)
39 40 41
    p = subp.Popen(cmd, stdout=subp.PIPE, stderr=subp.PIPE)
    return p.communicate()

42

43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
def run_cpython(config):
    py_file = config['py_file']
    out, _ = run([CPYTHON, py_file.strpath] + map(str, config['setup_args']))
    return float(out)


def run_pypy_nojit(config):
    py_file = config['py_file']
    out, _ = run([PYPY, '--jit', 'off', py_file.strpath] + map(str, config['setup_args']))
    return float(out)


def run_pypy(config):
    py_file = config['py_file']
    out, _ = run([PYPY, py_file.strpath] + map(str, config['setup_args']))
    return float(out)

60 61 62 63 64 65 66 67 68 69
rpy_wrapper = \
"""
def rpy_measure_%(name)s_%(target)s(%(args)s):
    from time import time
    t0 = time()
    %(rpy_fnc)s(%(args)s)
    t1 = time()
    return t1 - t0
"""

70

71
def wrap_with_measure_func(fnp, config, target):
72
    wrapper_config = {
73
        'name': config['rpy_fnc'].__name__,
74
        'target': target,
75
        'rpy_fnc': 'fnp',
76 77 78
        'args': ', '.join(['v%d' % i for i in range(len(config['llarg_ts']))])
    }
    tl_config = {'gc': 'none'}
79

80 81 82
    wrapper = rpy_wrapper % wrapper_config
    exec wrapper in locals()
    rpy_measure_fnc = locals()['rpy_measure_%(name)s_%(target)s' % wrapper_config]
83 84
    fnp, _ = fncptr_from_rpy_func(rpy_measure_fnc, config['llarg_ts'], rffi.DOUBLE,
                                  backend='c', gc='none')
85 86 87
    return fnp


88
def compile_rpython_c(config):
89 90
    print '\n\n'
    print '\033[33;1m------------------------------------- rpy_c -------------------------------------\033[0m'
91 92 93 94
    rpy_fnc = config['rpy_fnc']
    return wrap_with_measure_func(rpy_fnc, config, 'rpy_c')


95
def compile_rpython_mu(config):
96 97
    print '\n\n'
    print '\033[33;1m------------------------------------- rpy_mu -------------------------------------\033[0m'
98
    preload_libmu()
99

100 101
    fnp, _ = fncptr_from_rpy_func(config['rpy_fnc'], config['llarg_ts'], config['llres_t'],
                                  muemitdir=config['tmpdir'].strpath)
102

103
    return wrap_with_measure_func(fnp, config, 'rpy_mu')
104 105


106
def compile_mu(config):
107 108 109 110
    print '\n\n'
    print '\033[33;1m------------------------------------- mu -------------------------------------\033[0m'
    fnp, _ = fncptr_from_py_script(config['mu_build_fnc'], None, 'quicksort', config['llarg_ts'], config['llres_t'],
                                   muemitdir=config['tmpdir'].strpath)
111
    return wrap_with_measure_func(fnp, config, 'mu')
112 113


114
def compile_c(config):
115 116
    print '\n\n'
    print '\033[33;1m------------------------------------- c -------------------------------------\033[0m'
117 118 119 120 121 122 123
    c_fnc = rffi.llexternal(config['c_sym_name'], config['llarg_ts'], config['llres_t'],
                            compilation_info=rffi.ExternalCompilationInfo(
                                includes=['quicksort.h'],
                                include_dirs=[perf_target_dir.strpath],
                                separate_module_sources=['#include "quicksort.c"']
                            ), _nowrapper=True)

124
    return wrap_with_measure_func(c_fnc, config, 'c')
125 126


127
def get_stat(run_fnc, config, iterations=100):
128 129 130 131 132 133 134 135 136 137 138 139 140
    times = []
    for i in range(iterations):
        times.append(run_fnc(config))

    times.sort()
    avg = sum(times) / float(len(times))
    t_min = t_max = t_std = None
    if len(times) > 1:
        t_min = times[0]
        t_max = times[-1]
        squares = ((t - avg) ** 2 for t in times)
        t_std = math.sqrt(sum(squares) / (len(times) - 1))

John Zhang's avatar
John Zhang committed
141
    return {'average': avg, 't_min': t_min, 't_max': t_max, 'std_dev': t_std, 'data': times}
142 143


144
def get_stat_compiled(compile_fnc, config, iterations=100):
145 146
    def run_funcptr(fnp, config):
        args = config['setup'](*config['setup_args'])
John Zhang's avatar
John Zhang committed
147
        t = fnp(*args)
148
        config['teardown'](*args)
John Zhang's avatar
John Zhang committed
149
        return t
150 151

    fnp = compile_fnc(config)
152
    return get_stat(lambda config: run_funcptr(fnp, config), config, iterations)
153 154 155


def get_display_str(stat):
156 157 158 159 160 161
    output = "average: {average:.6f}\n" \
             "min: {t_min:.6f}\n" \
             "max: {t_max:.6f}\n" \
             "std_dev: {std_dev:.6f}\n" \
             "slowdown: {slowdown:.3f}x\n"
    return output.format(**stat)
162

163

164
def perf(config, iterations):
165
    results = {
166 167 168
        # 'cpython': get_stat(run_cpython, config, iterations=iterations),
        # 'pypy_nojit': get_stat(run_pypy_nojit, config, iterations=iterations),
        # 'pypy': get_stat(run_pypy, config, iterations=iterations),
169 170 171
        'rpy_c': get_stat_compiled(compile_rpython_c, config, iterations=iterations),
        'rpy_mu': get_stat_compiled(compile_rpython_mu, config, iterations=iterations),
        'c': get_stat_compiled(compile_c, config, iterations=iterations),
172
    }
173 174
    if config['mu_build_fnc']:
        results['mu'] = get_stat_compiled(compile_mu, config, iterations)
175

176 177 178 179
    baseline_target = 'c'
    for python, result in results.items():
        result['slowdown'] = result['average'] / results[baseline_target]['average']

180 181 182
    for python, result in results.items():
        print '\033[35m---- %(python)s ----\033[0m' % locals()
        print get_display_str(result)
183

184
    return results
185

186

187
def save_results(test_name, results, tmpdir):
188
    import json
189
    json_file_path = tmpdir.join('result_%(test_name)s.json' % locals())
190 191 192 193

    with json_file_path.open('w') as fp:
        json.dump(results, fp, indent=4, separators=(',', ':'))

194

195
def perf_fibonacci(N, iterations):
196
    from perftarget.fibonacci import fib, rpy_entry
197 198 199
    tmpdir = py.path.local(mkdtemp())
    print tmpdir

200 201 202
    config = {
        'py_file': perf_target_dir.join('fibonacci.py'),
        'c_file': perf_target_dir.join('fibonacci.c'),
203
        'rpy_fnc': rpy_entry,
204
        'c_sym_name': 'fib',
205 206
        'llarg_ts': [lltype.Signed],
        'llres_t': lltype.Signed,
207 208 209 210 211 212
        'setup_args': (N,),
        'setup': lambda N: (N, ),
        'teardown': lambda N: None,
        'libpath_mu': tmpdir.join('libfibonacci_mu.dylib'),
        'libpath_c': tmpdir.join('libfibonacci_c.dylib')
    }
213

214
    results = perf(config, iterations)
215 216 217
    results['test_name'] = 'fibonacci'
    results['input_size'] = N
    results['iterations'] = iterations
218
    return results
219 220


221
def perf_arraysum(N, iterations):
222 223 224 225 226 227 228 229 230 231
    from perftarget.arraysum import arraysum, setup, teardown
    tmpdir = py.path.local(mkdtemp())
    print tmpdir

    config = {
        'py_file': perf_target_dir.join('arraysum.py'),
        'c_file': perf_target_dir.join('arraysum.c'),
        'rpy_fnc': arraysum,
        'c_sym_name': 'arraysum',
        'llarg_ts': [rffi.CArrayPtr(rffi.LONGLONG), rffi.SIZE_T],
232
        'llres_t': rffi.LONGLONG,
233 234 235 236 237
        'setup_args': (N, ),
        'setup': setup,
        'teardown': teardown,
        'libpath_mu': tmpdir.join('libfibonacci_mu.dylib'),
        'libpath_c': tmpdir.join('libfibonacci_c.dylib')
238
    }
239

240
    results = perf(config, iterations)
241 242 243
    results['test_name'] = 'arraysum'
    results['input_size'] = N
    results['iterations'] = iterations
244 245 246
    return results


247 248 249 250 251 252 253 254 255 256 257
def get_tmpdir(testname, problemsize, iterations):
    from time import asctime
    timestamp = asctime().replace(' ', '-').replace(':', '')
    return py.path.local("/tmp/%s-%s-%d-%d" % (timestamp, testname, problemsize, iterations))


def move_pypy_udir(tmpdir):
    from rpython.tool.udir import udir
    run(['mv', udir.strpath, tmpdir.join('pypy_udir').strpath])


258
def perf_quicksort(N, iterations):
259
    from perftarget.quicksort import quicksort, build_quicksort_bundle, setup, teardown
260 261 262
    tmpdir = get_tmpdir('quicksort', N, iterations)
    tmpdir.mkdir()

263
    config = {
264
        'tmpdir': tmpdir,
265 266 267 268
        'py_file': perf_target_dir.join('quicksort.py'),
        'c_file': perf_target_dir.join('quicksort.c'),
        'rpy_fnc': quicksort,
        'c_sym_name': 'quicksort',
269
        'mu_build_fnc': build_quicksort_bundle,
270 271 272 273 274
        'llarg_ts': [rffi.CArrayPtr(rffi.LONGLONG), lltype.Signed, lltype.Signed],
        'llres_t': lltype.Void,
        'setup_args': (N,),
        'setup': setup,
        'teardown': teardown,
275 276
        'libpath_mu': tmpdir.join('libquicksort_mu' + libext),
        'libpath_c': tmpdir.join('libquicksort_c' + libext)
277 278 279
    }

    results = perf(config, iterations)
280 281 282
    results['test_name'] = 'quicksort'
    results['input_size'] = N
    results['iterations'] = iterations
283

284
    save_results('quicksort', results, tmpdir)
285

286
    move_pypy_udir(tmpdir)
287 288 289 290
    return results


def test_functional_fibonacci():
291
    perf_fibonacci(5, 1)
292 293 294


def test_functional_arraysum():
295
    perf_arraysum(100, 1)
296 297 298


def test_functional_quicksort():
299
    perf_quicksort(100, 5)
300

301
if __name__ == '__main__':
302 303
    import sys
    N = int(sys.argv[1])
304 305 306 307
    # fib_res = perf_fibonacci(40, 20)
    # save_results('fibonacci', fib_res)
    # arraysum_res = perf_arraysum(1000000, 20)
    # save_results('arraysum', arraysum_res)
308
    quicksort_res = perf_quicksort(N, 100)