Commit 25d9d0cf authored by John Zhang's avatar John Zhang

updated task configuration files to new format

parent eb0fe002
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
alloc_rpyc:
alloc:
iterations: 50
benchmark:
name: micro/alloc
iterations: 50
source: targetalloc.py
args:
- 10 # scale factor
- 1000000
result: alloc_rpyc_O3_data
language:
name: rpython
backend: c
environ:
MICROBENCH_SCALE_FACTOR: "1"
callback:
name: ClockCallback
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy
alloc_rpymu_zebu:
benchmark:
name: micro/alloc
iterations: 50
source: targetalloc.py
args:
- 1000000
result: alloc_rpyzebu_data
language:
name: rpython
backend: mu
impl: zebu
callback:
name: ClockCallback
environ:
MU_ZEBU: /Users/johnz/Documents/Work/mu-impl-fast
ZEBU_BUILD: release
MICROBENCH_SCALE_FACTOR: "1"
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy
name: clock
param: "6"
tasks:
rpyc:
language:
name: rpython
backend: c
source: targetalloc.py
zebu:
language:
name: rpython
backend: mu
impl: zebu
source: targetalloc.py
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# New format design decisions
# - putting scale factor as an argument to benchmark.
# This forces the scaling to be done at run time.
# - putting iterations outside benchmark, to be a key under task set
# - putting source as a key inside task
# - aggregating all results under one task set result file.
# - taking callback spec out to the task set level
# fields in which environs are expanded:
# - flags
# - args, vmarg
# - param
fib_python:
benchmark:
name: micro/fib # required
iterations: 10 # optional, default: 1
args: # optional, commandline arguments, default: none
- 5
source: fib.py # optional, default: $(benchmark)/$(benchmark).$(language.ext)
result: fib_py_data # optional, default: $(task)_data
language: python # required
callback: # required, can be a string or a mapping
name: ClockCallback # required, language defined. File placed in callbacks/$(language)/
param: "6" # optional, callback defined
runner: # optional, can be used to set different implementations
exec: pypy
fib_rpy_c:
benchmark:
name: micro/fib
iterations: 50
args:
- 5
source: targetfib.py
result: fib_rpyc_data
language:
name: rpython
backend: c # required
callback:
name: ClockCallback
param: "6"
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy # required by rpython
fib_rpy_mu_holstein:
benchmark:
name: micro/fib
iterations: 50
args:
- 5
source: targetfib.py
result: fib_rpymu_holstein_data
language:
name: rpython
backend: mu
impl: holstein
callback:
name: ClockCallback
param: "6"
environ: # can define environment variables that are passed to compiler and runner
PYPY_MU: /Users/johnz/Documents/Work/mu-client-pypy
MU_HOLSTEIN: /Users/johnz/Documents/Work/mu-impl-ref2
compiler:
pypy_dir: $PYPY_MU # can use environment variables in paths
flags:
- --mu-vmargs=vmLog=ERROR
runner:
flags:
- --vmLog=ERROR
fib_rpy_mu_zebu:
fib:
iterations: 5
benchmark:
name: micro/fib
iterations: 50
args:
- 10 # scale factor
- 5
source: targetfib.py
result: fib_rpymu_zebu_data_2
language:
name: rpython
backend: mu
impl: zebu
callback:
name: ClockCallback
name: clock
param: "6"
environ: # can define environment variables that are passed to compiler and runner
PYPY_MU: /Users/johnz/Documents/Work/mu-client-pypy
MU_ZEBU: /Users/johnz/Documents/Work/mu-impl-fast
ZEBU_BUILD: release
compiler:
pypy_dir: $PYPY_MU # can use environment variables in paths
fib_c:
benchmark:
name: micro/fib
iterations: 50
args:
- 5
source: fib.c
result: fib_c_data
language: c
callback:
name: clockcallback
param: "6" # precision
compiler:
flags:
- -O0
fib_mu:
benchmark:
name: micro/fib
iterations: 50
args:
- 5
source: build_fib.c
result: fib_mu_data
language:
name: mu
impl: zebu
callback:
name: clockcallback
param: "6"
environ: # can define environment variables that are passed to compiler and runner
PYPY_MU: /Users/johnz/Documents/Work/mu-client-pypy
MU_ZEBU: /Users/johnz/Documents/Work/mu-impl-fast
ZEBU_BUILD: release
compiler:
vmarg: "--aot-emit-dir=example" # currently expanding environment variables in vmarg is not supported.
# fib_mu_holstein:
# benchmark:
# name: micro/fib
# iterations: 10
# args:
# - 5
# source: build_fib.c
# result: fib_mu_data
# language:
# name: mu
# impl: holstein
# callback:
# name: clockcallback
# param: "6"
# environ: # can define environment variables that are passed to compiler and runner
# MU_HOLSTEIN: /Users/johnz/Documents/Work/mu-impl-ref2
# compiler:
# vmarg: vmLog=ERROR
# runner:
# flags:
# - --vmLog=ERROR
\ No newline at end of file
tasks:
py:
language: python
source: fib.py
runner:
exec: pypy
rpyc_O0:
language:
name: rpython
backend: c
source: targetfib.py
compiler:
flags:
- --gc=none
environ:
PYPY_C_CLANG_OPT: -O0
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: fib_rpyc_O0
rpyzebu:
language:
name: rpython
backend: mu
impl: zebu
source: targetfib.py
environ:
ZEBU_BUILD: release
PYPY_USESSION_DIR: example
PYPY_USESSION_BASENAME: fib_rpyzebu
rpyholstein:
language:
name: rpython
backend: mu
impl: holstein
source: targetfib.py
compiler:
flags:
- --mu-vmargs=vmLog=ERROR
runner:
flags:
- --vmLog=ERROR
environ:
PYPY_USESSION_DIR: example
PYPY_USESSION_BASENAME: fib_rpyzebu
c_O0:
language: c
source: fib.c
compiler:
flags:
- -O0
......@@ -3,51 +3,42 @@
# 0 don't throw an exception, but still have a catch block at the top
# 1 do throw an exception at the bottom of the recursion (to be caught by the top)
# 2 don't throw an exception, with no catch block
except_rpyc:
# the third argument is the scale factor
except:
benchmark:
name: micro/except
iterations: 1
source: targetexcept.py
args:
- 2000
- 1
language:
name: rpython
backend: c
environ:
PYPY_USESSION_DIR: example
PYPY_USESSION_BASENAME: except_rpyc
PYPY_C_CLANG_OPT: "-O3"
MICROBENCH_SCALE_FACTOR: "1"
callback:
name: ClockCallback
param: "6"
compiler:
pypy_dir: /home/isaacg/mu-client-pypy
except_rpymu_zebu:
benchmark:
name: micro/except
iterations: 1
source: targetexcept.py
args:
- 2000
- 1
result: alloc_rpyzebu_data
language:
name: rpython
backend: mu
impl: zebu
args:
- 2000
- 1
- 10
iterations: 10
callback:
name: ClockCallback
name: clock
param: "6"
environ:
PYPY_USESSION_DIR: example
PYPY_USESSION_BASENAME: except_rpymu
MU_ZEBU: /home/isaacg/mu-impl-fast
ZEBU_BUILD: release
MICROBENCH_SCALE_FACTOR: "1"
compiler:
pypy_dir: /home/isaacg/mu-client-pypy
tasks:
rpyc_O0:
language:
name: rpython
backend: c
source: targetexcept.py
environ:
PYPY_C_CLANG_OPT: -O0
PYPY_USESSION_BASENAME: "${MUBENCH_TASKSET_NAME}_${MUBENCH_TASK_NAME}"
rpyc_O3:
language:
name: rpython
backend: c
source: targetexcept.py
environ:
PYPY_C_CLANG_OPT: -O3
PYPY_USESSION_BASENAME: "${MUBENCH_TASKSET_NAME}_${MUBENCH_TASK_NAME}"
rpyzebu:
language:
name: rpython
backend: mu
impl: zebu
source: targetexcept.py
environ:
PYPY_USESSION_BASENAME: except_rpyzebu
# New format design decisions
# - putting scale factor as an argument to benchmark.
# This forces the scaling to be done at run time.
# - putting iterations outside benchmark, to be a key under task set
# - putting source as a key inside task
# - aggregating all results under one task set result file.
# - taking callback spec out to the task set level
# fields in which environs are expanded:
# - flags
# - args, vmarg
# - param
fib:
iterations: 5
benchmark:
name: micro/fib
args:
- 10 # scale factor
- 5
callback:
name: clock
param: "6"
tasks:
py:
language: python
source: fib.py
runner:
exec: pypy
rpyc_O0:
language:
name: rpython
backend: c
source: targetfib.py
compiler:
flags:
- --gc=none
environ:
PYPY_C_CLANG_OPT: -O0
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: fib_rpyc_O0
rpyzebu:
language:
name: rpython
backend: mu
impl: zebu
source: targetfib.py
environ:
ZEBU_BUILD: release
PYPY_USESSION_DIR: example
PYPY_USESSION_BASENAME: fib_rpyzebu
rpyholstein:
language:
name: rpython
backend: mu
impl: holstein
source: targetfib.py
compiler:
flags:
- --mu-vmargs=vmLog=ERROR
runner:
flags:
- --vmLog=ERROR
environ:
PYPY_USESSION_DIR: example
PYPY_USESSION_BASENAME: fib_rpyzebu
c_O0:
language: c
source: fib.c
compiler:
flags:
- -O0
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# NOTE: the scale factor of this test is specified at compile time
qs_rpyc_O0:
quicksort:
benchmark:
name: micro/quicksort
iterations: 10
source: targetquicksort.py
environ:
PYPY_C_CLANG_OPT: -O0
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: qs_rpyc_O0
language:
name: rpython
backend: c # required
callback:
name: ClockCallback
param: "6"
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy # required by rpython
flags:
- --gc=none
args:
- 1000
- 1000
- 100
qs_rpyc_O1:
benchmark:
name: micro/quicksort
iterations: 10
source: targetquicksort.py
environ:
MICROBENCH_SCALE_FACTOR: "1000"
PYPY_C_CLANG_OPT: -O1
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: qs_rpyc_O1
language:
name: rpython
backend: c # required
# args:
# - &scale_factor "1000"
iterations: 20
callback:
name: ClockCallback
name: clock
param: "6"
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy # required by rpython
flags:
- --gc=none
args:
- 1000
- 1000
- 100
qs_rpyc_O2:
benchmark:
name: micro/quicksort
iterations: 10
source: targetquicksort.py
environ:
MICROBENCH_SCALE_FACTOR: "1000"
PYPY_C_CLANG_OPT: -O2
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: qs_rpyc_O2
language:
name: rpython
backend: c # required
callback:
name: ClockCallback
param: "6"
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy # required by rpython
flags:
- --gc=none
args:
- 1000
- 1000
- 100
qs_rpyc_O3:
benchmark:
name: micro/quicksort
iterations: 10
source: targetquicksort.py
environ:
MICROBENCH_SCALE_FACTOR: "1000"
PYPY_C_CLANG_OPT: -O3
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: qs_rpyc_O3
language:
name: rpython
backend: c # required
callback:
name: ClockCallback
param: "6"
compiler:
pypy_dir: /Users/johnz/Documents/Work/mu-client-pypy # required by rpython
flags:
- --gc=none
args:
- 1000
- 1000
- 100
qs_rpyzebu:
benchmark:
name: micro/quicksort
iterations: 10
source: targetquicksort.py
language:
name: rpython
backend: mu
impl: zebu
callback:
name: ClockCallback
param: "6"
environ: # can define environment variables that are passed to compiler and runner
PYPY_MU: /Users/johnz/Documents/Work/mu-client-pypy
MU_ZEBU: /Users/johnz/Documents/Work/mu-impl-fast
ZEBU_BUILD: release
MICROBENCH_SCALE_FACTOR: "1000"
PYPY_USESSION_DIR: example # controls where PyPy puts usession directory
PYPY_USESSION_BASENAME: qs_rpyzebu
compiler:
pypy_dir: $PYPY_MU # can use environment variables in paths
args:
- 1000
- 1000
- 100
flags:
- --mu-vmargs=--aot-emit-dir=example/zebu-emit
- --mu-suplibdir=example
PYPY_USESSION_DIR: example
tasks:
rpyc_O0:
language:
name: rpython
backend: c
source: targetquicksort.py
environ:
PYPY_C_CLANG_OPT: -O0
PYPY_USESSION_BASENAME: &basename "${MUBENCH_TASKSET_NAME}_${MUBENCH_TASK_NAME}"
flags:
- --gc=none
compiler:
args:
- &scale_factor 1000
rpyc_O3:
language:
name: rpython
backend: c
source: targetquicksort.py
environ:
PYPY_C_CLANG_OPT: -O3
PYPY_USESSION_BASENAME: *basename
flags:
- --gc=none
compiler:
args:
- *scale_factor
rpyczebu:
language:
name: rpython
backend: mu
impl: zebu
source: targetquicksort.py
environ:
PYPY_USESSION_BASENAME: *basename
compiler:
args:
- *scale_factor
......@@ -71,7 +71,7 @@ class RPython(Language):
assert v in env, '%s needs to be defined in environ.'
# target configuration arguments
cc['args'] = list(map(lambda a: expandenv(a, task.env), cc.get('args', [])))
cc['args'] = list(map(lambda a: expandenv(str(a), task.env), cc.get('args', [])))
# flags
cc['flags'] = list(map(lambda a: expandenv(a, task.env), cc.get('flags', [])))
......
......@@ -127,11 +127,12 @@ class TaskSet:
return data
@staticmethod
def from_file(config_file):
with open(config_file) as fp:
config_dic = yaml.load(fp.read())
def from_yaml(yaml_s, run_dir):
config_dic = yaml.load(yaml_s)
run_dir = Path(config_file).parent
from pprint import PrettyPrinter
pprint = PrettyPrinter().pprint
pprint(config_dic)
tasksets = []
for name, conf in config_dic.items():
......@@ -167,11 +168,18 @@ class TaskSet:
for task_name, task_conf in conf['tasks'].items():
try:
# try:
ts.add_task(Task(ts, task_name, **task_conf))
except Exception as e:
task_print(task_name, crayons.red('parsing configuration failed.'))
logger.critical(crayons.red(str(e)))
# except Exception as e:
# task_print(task_name, crayons.red('parsing configuration failed.'))
# logger.critical(crayons.red(str(e)))
tasksets.append(ts)
return tasksets
@staticmethod
def from_file(config_file):
with open(config_file) as fp:
return TaskSet.from_yaml(fp.read(), Path(config_file).parent)
......@@ -18,34 +18,26 @@ class A:
pass
DEFAULT_SCALE_FACTOR = 1000000000
def target(driver, args):
from callback import get_callback
cb_cls = get_callback(args[0])
import os
SCALE_FACTOR = int(
os.environ.get('MICROBENCH_SCALE_FACTOR', str(DEFAULT_SCALE_FACTOR)))
def main(argv):
cb_param = argv[1]
iterations = int(argv[2])
resfile = argv[3]
cb = cb_cls(cb_param)
n = int(argv[4])
# program args
scale_factor = int(argv[2])
n = int(argv[3])
cb = cb_cls(cb_param)
lst = []
for i in range(iterations):
cb.begin()
for j in range(SCALE_FACTOR):
for k in range(n):
lst.append(A())
cb.end()
cb.report(resfile)
return int(not (len(lst) == n * SCALE_FACTOR * iterations))
cb.begin()
for j in range(scale_factor):
for k in range(n):
lst.append(A())
cb.end()
cb.report()
return int(not (len(lst) == n * scale_factor))
return main, None
DEFAULT_SCALE_FACTOR = 1000000000;
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");