GitLab will be upgraded to the 12.10.14-ce.0 on 28 Sept 2020 at 2.00pm (AEDT) to 2.30pm (AEDT). During the update, GitLab and Mattermost services will not be available. If you have any concerns with this, please talk to us at N110 (b) CSIT building.

Commit a42efd9e authored by John Zhang's avatar John Zhang

environment variable resolution

parent 3d926284
......@@ -98,6 +98,30 @@ Each task can overwrite the task set environment variables by redefining it in i
Note that the system-wide environment variables are visible in the framework.
This is especially useful for path related variables.
In addition, project-wise local environment variables can be defined in the `ENVIRON` variable in `settings.py` under the project root directory (they can be overwritten by task set `environ`):
```python
ENVIRON = {
"MU_ZEBU": "/path/to/mu-impl-fast",
"PYPY": "/path/to/mu-client-pypy",
...
}
```
In general, environment variables in the following keys are expanded:
```yaml
args:
param:
flags:
vmarg:
```
Below are some predefined environment variables:
| name | value |
| ---- | ----- |
| `$MUBENCH_TASKSET_NAME` | The taskset name |
| `$MUBENCH_TASK_NAME` | The task name, this will be different for each task |
### `outdir`
The directory to where all the output and temporary files are generated.
The default is the directory where the configuration file is in.
......@@ -179,6 +203,9 @@ The Python interpreter can be specified in the `exec` subkey, `python` is used b
### RPython
```yaml
environ:
PYPY: # required
MU_(IMPL): # required
language:
name: rpython
backend: # required, string
......@@ -204,9 +231,8 @@ Currently known implementations are `holstein` and `zebu`.
This is required by the Mu backend of the RPython compiler,
and it can be specified in `environ` or inherited from system-wide environment.
`pypy_dir` key is required to tell the compiler where the
[mu-client-pypy](https://gitlab.anu.edu.au/mu/mu-client-pypy) repository is cloned.
The use of environment variable here will be automatically resolved.
`PYPY` environment variable is required to point to the cloned
[mu-client-pypy](https://gitlab.anu.edu.au/mu/mu-client-pypy) repository.
An additional list of flags to the RPython compiler can be specified in the `flags` key.
The default flags `-O3` and `--no-shared` are always automatically included.
......@@ -236,6 +262,8 @@ Addional compiler flags can be specified in `flags`.
### Mu
```yaml
environ:
MU_(IMPL): # required, string
language:
name: mu
impl: # required, string
......
......@@ -5,6 +5,11 @@
# - putting source as a key inside task
# - aggregating all results under one task set result file.
# - taking callback spec out to the task set level
# fields in which environs are expanded:
# - flags
# - args, vmarg
# - param
fib:
iterations: 5
benchmark:
......@@ -27,7 +32,7 @@ fib:
backend: c
source: targetfib.py
compiler:
pypy_dir: $PYPY_MU
pypydir: $PYPY_MU
flags:
- --gc=none
environ:
......@@ -41,7 +46,7 @@ fib:
impl: zebu
source: targetfib.py
compiler:
pypy_dir: $PYPY_MU
pypydir: $PYPY_MU
environ:
ZEBU_BUILD: release
PYPY_USESSION_DIR: example
......@@ -53,7 +58,7 @@ fib:
impl: holstein
source: targetfib.py
compiler:
pypy_dir: $PYPY_MU
pypydir: $PYPY_MU
flags:
- --mu-vmargs=vmLog=ERROR
runner:
......
......@@ -43,7 +43,8 @@ class C(Language):
# flags
flags = []
include_flags = ['-I%(callback_dir)s' % locals()]
flags.extend(cc.get('flags', []))
cc_flags = list(map(lambda a: expandenv(a, task.env), cc.get('flags', [])))
flags.extend(cc_flags)
flags.extend(include_flags)
target_dir = task.output_dir
target_name = '%s-%s' % (task.taskset.name, task.name)
......
......@@ -39,13 +39,13 @@ class Mu(Language):
@classmethod
def check_compiler(cls, cc, lc, task):
impl = lc['impl']
env = task.evn
env = task.env
# check Mu impl environment variable
v = 'MU_%s' % (impl.upper())
assert v in env, '%s needs to be defined in environ.'
mu_dir = Path(env[v])
mudir = Path(env[v])
exe = cc.get('exec', cls.default_exec)
cc['exec'] = expandenv(exe, env)
......@@ -54,9 +54,9 @@ class Mu(Language):
include_dirs = cc.get('include_dirs', [])
include_dirs.append(CALLBACKS_DIR / cls.name)
if impl == 'holstein':
include_dirs.append(mu_dir / 'cbinding')
include_dirs.append(mudir / 'cbinding')
elif impl == 'zebu':
include_dirs.append(mu_dir / 'src' / 'vm' / 'api')
include_dirs.append(mudir / 'src' / 'vm' / 'api')
cc['include_dirs'] = include_dirs
# library dirs
......@@ -64,15 +64,17 @@ class Mu(Language):
libmu_dir = ''
libname = ''
if impl == 'holstein':
libmu_dir = mu_dir / 'cbinding'
libmu_dir = mudir / 'cbinding'
libname = 'murefimpl2start'
elif impl == 'zebu':
libmu_dir = mu_dir / 'target' / env.get('ZEBU_BUILD', 'release')
libmu_dir = mudir / 'target' / env.get('ZEBU_BUILD', 'release')
libname = 'mu'
library_dirs.append(libmu_dir)
cc['library_dirs'] = library_dirs
cc.setdefault('libmu_link_name', libname)
# vmarg
cc['vmarg'] = expandenv(cc.get('vmarg', ""), task.env)
return cc
@classmethod
......@@ -131,7 +133,7 @@ class Mu(Language):
# Then run the build program to get the boot image
target = target_dir / target_name
cmd = [build_target]
cmd.append(cc.get('vmarg', ""))
cmd.append(cc['vmarg'])
cmd.append(target)
cls.run_in_subproc(cmd, task.env)
......
......@@ -36,8 +36,8 @@ class RPython(Language):
def check_lang(cls, lc):
assert 'backend' in lc, "backend not specified."
backend = lc['backend']
assert backend in (
'c', 'mu'), "invalid backend: %(backend)s." % locals()
assert backend in ('c', 'mu'), \
"invalid backend: %(backend)s." % locals()
if backend == 'mu':
assert 'impl' in lc, "Mu impl needs to be specified for Mu backend."
assert lc['impl'] in cls.known_impls, 'invalid impl: %(impl)s' % lc
......@@ -52,19 +52,17 @@ class RPython(Language):
cc['exec'] = expandenv(python, env)
# rpython
assert 'pypy_dir' in cc, "pypy_dir not defined"
pypy_dir = Path(expandenv(cc['pypy_dir'], env))
rpython_py = pypy_dir / 'rpython' / 'bin' / 'rpython'
assert 'PYPY' in env, 'PYPY needs to be defined in environ'
pypydir = Path(env['PYPY'])
rpython_py = pypydir / 'rpython' / 'bin' / 'rpython'
assert rpython_py.exists(), '%(rpython_py)s not found' % locals()
cc['rpython'] = rpython_py
cc['pypy_dir'] = pypy_dir
# set PYTHONPATH
# include: pypy_dir, callbacks/rpython
pypy_dir = cc['pypy_dir']
# include: pypydir, callbacks/rpython
pythonpath = env.get('PYTHONPATH', '')
callback_dir = CALLBACKS_DIR / cls.name
pythonpath = '%(pypy_dir)s:%(callback_dir)s:%(pythonpath)s' % locals()
pythonpath = '%(pypydir)s:%(callback_dir)s:%(pythonpath)s' % locals()
env['PYTHONPATH'] = pythonpath
# check Mu impl environment variable
......@@ -73,7 +71,10 @@ class RPython(Language):
assert v in env, '%s needs to be defined in environ.'
# target configuration arguments
cc.setdefault('args', [])
cc['args'] = list(map(lambda a: expandenv(a, task.env), cc.get('args', [])))
# flags
cc['flags'] = list(map(lambda a: expandenv(a, task.env), cc.get('flags', [])))
return cc
......@@ -94,7 +95,8 @@ class RPython(Language):
rc['exec'] = Path(expandenv(str(rc['exec']), env))
holstein_default_flags = ['--uPtrHack=True']
rc['flags'] = holstein_default_flags + rc.get('flags', [])
rc_flags = list(map(lambda a: expandenv(a, task.env), rc.get('flags', [])))
rc['flags'] = holstein_default_flags + rc_flags
return rc
......@@ -108,6 +110,7 @@ class RPython(Language):
# flags
bk = task.lang['backend']
flags = []
flags.extend(cls.default_flags)
flags.append('--backend=%(bk)s' % locals())
if bk == 'mu':
flags.append('--mu-impl=%(impl)s' % task.lang)
......@@ -116,7 +119,7 @@ class RPython(Language):
emit_dir = task.output_dir / ('%s_%s-emit' % (task.taskset.name, task.name))
flags.append('--mu-vmargs=--aot-emit-dir=%s' % emit_dir)
flags.extend(task.compiler.get('flags', cls.default_flags))
flags.extend(task.compiler['flags'])
target_dir = task.output_dir
target_name = '%s-%s' % (task.taskset.name, task.name)
target = target_dir / target_name
......
......@@ -34,8 +34,9 @@ class Task:
def __init__(self, taskset, name, **conf):
self.taskset = taskset
self.name = name
self.env = taskset.env.copy()
self.env.update(conf.get('environ', {})) # os + taskset + task defined
self.env = taskset.env.copy() # based on taskset environ
self.env.update(conf.get('environ', {}))
self.env['MUBENCH_TASK_NAME'] = name
self.output_dir = taskset.output_dir
# benchmark
......
......@@ -24,6 +24,8 @@ from mubench.exceptions import ExecutionFailure
from mubench.models.task import Task, dictify
from mubench import SUITE_DIR
from mubench.models.result import Result
from mubench.conf import settings
from mubench.util import expandenv
logger = logging.getLogger(__name__)
......@@ -43,10 +45,19 @@ class TaskSet:
self.callback = callback
self.runnerwrap = runnerwrap
self.output_dir = kwds['output_dir']
self.env = kwds['env']
self.resfile = kwds['resfile']
self.tasks = []
# environ
self.env = os.environ.copy() # base on os.environ
self.env.update(getattr(settings, 'ENVIRON', {})) # local settings
self.env.update(kwds['env']) # taskset definitions
self.env['MUBENCH_TASKSET_NAME'] = name # taskset name
# expand environs in benchmark args and callback param
benchmark['args'] = list(map(lambda a: expandenv(str(a), self.env), benchmark['args']))
callback['param'] = expandenv(callback['param'], self.env)
def add_task(self, task):
self.tasks.append(task)
......@@ -148,11 +159,8 @@ class TaskSet:
d['param'] = "" # default to ""
conf['callback'] = d
env = os.environ.copy()
env.update(conf.get('environ', {})) # all os.environ and task set defined
ts = TaskSet(name, conf['benchmark'], conf['iterations'], conf['callback'],
output_dir=output_dir, resfile=resfile, env=env)
output_dir=output_dir, resfile=resfile, env=conf.get('environ', {}))
for task_name, task_conf in conf['tasks'].items():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment