taskset.py 11.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
#!/usr/bin/env python3
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16
import os
17
import sys
Zixian Cai's avatar
Zixian Cai committed
18 19 20
import yaml
import logging
import crayons
21
import json
Zixian Cai's avatar
Zixian Cai committed
22 23

from pathlib import Path
Zixian Cai's avatar
Zixian Cai committed
24
from types import SimpleNamespace
25

26
from mubench import SUITE_DIR, CALLBACKS_DIR
John Zhang's avatar
John Zhang committed
27
from mubench.conf import settings
Zixian Cai's avatar
Zixian Cai committed
28 29
from mubench.utils import expandenv, dictify, run_in_subproc, ExecutionFailure
from mubench.utils import add_path_to_ld_library_path
30
from mubench.lang import get_lang, Language
31
from mubench.models.trails import Trails
32
from mubench.models.callback import CALLBACK_BY_NAME
33
from collections import defaultdict
Zixian Cai's avatar
Zixian Cai committed
34 35 36 37 38 39 40 41 42 43 44 45

logger = logging.getLogger(__name__)


def task_print(task_name, message):
    logger.info("[{}] {}".format(
        crayons.yellow(task_name),
        message
    ))


class TaskSet:
Zixian Cai's avatar
Zixian Cai committed
46 47
    def __init__(self, name, benchmark, iterations, callback, runnerwrap=None,
                 **kwds):
48 49 50
        self.name = name
        self.benchmark = benchmark
        self.iterations = iterations
51
        self.runnerwrap = runnerwrap
52
        self.output_dir = kwds['output_dir']
53
        self.resfile = kwds['resfile']
54
        self.tasks = []
55
        self.comparison = kwds['comparisons']
56

John Zhang's avatar
John Zhang committed
57
        # environ
Zixian Cai's avatar
Zixian Cai committed
58 59 60 61 62
        self.env = os.environ.copy()  # base on os.environ
        self.env.update(getattr(settings, 'ENVIRON', {}))  # local settings
        self.env['MUBENCH_TASKSET_NAME'] = name  # taskset name
        ts_env = kwds['env']  # taskset definitions
        for v in ts_env:  # first expand the environs (based on what's been defined so far)
63 64
            ts_env[v] = expandenv(ts_env[v], self.env)
        self.env.update(ts_env)
John Zhang's avatar
John Zhang committed
65

66 67 68 69 70
        # expand environs in benchmark args and callback param and paths
        def get_expanded_list(l):
            return list(map(lambda a: expandenv(str(a), self.env), l))

        benchmark['args'] = get_expanded_list(benchmark['args'])
John Zhang's avatar
John Zhang committed
71
        callback['param'] = expandenv(callback['param'], self.env)
72 73 74
        callback['include_dirs'] = get_expanded_list(callback['include_dirs'])
        callback['library_dirs'] = get_expanded_list(callback['library_dirs'])
        callback['extra_srcs'] = get_expanded_list(callback['extra_srcs'])
John Zhang's avatar
John Zhang committed
75

76 77
        self.callback = CALLBACK_BY_NAME[callback['name']](callback, self)

78
        # compiled callback shared library
79
        add_path_to_ld_library_path(str(self.output_dir), self.env)
80

81
        for d in self.callback.library_dirs:
82
            add_path_to_ld_library_path(d, self.env)
83

84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
    @staticmethod
    def from_config_dict(name, conf_d, conf_dir=None):
        # output directory
        output_dir = Path(conf_d.get('outdir', str(conf_dir)))

        # check iterations
        assert 'iterations' in conf_d, 'iterations not defined'

        # check benchmark
        # check name
        assert 'benchmark' in conf_d, 'benchmark not defined'
        assert (SUITE_DIR / conf_d['benchmark']['name']).exists(), \
            "benchmark %(name)s not found" % conf_d['benchmark']
        conf_d['benchmark'].setdefault('args', [])

        # check record file
        # check record
        resfile = Path(conf_d.get('recfile', '%(name)s.json' % locals()))
        if not resfile.is_absolute():
            resfile = output_dir / resfile
John Zhang's avatar
John Zhang committed
104 105
            if not resfile.parent.exists():
                resfile.parent.mkdir(parents=True)
106 107 108 109 110 111

        # check callback
        assert 'callback' in conf_d, 'callback not defined'
        d = dictify(conf_d['callback'])
        if 'param' not in d or d['param'] is None:
            d['param'] = ""  # default to ""
112 113 114 115 116 117
        if 'include_dirs' not in d or d['include_dirs'] is None:
            d['include_dirs'] = []  # default to []
        if 'library_dirs' not in d or d['library_dirs'] is None:
            d['library_dirs'] = []  # default to []
        if 'libraries' not in d or d['libraries'] is None:
            d['libraries'] = []  # default to []
118 119 120 121
        if 'extra_srcs' not in d or d['extra_srcs'] is None:
            d['extra_srcs'] = []  # default to []
        if 'flags' not in d or d['flags'] is None:
            d['flags'] = []  # default to []
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
        conf_d['callback'] = d

        # add comparison
        comparisons = []
        if 'compare' in conf_d:
            for cmp in conf_d["compare"]:
                comparisons.append(SimpleNamespace(op1=cmp[0], op2=cmp[1]))

        ts = TaskSet(name, conf_d['benchmark'], conf_d['iterations'],
                     conf_d['callback'],
                     output_dir=output_dir, resfile=resfile,
                     env=conf_d.get('environ', {}),
                     comparisons=comparisons)

        # add tasks
        for task_name, task_conf in conf_d['tasks'].items():
            try:
                ts.tasks.append(Task(ts, task_name, **task_conf))
            except Exception as e:
141 142 143 144
                # task_print(task_name,
                #            crayons.red('parsing configuration failed.'))
                # logger.critical(crayons.red(str(e)))
                ts.tasks.append(Task(ts, task_name, **task_conf))
145 146

        return ts
Zixian Cai's avatar
Zixian Cai committed
147

148
    def run(self, skipcomp_l):
149
        # compile callback into shared library first
150
        libcb = self.callback.compile_dylib(self.output_dir, self.env)
151

John Zhang's avatar
John Zhang committed
152 153
        # compile first
        targets = {}
154 155
        for task in self.tasks:
            if not task.lang_cls.compiled:  # interpreted
John Zhang's avatar
John Zhang committed
156
                targets[task] = task.srcfile
157 158
            else:  # need compilation
                if task.name in skipcomp_l:  # skip compilation -> assume default target
159 160 161 162 163 164 165 166 167 168 169 170
                    targets[task] = task.get_default_target()
                else:
                    task_print(task.name, 'compiling...')
                    try:
                        target = task.compile()
                        task_print(task.name, 'target %s generated' % target)
                        targets[task] = target
                    except ExecutionFailure as e:
                        task_print(task.name, crayons.red('FAILED'))
                        logger.critical(crayons.red(str(e)))
                        errlog_file = self.output_dir / (task.name + '.log')
                        e.dump(errlog_file)
171
                        task_print(task.name,
172 173
                                   crayons.red(
                                       'error output written to %s' % errlog_file))
Zixian Cai's avatar
Zixian Cai committed
174 175
                        if settings.FAIL_EARLY:
                            exit(e.exec_res.returncode)
John Zhang's avatar
John Zhang committed
176 177

        # run
178
        data = {t: [] for t in targets}  # only run tasks that have a target
John Zhang's avatar
John Zhang committed
179

Zixian Cai's avatar
Zixian Cai committed
180
        # Generating record
John Zhang's avatar
John Zhang committed
181
        for i in range(self.iterations):
John Zhang's avatar
John Zhang committed
182
            logger.info("Running iteration %d..." % i)
John Zhang's avatar
John Zhang committed
183
            keys = list(data.keys())
184
            for task in keys:
185 186 187 188 189
                target = targets[task]
                try:
                    res, t_proc = task.run(target)
                    task.add_datapoint(res.stdout, res.stderr, t_proc)
                    data[task].append({
190 191
                        'stdout': res.stdout,
                        'stderr': res.stderr,
192 193 194 195 196 197 198 199
                        't_proc': t_proc
                    })
                except ExecutionFailure as e:
                    task_print(task.name, crayons.red('FAILED'))
                    logger.critical(crayons.red(str(e)))
                    errlog_file = self.output_dir / (task.name + '.log')
                    e.dump(errlog_file)
                    task_print(task.name, crayons.red(
200
                        'error output written to %s' % errlog_file))
201
                    del data[task]
Zixian Cai's avatar
Zixian Cai committed
202 203
                    if settings.FAIL_EARLY:
                        exit(e.exec_res.returncode)
John Zhang's avatar
John Zhang committed
204

John Zhang's avatar
John Zhang committed
205 206 207
        self.results = {task.name: task.get_result() for task in data}
        return self.results

208 209
    def get_log(self):
        log = {
210 211 212
            'name': self.name,
            'iterations': self.iterations,
            'benchmark': self.benchmark,
213
            'callback': {k: self.callback[k] for k in ('name', 'param')},
John Zhang's avatar
John Zhang committed
214
            'datapoints': {t.name: t.stats for t in self.tasks},
215
        }
216
        return log
217

218

219 220 221 222 223
class Task:
    """
    An task of benchmark performance measurement;
    corresponds to the outmost level mapping in YAML configuration file
    """
224

225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
    def __init__(self, taskset, name, **conf):
        self.taskset = taskset
        self.name = name
        self.env = taskset.env.copy()  # based on taskset environ
        self.env['MUBENCH_TASK_NAME'] = name
        task_env = conf.get('environ', {})
        for v in task_env:
            task_env[v] = expandenv(task_env[v], self.env)
        self.env.update(task_env)

        self.output_dir = taskset.output_dir

        # benchmark
        self.benchmark = taskset.benchmark

        # callback
        self.callback = taskset.callback

        # check source
        assert 'source' in conf, 'source not defined'
        src = SUITE_DIR / self.benchmark['name'] / conf['source']
        assert src.exists(), "source file %(src)s not found" % locals()
        conf['source'] = src
        self.srcfile = src

        # language
        lang_d = dictify(conf.get('language', {}))
        assert 'name' in lang_d, 'language not defined'
        self.lang_cls = get_lang(lang_d['name'])
        self.lang = self.lang_cls.check_lang(lang_d)

        # set defaults for others
        self.compiler = self.lang_cls.check_compiler(conf.get('compiler', {}),
                                                     self.lang, self)
        self.runner = self.lang_cls.check_runner(conf.get('runner', {}),
                                                 self.lang, self)

        self.config = conf

264
        self.stats = []
265 266 267 268 269 270 271 272 273

    def compile(self):
        if self.lang_cls.compiled:
            return self.lang_cls.compile(self)

    def run(self, target):
        res = self.lang_cls.run(target, self)
        return res

274 275 276
    def get_default_target(self):
        return self.lang_cls.get_default_target(self)

277 278 279
    # TODO: maybe refactor this.
    # Results and data should not be part of a taskset/task,
    # but rather the Result should be *about* a TaskSet
280
    def add_datapoint(self, stdout, stderr, t_proc):
281 282 283
        stat_d = self.callback.extract_stat(stdout, stderr)
        stat_d['t_proc'] = float(t_proc)
        self.stats.append(stat_d)
284 285

    def get_result(self):
286 287 288 289 290 291 292 293 294
        result = defaultdict(list)
        for pair in self.stats:
            for k, v in pair.items():
                result[k].append(v)
        result = {k: Trails("{}:{} {}".format(self.taskset.name,
                                              self.name,
                                              k), v)
                  for k, v in result.items()}
        return result
295 296 297 298 299 300 301 302 303 304 305

    def __str__(self):
        return self.name


def load_yaml(yaml_s, run_dir):
    config_d = yaml.load(yaml_s)
    tasksets = []
    for name, ts_conf_d in config_d.items():
        tasksets.append(TaskSet.from_config_dict(name, ts_conf_d, run_dir))
    return tasksets
306 307


308 309 310
def load_file(config_files):
    tasksets = []
    for config_file in config_files:
Zixian Cai's avatar
Zixian Cai committed
311
        config_file = Path(config_file).resolve()
312
        with open(config_file) as fp:
Zixian Cai's avatar
Zixian Cai committed
313 314
            tasksets.extend(load_yaml(fp.read(),
                                      config_file.parent))
315
    return tasksets