WARNING! Access to this system is limited to authorised users only.
Unauthorised users may be subject to prosecution.
Unauthorised access to this system is a criminal offence under Australian law (Federal Crimes Act 1914 Part VIA)
It is a criminal offence to:
(1) Obtain access to data without authority. -Penalty 2 years imprisonment.
(2) Damage, delete, alter or insert data without authority. -Penalty 10 years imprisonment.
User activity is monitored and recorded. Anyone using this system expressly consents to such monitoring and recording.

To protect your data, the CISO officer has suggested users to enable 2FA as soon as possible.
Currently 2.7% of users enabled 2FA.

taskset.py 12.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#!/usr/bin/env python3
# Copyright 2017 The Australian National University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

16
import os
17
import sys
Zixian Cai's avatar
Zixian Cai committed
18
19
20
import yaml
import logging
import crayons
21
import json
Zixian Cai's avatar
Zixian Cai committed
22
23

from pathlib import Path
Zixian Cai's avatar
Zixian Cai committed
24
from types import SimpleNamespace
25

26
from mubench import SUITE_DIR, CALLBACKS_DIR
John Zhang's avatar
John Zhang committed
27
from mubench.conf import settings
28
from mubench.util import expandenv, dictify, run_in_subproc, ExecutionFailure
John Zhang's avatar
John Zhang committed
29
from mubench.util import add_path_to_ld_library_path
30
from mubench.lang import get_lang, Language
31
from mubench.models.result import Result
32
from mubench.models import cbconf
Zixian Cai's avatar
Zixian Cai committed
33
34
35
36
37
38
39
40
41
42
43
44

logger = logging.getLogger(__name__)


def task_print(task_name, message):
    logger.info("[{}] {}".format(
        crayons.yellow(task_name),
        message
    ))


class TaskSet:
Zixian Cai's avatar
Zixian Cai committed
45
46
    def __init__(self, name, benchmark, iterations, callback, runnerwrap=None,
                 **kwds):
47
48
49
        self.name = name
        self.benchmark = benchmark
        self.iterations = iterations
50
        self.callback = callback
51
        self.runnerwrap = runnerwrap
52
        self.output_dir = kwds['output_dir']
53
        self.resfile = kwds['resfile']
54
        self.tasks = []
55
        self.comparison = kwds['comparisons']
56

John Zhang's avatar
John Zhang committed
57
        # environ
Zixian Cai's avatar
Zixian Cai committed
58
59
60
61
62
        self.env = os.environ.copy()  # base on os.environ
        self.env.update(getattr(settings, 'ENVIRON', {}))  # local settings
        self.env['MUBENCH_TASKSET_NAME'] = name  # taskset name
        ts_env = kwds['env']  # taskset definitions
        for v in ts_env:  # first expand the environs (based on what's been defined so far)
63
64
            ts_env[v] = expandenv(ts_env[v], self.env)
        self.env.update(ts_env)
John Zhang's avatar
John Zhang committed
65

66
67
68
69
70
71
72
73
74
        # configure callback
        conf_func = getattr(cbconf, 'configure_cb_%(name)s' % callback)
        conf_func(callback, self.env)

        # expand environs in benchmark args and callback param and paths
        def get_expanded_list(l):
            return list(map(lambda a: expandenv(str(a), self.env), l))

        benchmark['args'] = get_expanded_list(benchmark['args'])
John Zhang's avatar
John Zhang committed
75
        callback['param'] = expandenv(callback['param'], self.env)
76
77
78
        callback['include_dirs'] = get_expanded_list(callback['include_dirs'])
        callback['library_dirs'] = get_expanded_list(callback['library_dirs'])
        callback['extra_srcs'] = get_expanded_list(callback['extra_srcs'])
John Zhang's avatar
John Zhang committed
79

80
        # compiled callback shared library
John Zhang's avatar
John Zhang committed
81
82
        libext = '.dylib' if sys.platform == 'darwin' else '.so'
        add_path_to_ld_library_path(str(self.output_dir), self.env)
83
        self.callback['dylib'] = self.output_dir / ('libcb_%(name)s' % self.callback + libext)
84

John Zhang's avatar
John Zhang committed
85
        for d in self.callback['library_dirs']:
86
            add_path_to_ld_library_path(d, self.env)
John Zhang's avatar
John Zhang committed
87

88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
    @staticmethod
    def from_config_dict(name, conf_d, conf_dir=None):
        # output directory
        output_dir = Path(conf_d.get('outdir', str(conf_dir)))

        # check iterations
        assert 'iterations' in conf_d, 'iterations not defined'

        # check benchmark
        # check name
        assert 'benchmark' in conf_d, 'benchmark not defined'
        assert (SUITE_DIR / conf_d['benchmark']['name']).exists(), \
            "benchmark %(name)s not found" % conf_d['benchmark']
        conf_d['benchmark'].setdefault('args', [])

        # check record file
        # check record
        resfile = Path(conf_d.get('recfile', '%(name)s.json' % locals()))
        if not resfile.is_absolute():
            resfile = output_dir / resfile
John Zhang's avatar
John Zhang committed
108
109
            if not resfile.parent.exists():
                resfile.parent.mkdir(parents=True)
110
111
112
113
114
115

        # check callback
        assert 'callback' in conf_d, 'callback not defined'
        d = dictify(conf_d['callback'])
        if 'param' not in d or d['param'] is None:
            d['param'] = ""  # default to ""
116
117
118
119
120
121
        if 'include_dirs' not in d or d['include_dirs'] is None:
            d['include_dirs'] = []  # default to []
        if 'library_dirs' not in d or d['library_dirs'] is None:
            d['library_dirs'] = []  # default to []
        if 'libraries' not in d or d['libraries'] is None:
            d['libraries'] = []  # default to []
122
123
124
125
        if 'extra_srcs' not in d or d['extra_srcs'] is None:
            d['extra_srcs'] = []  # default to []
        if 'flags' not in d or d['flags'] is None:
            d['flags'] = []  # default to []
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
        conf_d['callback'] = d

        # add comparison
        comparisons = []
        if 'compare' in conf_d:
            for cmp in conf_d["compare"]:
                comparisons.append(SimpleNamespace(op1=cmp[0], op2=cmp[1]))

        ts = TaskSet(name, conf_d['benchmark'], conf_d['iterations'],
                     conf_d['callback'],
                     output_dir=output_dir, resfile=resfile,
                     env=conf_d.get('environ', {}),
                     comparisons=comparisons)

        # add tasks
        for task_name, task_conf in conf_d['tasks'].items():
            try:
                ts.tasks.append(Task(ts, task_name, **task_conf))
            except Exception as e:
                task_print(task_name,
                           crayons.red('parsing configuration failed.'))
                logger.critical(crayons.red(str(e)))
148
                # ts.tasks.append(Task(ts, task_name, **task_conf))
149
150

        return ts
Zixian Cai's avatar
Zixian Cai committed
151

152
    def run(self, skipcomp_l):
153
154
155
        # compile callback into shared library first
        self.compile_callback()

John Zhang's avatar
John Zhang committed
156
157
        # compile first
        targets = {}
158
159
        for task in self.tasks:
            if not task.lang_cls.compiled:  # interpreted
John Zhang's avatar
John Zhang committed
160
                targets[task] = task.srcfile
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
            else:   # need compilation
                if task.name in skipcomp_l: # skip compilation -> assume default target
                    targets[task] = task.get_default_target()
                else:
                    task_print(task.name, 'compiling...')
                    try:
                        target = task.compile()
                        task_print(task.name, 'target %s generated' % target)
                        targets[task] = target
                    except ExecutionFailure as e:
                        task_print(task.name, crayons.red('FAILED'))
                        logger.critical(crayons.red(str(e)))
                        errlog_file = self.output_dir / (task.name + '.log')
                        e.dump(errlog_file)
                        task_print(task.name, crayons.red(
176
                            'error output written to %s' % errlog_file))
John Zhang's avatar
John Zhang committed
177
178

        # run
179
        data = {t: [] for t in targets}  # only run tasks that have a target
John Zhang's avatar
John Zhang committed
180

Zixian Cai's avatar
Zixian Cai committed
181
        # Generating record
John Zhang's avatar
John Zhang committed
182
        for i in range(self.iterations):
John Zhang's avatar
John Zhang committed
183
            logger.info("Running iteration %d..." % i)
John Zhang's avatar
John Zhang committed
184
            keys = list(data.keys())
185
            for task in keys:
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
                target = targets[task]
                try:
                    res, t_proc = task.run(target)
                    task.add_datapoint(res.stdout, res.stderr, t_proc)
                    data[task].append({
                        'stdout': str(res.stdout, encoding='utf-8'),
                        'stderr': str(res.stderr, encoding='utf-8'),
                        't_proc': t_proc
                    })
                except ExecutionFailure as e:
                    task_print(task.name, crayons.red('FAILED'))
                    logger.critical(crayons.red(str(e)))
                    errlog_file = self.output_dir / (task.name + '.log')
                    e.dump(errlog_file)
                    task_print(task.name, crayons.red(
201
                        'error output written to %s' % errlog_file))
202
                    del data[task]
John Zhang's avatar
John Zhang committed
203

204
        self.callback['dylib'] = str(self.callback['dylib'])    # convert into string for dumping
205
206
207
208
        record = {
            'name': self.name,
            'iterations': self.iterations,
            'benchmark': self.benchmark,
209
            'callback': self.callback,
210
            'results': {t.name: data[t] for t in data},
211
212
213
214
        }

        # save to result file
        with self.resfile.open('w') as fp:
Zixian Cai's avatar
Zixian Cai committed
215
            json.dump(record, fp, indent=2, separators=(', ', ': '))
John Zhang's avatar
John Zhang committed
216

217
218
        # TODO: restructure this
        for task in data:
Zixian Cai's avatar
Zixian Cai committed
219
220
            task.aggregate_datapoint()

221
222
223
        self.results = {task.name: task.get_result() for task in data}
        return self.results
        # TODO: debug
Zixian Cai's avatar
Zixian Cai committed
224

225
226
227
228
229
    def compile_callback(self):
        cmd = []
        cc = self.env.get('CC', 'clang')
        cmd.append(cc)

230
        cmd.extend(['-shared', '-fPIC'])
231
232

        # include_dirs
233
        cmd.extend(map(lambda s: '-I' + s, self.callback['include_dirs']))
234
235

        # library_dirs
236
        cmd.extend(map(lambda s: '-L' + s, self.callback['library_dirs']))
237
238

        # libraries
239
        cmd.extend(map(lambda s: '-l' + s, self.callback['libraries']))
John Zhang's avatar
John Zhang committed
240
241

        # flags
242
        cmd.extend(self.callback['flags'])
243
244
245
246
247
248

        # output
        cmd.extend(['-o', self.callback['dylib']])

        # source
        cmd.append(CALLBACKS_DIR / ('cb_%(name)s.c' % self.callback))
249
        cmd.extend(self.callback['extra_srcs'])
250

251
        run_in_subproc(cmd, self.env)
252

253
254
255
256
257
class Task:
    """
    An task of benchmark performance measurement;
    corresponds to the outmost level mapping in YAML configuration file
    """
258

259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
    def __init__(self, taskset, name, **conf):
        self.taskset = taskset
        self.name = name
        self.env = taskset.env.copy()  # based on taskset environ
        self.env['MUBENCH_TASK_NAME'] = name
        task_env = conf.get('environ', {})
        for v in task_env:
            task_env[v] = expandenv(task_env[v], self.env)
        self.env.update(task_env)

        self.output_dir = taskset.output_dir

        # benchmark
        self.benchmark = taskset.benchmark

        # callback
        self.callback = taskset.callback

        # check source
        assert 'source' in conf, 'source not defined'
        src = SUITE_DIR / self.benchmark['name'] / conf['source']
        assert src.exists(), "source file %(src)s not found" % locals()
        conf['source'] = src
        self.srcfile = src

        # language
        lang_d = dictify(conf.get('language', {}))
        assert 'name' in lang_d, 'language not defined'
        self.lang_cls = get_lang(lang_d['name'])
        self.lang = self.lang_cls.check_lang(lang_d)

        # set defaults for others
        self.compiler = self.lang_cls.check_compiler(conf.get('compiler', {}),
                                                     self.lang, self)
        self.runner = self.lang_cls.check_runner(conf.get('runner', {}),
                                                 self.lang, self)

        self.config = conf

        self.data_callback = []
        self.data_t_proc = []

    def compile(self):
        if self.lang_cls.compiled:
            return self.lang_cls.compile(self)

    def run(self, target):
        res = self.lang_cls.run(target, self)
        return res

309
310
311
    def get_default_target(self):
        return self.lang_cls.get_default_target(self)

312
313
314
    # TODO: maybe refactor this.
    # Results and data should not be part of a taskset/task,
    # but rather the Result should be *about* a TaskSet
315
316
    def add_datapoint(self, stdout, stderr, t_proc):
        if self.callback['name'] == 'clock':
317
318
319
320
321
            try:
                dp = float(stdout.split()[-1])
            except:
                raise RuntimeError(
                    "Cannot extract duration from last line of stdout")
322
323
324
325
        else:
            msg = "'%(name)s' callback output processing not implemented" % self.callback
            raise NotImplementedError(msg)
        self.data_callback.append(dp)
326

327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
        self.data_t_proc.append(float(t_proc))

    def aggregate_datapoint(self):
        self.result_callback = Result(self.data_callback,
                                      "{}:{} callback".format(self.taskset.name,
                                                              self.name))
        self.result_t_proc = Result(self.data_t_proc,
                                    "{}:{} t_proc".format(self.taskset.name,
                                                          self.name))

    def get_result(self):
        return SimpleNamespace(callback=self.result_callback,
                               t_proc=self.result_t_proc)

    def __str__(self):
        return self.name


def load_yaml(yaml_s, run_dir):
    config_d = yaml.load(yaml_s)
    tasksets = []
    for name, ts_conf_d in config_d.items():
        tasksets.append(TaskSet.from_config_dict(name, ts_conf_d, run_dir))
    return tasksets
351
352


353
354
355
def load_file(config_file):
    with open(config_file) as fp:
        return load_yaml(fp.read(), Path(config_file).parent)