WARNING! Access to this system is limited to authorised users only.
Unauthorised users may be subject to prosecution.
Unauthorised access to this system is a criminal offence under Australian law (Federal Crimes Act 1914 Part VIA)
It is a criminal offence to:
(1) Obtain access to data without authority. -Penalty 2 years imprisonment.
(2) Damage, delete, alter or insert data without authority. -Penalty 10 years imprisonment.
User activity is monitored and recorded. Anyone using this system expressly consents to such monitoring and recording.

To protect your data, the CISO officer has suggested users to enable 2FA as soon as possible.
Currently 2.7% of users enabled 2FA.

Commit 6254c8dc authored by Zixian Cai's avatar Zixian Cai
Browse files

Rudimentary reporting for local benchmarking

parent b3118de3
import logging
import click
from mubench.models import TaskSet
from mubench.models import LocalProject, Revision
logger = logging.getLogger(__name__)
@click.argument('file', type=click.Path(exists=True))
@click.option('--analyse', default=False)
def local(file, analyse):
ts = TaskSet.from_file(file)
if analyse:
@click.option('--compare', is_flag=True, default=False)
def local(file, compare):
p = LocalProject(file=file)
revision = Revision(p)
report = revision.generate_report(compare=compare)
from .project import BaseProject, GitLabProject
from .project import BaseProject, GitLabProject, LocalProject
from .report import Report
from .result import Result
from .revision import Revision
......@@ -17,3 +17,12 @@ class BaseProject:
class GitLabProject(BaseProject):
def download_revision(self):
class LocalProject(BaseProject):
def __init__(self, **kwargs):
def task_set(self):
from mubench.models import TaskSet
return TaskSet.from_file(self.file)
def get_prev_result(revision, task):
class Report:
Collection of results corresponding to a Revision
def __init__(self, results, revision, compare):
self.results = results
if compare:
for result in self.results:
result.compare(get_prev_result(revision, result))
def print(self):
for task in self.results:
......@@ -14,9 +14,14 @@ class Result:
self.median = median(data_points)
self.std = std(data_points)
def compare(self, prev_revision):
prev_mean = prev_revision.mean
def compare(self, prev_res):
prev_mean = prev_res.mean
decreased = significant_decrease(prev_mean, self.raw)
increased = significant_increase(prev_mean, self.raw)
changed = significant_change(prev_mean, self.raw)
return decreased, changed, increased
def __str__(self):
return "Mean: {}\nMedian: {}\nStd: {}".format(self.mean,
from mubench.models import Report
class Revision:
A revision is a snapshot of a project.
It has a hash string, time of creation, and other metadata (branch, etc.).
def __init__(self):
def __init__(self, proj):
self.proj = proj
def run_tasks(self):
Run tasks and store the result as attribute
ts = self.proj.task_set()
self.results = {task: task.get_result() for task in ts.tasks}
def store_result(self):
......@@ -20,12 +25,12 @@ class Revision:
def generate_report(self):
def generate_report(self, compare=True):
Generate result from history data points and store it in database
return Report(self.results, revision=self, compare=compare)
def get_report(self):
......@@ -90,7 +90,8 @@ class Task:
def get_result(self):
if self.result_file.exists():
with open(self.result_file, "r") as result_file:
return Result(map(float, result_file.read().splitlines()))
return Result(
[float(r) for r in result_file.read().splitlines()])
logger.warning("Result {} doesn't exist")
return None
......@@ -35,5 +35,6 @@ def get_callback(cb_name):
NOTE: new Callback class implementations should be added here
from callback.cb_print import PrintCallback
from callback.cb_time import TimeCallback
cb_cls = locals()[cb_name]
return cb_cls
import time
from callback import Callback
class TimeCallback(Callback):
def __init__(self, param_s):
print("initialised PrintCallback with: %s" % param_s)
self.data = []
self.time = 0
def begin(self, **kwds):
print("begin measurement")
self.time = time.time()
def end(self, **kwds):
print("end measurement")
self.data.append(time.time() - self.time)
def report(self, file_name):
with open(file_name, 'w') as fp:
fp.writelines("\n".join(map(str, self.data)))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment