...
 
Commits (5)
......@@ -42,6 +42,13 @@ fop:
after_script:
- git clean -df -e "downloads"
graphchi:
script:
- ant -Dbuild.target-jar=dacapo.jar graphchi
- java -jar dacapo.jar graphchi -s livejournal
after_script:
- git clean -df -e "downloads"
h2:
script:
- ant -Dbuild.target-jar=dacapo.jar h2
......
......@@ -6,6 +6,10 @@ RELEASE PROCEDURES
dacapo.version=3.27
dacapo.nickname=uncalibrated
Also determine the dacapo-dl remote git repository raw data access URL and commit:
dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
dacapo-dl.commit=HEAD
2. Make fresh git clone
git clone https://github.com/dacapobench/dacapobench.git
......
......@@ -22,8 +22,8 @@ public class Avrora extends Benchmark {
String[] args;
public Avrora(Config config, File scratch) throws Exception {
super(config, scratch);
public Avrora(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("avrora.Main", true, loader);
this.method = clazz.getMethod("main", String[].class);
}
......@@ -31,7 +31,7 @@ public class Avrora extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
}
@Override
......
......@@ -27,21 +27,19 @@ size large args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/simplex
"mapVASR.png" digest 0x3086b028662d865e0913602823c0b339dc599b0b,
"bavaria.png" digest 0xf9e58383e2418b8a065e895cbef8132d9099a599;
/** NOTE: huge size not runnable.
* requires later huge data refactoring changes
*/
size huge args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/usMetro.svg", "${SCRATCH}/batik/mapVASR.svg",
"${SCRATCH}/batik/bavaria.svg",
"${SCRATCH}/batik/lakeTauca.svg", "${SCRATCH}/batik/river.svg", "${SCRATCH}/batik/tenerife.svg",
"${SCRATCH}/batik/france.svg", "${SCRATCH}/batik/nrwpTopography.svg",
"${SCRATCH}/batik/velhop.svg", "${SCRATCH}/batik/polytopeCube.svg", "${SCRATCH}/batik/ukRef.svg",
"${SCRATCH}/batik/btw.svg", "${SCRATCH}/batik/lyon.svg", "${SCRATCH}/batik/ohridTopographic.svg",
"${SCRATCH}/batik/dcStreet.svg", "${SCRATCH}/batik/chicoutimi.svg",
"${SCRATCH}/batik/usElection.svg", "${SCRATCH}/batik/moldova.svg", "${SCRATCH}/batik/mapPolitical.svg",
"${SCRATCH}/batik/hague.svg", "${SCRATCH}/batik/locGermany.svg",
"${SCRATCH}/batik/mowbrayOSM.svg", "${SCRATCH}/batik/mapWorld.svg", "${SCRATCH}/batik/sierpinski.svg",
"${SCRATCH}/batik/phi.svg", "${SCRATCH}/batik/simplex.svg", "${SCRATCH}/batik/mapLakes.svg",
"${SCRATCH}/batik/Tettstedskart.svg", "${SCRATCH}/batik/mapSanFran.svg", "${SCRATCH}/batik/serbia.svg"
size huge args "-d","${SCRATCH}","-scriptSecurityOff","${DATA}/batik/usMetro.svg", "${DATA}/batik/mapVASR.svg",
"${DATA}/batik/bavaria.svg",
"${DATA}/batik/lakeTauca.svg", "${DATA}/batik/river.svg", "${DATA}/batik/tenerife.svg",
"${DATA}/batik/france.svg", "${DATA}/batik/nrwpTopography.svg",
"${DATA}/batik/velhop.svg", "${DATA}/batik/polytopeCube.svg", "${DATA}/batik/ukRef.svg",
"${DATA}/batik/btw.svg", "${DATA}/batik/lyon.svg", "${DATA}/batik/ohridTopographic.svg",
"${DATA}/batik/dcStreet.svg", "${DATA}/batik/chicoutimi.svg",
"${DATA}/batik/usElection.svg", "${DATA}/batik/moldova.svg", "${DATA}/batik/mapPolitical.svg",
"${DATA}/batik/hague.svg", "${DATA}/batik/locGermany.svg",
"${DATA}/batik/mowbrayOSM.svg", "${DATA}/batik/mapWorld.svg", "${DATA}/batik/sierpinski.svg",
"${DATA}/batik/phi.svg", "${DATA}/batik/simplex.svg", "${DATA}/batik/mapLakes.svg",
"${DATA}/batik/Tettstedskart.svg", "${DATA}/batik/mapSanFran.svg", "${DATA}/batik/serbia.svg"
output stdout digest 0x3ebb7954cd8d418aad42f3ad6d71a1b512892178,
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"usMetro.png" digest 0xc8c2e79878cbce8ef4ea2e4b617096a9a53b5fe7,
......
......@@ -30,7 +30,7 @@
</target>
<target name="data">
<copy file="${bm-data}/batik.zip" todir="${bm-dat}"/>
<copy file="${bm-data}/${bm-name}.zip" todir="${bm-dat}"/>
</target>
<target name="jar" depends="deps">
......
......@@ -23,8 +23,8 @@ public class Batik extends Benchmark {
private String[] args;
private final Constructor<?> constructor;
public Batik(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Batik(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.apache.batik.apps.rasterizer.Main", true, loader);
this.method = clazz.getMethod("execute");
this.constructor = clazz.getConstructor(String[].class);
......@@ -33,7 +33,7 @@ public class Batik extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
}
public void iterate(String size) throws Exception {
......
......@@ -203,5 +203,4 @@
<delete file="${temp.file}"/>
<jar jarfile="${build.target-jar}" basedir="${bm-output-dir}" update="true"/>
</target>
</project>
......@@ -33,8 +33,8 @@ public class Eclipse extends Benchmark {
private final Method run;
private final Method shutdown;
public Eclipse(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Eclipse(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.eclipse.core.runtime.adaptor.EclipseStarter", true, loader);
this.method = clazz.getMethod("startup", String[].class, Runnable.class);
this.isRunning = clazz.getMethod("isRunning");
......
......@@ -24,8 +24,8 @@ public class Fop extends Benchmark {
private String[] inputs;
private String[] outputs;
public Fop(Config config, File scratch) throws Exception {
super(config, scratch);
public Fop(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("org.apache.fop.cli.Main", true, loader);
this.method = clazz.getMethod("startFOP", new Class[] { String[].class });
}
......@@ -33,7 +33,7 @@ public class Fop extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
collectInputs();
}
......
<!--
* Copyright (c) 2009 The Australian National University.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Apache License v2.0.
* You may obtain the license at
*
* http://www.opensource.org/licenses/apache2.0.php
-->
<project name="graphchi" default="all" basedir="../..">
<description>DaCapo GraphChi benchmark</description>
<property file="dacapo.properties"/>
<property name="bm-name" value="graphchi"/>
<property name="bm-version" value="0.2.2"/>
<property name="bm-src" value="${bm-name}-java-src-${bm-version}.tar.gz"/>
<property name="bm-git-url" value="${github.url}/GraphChi/graphchi-java"/>
<property name="bm-git-commit" value="e5136d52512823fb34594e2851cbb23d62dd7ac8"/>
<property name="netflix-mm-dl" value="netflix_mm.gz"/>
<property name="netflix-mm-url" value="https://courses.cs.washington.edu/courses/cse547/16sp/datasets"/>
<property name="livejournal-dl" value="soc-LiveJournal1.txt.gz"/>
<property name="livejournal-url" value="https://snap.stanford.edu/data"/>
<import file="../common.xml"/>
<property name="mvn-home" location="${toolsdir}/apache-maven-${mvn.version}/bin"/>
<property name="mvn-exe" value="${mvn-home}/mvn"/>
<condition property="mvn-exe" value="${mvn-home}/mvn.bat">
<os family="windows"/>
</condition>
<target name="source">
<antcall target="check-source">
<param name="target-dir" value="${bm-downloads}"/>
<param name="target-url" value="${netflix-mm-url}"/>
<param name="target-file" value="${netflix-mm-dl}"/>
</antcall>
<antcall target="check-source">
<param name="target-dir" value="${bm-downloads}"/>
<param name="target-url" value="${livejournal-url}"/>
<param name="target-file" value="${livejournal-dl}"/>
</antcall>
</target>
<target name="unpack" depends="untar, patch"/>
<target name="bm-build">
<mkdir dir="${bm-build-dir}"/>
<exec executable="${mvn-exe}" dir="${bm-build-dir}" failonerror="yes">
<arg line="assembly:assembly -DdescriptorId=jar-with-dependencies"/>
</exec>
</target>
<target name="data">
<mkdir dir="${bm-data}/${bm-name}"/>
<gunzip src="${bm-downloads}/${netflix-mm-dl}" dest="${bm-data}/${bm-name}"/>
<gunzip src="${bm-downloads}/${livejournal-dl}" dest="${bm-data}/${bm-name}"/>
<zip destfile="${bm-dat}/${bm-name}.zip">
<fileset dir="${bm-data}/" includes="${bm-name}/**"/>
</zip>
<delete dir="${bm-data}/${bm-name}"/>
</target>
<target name="jar">
<copy file="${bm-build-dir}/target/${bm-name}-java-${bm-version}-jar-with-dependencies.jar"
tofile="${bm-jars}/${bm-name}-${bm-version}.jar"/>
</target>
</project>
<HTML>
<HEAD>
<TITLE>DaCapo Benchmarks: GraphChi</TITLE>
<META HTTP-EQUIV="Description" NAME="DaCapo Benchmark: GraphChi"
CONTENT="The DaCapo Benchmark: GraphChi.">
<META HTTP-EQUIV="Keywords" NAME="Keywords"
CONTENT="dacapo benchmarks performance java graphchi">
<LINK REL="stylesheet" TYPE="text/css" HREF="../dacapo.css"/>
</HEAD>
<BODY>
<CENTER><H3>GraphChi</H3></CENTER>
<H4>Description:</H4>
<H4>Characteristics:</H4>
<H4>References:</H4>
<H4>Harness:</H4>
<A HREF="index.html" TARGET="_top">Javadoc for fop harness.</A>
</BODY>
</HTML>
\ No newline at end of file
f5d4925861787fca675520cddee88126
\ No newline at end of file
benchmark graphchi
class org.dacapo.harness.GraphChi
thread-model per_cpu
jars "graphchi-0.2.2.jar";
// TODO: calibrate & validate
size netflix args "ALSMatrixFactorization", "${SCRATCH}/graphchi/netflix_mm", "10";
size livejournal args "Pagerank", "${SCRATCH}/graphchi/soc-LiveJournal1.txt", "10", "edgelist";
description
short "A disk-based graph computation engine.",
long "A disk-based system for computing efficiently on graphs with billions of edges, by using parallel sliding windows method on smaller shards of graphs.",
copyright "Copyright 2012 Aapo Kyrola, Guy Blelloch, Carlos Guestrin / Carnegie Mellon University",
license "Apache License, Version 2.0",
url "https://github.com/GraphChi/graphchi-java",
version "0.2.2",
threads "Explicitly multi-threaded, scaled to available processors.";
diff --git orig/pom.xml ./pom.xml
index 52d0b86..1a7857a 100644
--- orig/pom.xml
+++ ./pom.xml
@@ -47,7 +47,7 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>2.9.0-1</version>
+ <version>2.11.2</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
@@ -73,11 +73,6 @@
<artifactId>hadoop-core</artifactId>
<version>0.20.2</version>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math</artifactId>
- <version>2.0</version>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math</artifactId>
diff --git orig/src/main/java/edu/cmu/graphchi/ChiLogger.java ./src/main/java/edu/cmu/graphchi/ChiLogger.java
index fd67754..0c7ac42 100644
--- orig/src/main/java/edu/cmu/graphchi/ChiLogger.java
+++ ./src/main/java/edu/cmu/graphchi/ChiLogger.java
@@ -63,18 +63,18 @@ public class ChiLogger {
StringBuilder sb = new StringBuilder();
// Minimize memory allocations here.
- dat.setTime(record.getMillis());
- args[0] = dat;
-
-
- // Date and time
- StringBuffer text = new StringBuffer();
- if (formatter == null) {
- formatter = new MessageFormat(format);
- }
- formatter.format(args, text, null);
- sb.append(text);
- sb.append(" ");
+ // dat.setTime(record.getMillis());
+ // args[0] = dat;
+
+
+ // // Date and time
+ // StringBuffer text = new StringBuffer();
+ // if (formatter == null) {
+ // formatter = new MessageFormat(format);
+ // }
+ // formatter.format(args, text, null);
+ // sb.append(text);
+ // sb.append(" ");
// Logger name
/*
* Copyright (c) 2009 The Australian National University.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Apache License v2.0.
* You may obtain the license at
*
* http://www.opensource.org/licenses/apache2.0.php
*/
package org.dacapo.harness;
import java.io.File;
import java.util.Arrays;
import org.dacapo.harness.Benchmark;
import org.dacapo.parser.Config;
public class GraphChi extends Benchmark {
private String[] args;
private Class<?> cls;
public GraphChi(Config config, File scratch) throws Exception {
super(config, scratch, false);
}
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
String[] config_args = config.preprocessArgs(size, scratch);
String strAppClsName = config_args[0];
args = Arrays.copyOfRange(config_args, 1, config_args.length);
cls = Class.forName("edu.cmu.graphchi.apps." + strAppClsName, true, loader);
this.method = cls.getMethod("main", new Class[] { String[].class });
}
public void iterate(String size) throws Exception {
method.invoke(null, (Object)args);
}
}
......@@ -35,8 +35,8 @@ public class H2 extends Benchmark {
private Method iterationTPCC;
private Method postIterationTPCC;
public H2(Config config, File scratch) throws Exception {
super(config, scratch, false);
public H2(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
}
@Override
......@@ -54,7 +54,7 @@ public class H2 extends Benchmark {
TreeMap<String,String> threadMap = new TreeMap<String,String>();
TreeMap<String,String[]> argMap = new TreeMap<String,String[]>();
for(String size: config.getSizes()) {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
argMap.put(size,args);
threadMap.put(size, Integer.toString(config.getThreadCount(size)));
......
......@@ -22,8 +22,8 @@ public class Jython extends Benchmark {
private Method pySetArgsMethod;
public Jython(Config config, File scratch) throws Exception {
super(config, scratch);
public Jython(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("org.python.util.jython", true, loader);
this.method = clazz.getMethod("main", String[].class);
Class<?> pyClass = Class.forName("org.python.core.PySystemState", true, loader);
......@@ -46,7 +46,7 @@ public class Jython extends Benchmark {
* script sees. Hence the Py.setArgv call, followed by the jython.main call.
*/
public void iterate(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
pySetArgsMethod.invoke(null, (Object) args);
method.invoke(null, (Object) args);
}
......
......@@ -43,7 +43,7 @@
</target>
<target name="data" depends="lucene-enwiki-bench-data">
<target name="data">
<mkdir dir="${bm-data}/luindex"/>
<mkdir dir="${bm-data}/luindex/kjv"/>
......@@ -57,7 +57,7 @@
</zip>
</target>
<target name="lucene-enwiki-bench-data-check">
<!-- <target name="lucene-enwiki-bench-data-check">
<available property="lucene-enwiki-bench-data-exists" file="${bm-data}/luindex/enwiki/enwiki.txt"/>
<condition property="lucene-enwiki-bench-data-valid-md5">
<and>
......@@ -74,6 +74,6 @@
</exec>
<mkdir dir="${bm-data}/luindex/enwiki"/>
<move file="${lucene-build-dir}/benchmark/work/enwiki.txt" todir="${bm-data}/luindex/enwiki"/>
</target>
</target> -->
</project>
......@@ -25,13 +25,13 @@ public class Luindex extends Benchmark {
private final Object benchmark;
private final Class<?> clazz;
public Luindex(Config config, File scratch) throws Exception {
super(config, scratch);
public Luindex(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
this.clazz = Class.forName("org.dacapo.luindex.Index", true, loader);
Constructor<?> cons = clazz.getConstructor(File.class);
Constructor<?> cons = clazz.getConstructor(File.class, File.class);
useBenchmarkClassLoader();
try {
benchmark = cons.newInstance(scratch);
benchmark = cons.newInstance(scratch, data);
} finally {
revertClassLoader();
}
......@@ -55,7 +55,7 @@ public class Luindex extends Benchmark {
public void iterate(String size) throws Exception {
if (getVerbose())
System.out.println("luindex benchmark starting");
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
final File INDEX_DIR = new File(scratch, "index");
......
......@@ -13,10 +13,7 @@ size default args "--dirwalk", "${SCRATCH}/luindex/shakespeare","${SCRATCH}/luin
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"index/segments_1" bytes 136;
/* NOTE: not including enwiki.txt in the dacapo.jar
* When the mechanism for dealing with large dataset is in place,
* this should be modified. */
size huge args "--linedoc", "${SCRATCH}/../bms/luindex/data/luindex/enwiki/enwiki.txt"
size huge args "--linedoc", "${DATA}/luindex/enwiki/enwiki-1.txt", "${DATA}/luindex/enwiki/enwiki-2.txt"
output stdout digest 0x3067fa0a00d9311e95cf963dab1dfea1fca4e993,
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"index/segments_1" bytes 137;
......
......@@ -64,9 +64,11 @@ import org.apache.lucene.index.IndexWriterConfig;
public class Index {
private final File scratch;
private final File data;
public Index(File scratch) {
public Index(File scratch, File data) {
this.scratch = scratch;
this.data = data;
}
/**
......@@ -83,8 +85,8 @@ public class Index {
System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory");
}
indexDocs(writer, docDir);
File prefix = docDir.getAbsolutePath().contains(scratch.getAbsolutePath()) ? scratch : data;
indexDocs(writer, docDir, prefix);
System.out.println("Optimizing...");
writer.forceMerge(1);
}
......@@ -100,23 +102,24 @@ public class Index {
IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
File txtFile = new File(args[0]);
for (int idx = 0; idx < args.length; idx ++) {
File txtFile = new File(args[idx]);
if (!txtFile.exists() || !txtFile.canRead()) {
System.out.println("Document directory '" + txtFile.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory");
}
if (!txtFile.exists() || !txtFile.canRead()) {
System.out.println("Document directory '" + txtFile.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory");
}
BufferedReader reader = new BufferedReader(new FileReader(txtFile));
reader.readLine(); // skip header line
int nLines = (args.length > 1) ? Integer.parseInt(args[1]) : Integer.MAX_VALUE;
String line = reader.readLine();
int n = 0;
while (line != null && n < nLines) {
System.out.println("adding " + line.substring(0, line.indexOf(SEP)));
writer.addDocument(getLuceneDocFromLine(line));
line = reader.readLine();
n ++;
BufferedReader reader = new BufferedReader(new FileReader(txtFile));
reader.readLine(); // skip header line
String line = reader.readLine();
int n = 0;
while (line != null) {
System.out.println("adding " + line.substring(0, line.indexOf(SEP)));
writer.addDocument(getLuceneDocFromLine(line));
line = reader.readLine();
n ++;
}
}
System.out.println("Optimizing...");
......@@ -164,10 +167,10 @@ public class Index {
* @param file
* @throws IOException
*/
void indexDocs(IndexWriter writer, File file) throws IOException {
void indexDocs(IndexWriter writer, File file, File prefix) throws IOException {
/* Strip the absolute part of the path name from file name output */
int scratchP = scratch.getCanonicalPath().length() + 1;
int prefixIdx = prefix.getCanonicalPath().length() + 1;
/* do not try to index files that cannot be read */
if (file.canRead()) {
......@@ -177,11 +180,11 @@ public class Index {
if (files != null) {
Arrays.sort(files);
for (int i = 0; i < files.length; i++) {
indexDocs(writer, new File(file, files[i]));
indexDocs(writer, new File(file, files[i]), prefix);
}
}
} else {
System.out.println("adding " + file.getCanonicalPath().substring(scratchP));
System.out.println("adding " + file.getCanonicalPath().substring(prefixIdx));
try {
Document doc = new Document();
FieldType docFT = new FieldType();
......
......@@ -20,8 +20,8 @@ import org.dacapo.parser.Config;
public class Lusearch extends org.dacapo.harness.Benchmark {
private final Object benchmark;
public Lusearch(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Lusearch(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.lusearch.Search", true, loader);
this.method = clazz.getMethod("main", String[].class);
Constructor<?> cons = clazz.getConstructor();
......@@ -35,6 +35,6 @@ public class Lusearch extends org.dacapo.harness.Benchmark {
@Override
public void iterate(String size) throws Exception {
method.invoke(benchmark, (Object) (config.preprocessArgs(size, scratch)));
method.invoke(benchmark, (Object) (config.preprocessArgs(size, scratch, data)));
}
}
......@@ -4,7 +4,7 @@ benchmark lusearch
jars "dacapo-lusearch.jar", "lucene-core-7.1.0-SNAPSHOT.jar", "lucene-demo-7.1.0-SNAPSHOT.jar", "lucene-queryparser-7.1.0-SNAPSHOT.jar";
size small args "-index","${SCRATCH}/lusearch/index-default",
"-queries","${SCRATCH}/lusearch/queries-default/query",
"-queries","${SCRATCH}/lusearch/queries-default/",
"-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "8",
"-threads", "${THREADS}"
......@@ -23,7 +23,7 @@ size small args "-index","${SCRATCH}/lusearch/index-default",
"lusearch.out7" digest(text) 0xa4ab2577aa78ead6fec346ee1d8ad1edcec0a55d;
size default args "-index","${SCRATCH}/lusearch/index-default",
"-queries", "${SCRATCH}/lusearch/queries-default/query",
"-queries", "${SCRATCH}/lusearch/queries-default/",
"-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "128",
"-threads", "${THREADS}"
......@@ -161,9 +161,9 @@ size default args "-index","${SCRATCH}/lusearch/index-default",
"lusearch.out126" digest(text) 0x22e7c16878292805cae3579bf735b6a6acac0c3b,
"lusearch.out127" digest(text) 0x90b1433771874e342bb28ed9cc68b5b738197c66;
size huge args "-index","${SCRATCH}/lusearch/index-enwiki",
size huge args "-index","${DATA}/lusearch/index-enwiki",
"-field", "body",
"-queries", "${SCRATCH}/lusearch/queries-enwiki/query",
"-queries", "${DATA}/lusearch/queries-enwiki/",
"-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "2048",
"-threads", "${THREADS}"
......
......@@ -214,7 +214,7 @@ public class Search {
reader = new OneNormsReader(reader, normsField);*/
searcher = new IndexSearcher(reader);
String query = queryBase + (id < 10 ? "00" : (id < 100 ? "0" : "")) + id + ".txt";
String query = queryBase + "query" + (id < 10 ? "00" : (id < 100 ? "0" : "")) + id + ".txt";
in = new BufferedReader(new FileReader(query));
out = new PrintWriter(new BufferedWriter(new FileWriter(outBase + id)));
......
......@@ -32,8 +32,8 @@ public class Pmd extends Benchmark {
String[] args;
public Pmd(Config config, File scratch) throws Exception {
super(config, scratch);
public Pmd(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("net.sourceforge.pmd.PMD", true, loader);
this.method = clazz.getMethod("main", String[].class);
......@@ -55,7 +55,7 @@ public class Pmd extends Benchmark {
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
}
......
......@@ -25,8 +25,8 @@ public class Sunflow extends org.dacapo.harness.Benchmark {
private final Method beginMethod;
private final Method endMethod;
public Sunflow(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Sunflow(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.sunflow.Benchmark", true, loader);
this.method = clazz.getMethod("kernelMain");
this.beginMethod = clazz.getMethod("kernelBegin");
......@@ -46,7 +46,7 @@ public class Sunflow extends org.dacapo.harness.Benchmark {
* @param size The "size" of the iteration (small, default, large)
*/
public void preIteration(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
useBenchmarkClassLoader();
try {
sunflow = constructor.newInstance(Integer.parseInt(args[0]),
......
......@@ -35,8 +35,8 @@ public class Tomcat extends Benchmark {
* @param scratch Scratch directory
* @throws Exception When something goes wrong
*/
public Tomcat(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Tomcat(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
this.clazz = Class.forName("org.dacapo.tomcat.Control", true, loader);
this.method = clazz.getMethod("exec", String.class);
......
......@@ -26,8 +26,8 @@ public class Tradebeans extends Benchmark {
private Method initializeMethod;
private Method shutdownMethod;
public Tradebeans(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Tradebeans(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
// Find the launcher
Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader);
......@@ -43,7 +43,7 @@ public class Tradebeans extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
String dtSize = "medium";
if (args.length == 1)
dtSize = args[0];
......
......@@ -24,8 +24,8 @@ public class Tradesoap extends Benchmark {
private Method initializeMethod;
private Method shutdownMethod;
public Tradesoap(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Tradesoap(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader);
this.initializeMethod = clazz.getMethod("initialize", new Class[] { File.class, Integer.TYPE, String.class, Boolean.TYPE });
this.method = clazz.getMethod("performIteration", new Class[] {});
......@@ -39,7 +39,7 @@ public class Tradesoap extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
String dtSize = "medium";
if (args.length == 1)
dtSize = args[0];
......
......@@ -28,8 +28,8 @@ public class Xalan extends Benchmark {
Object benchmark;
Method createWorkersMethod;
public Xalan(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Xalan(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.xalan.XSLTBench", true, loader);
this.method = clazz.getMethod("doWork", int.class);
createWorkersMethod = clazz.getMethod("createWorkers", int.class);
......
......@@ -127,7 +127,7 @@ BUILDING:
<isset property="build.is-official"/>
</condition>
<!-- set target jar name -->
<property name="build.target-jar" value="dacapo.jar"/>
<property name="build.target-jar" value="dacapo-${build.version}.jar"/>
</target>
<target name="init" depends="set-build-version-info">
......@@ -238,14 +238,9 @@ BUILDING:
<!-- =================================
target: dist
================================= -->
<target name="dist" depends="harness,compile">
<move file="${build.target-jar}" tofile="dacapo-${build.version}.jar"/>
</target>
<target name="dist" depends="harness,compile,update-externdata-info"/>
<target name="doc" depends="init,bootstrap,harness" description="make the top level documentation">
<jar />
</target>
<target name="doc" depends="init,bootstrap,harness" description="make the top level documentation"/>
<!-- =================================
......@@ -392,6 +387,7 @@ See ${log.build.report} for a summary of benchmark build status.
</and>
</condition>
</fail>
<!-- <antcall target="update-externdata-md5"/> -->
</sequential>
</macrodef>
......@@ -410,6 +406,9 @@ See ${log.build.report} for a summary of benchmark build status.
<target name="fop" depends="init,harness">
<build.bm benchmark="fop"/>
</target>
<target name="graphchi" depends="init,harness">
<build.bm benchmark="graphchi"/>
</target>
<target name="jython" depends="init,harness">
<build.bm benchmark="jython"/>
</target>
......@@ -438,4 +437,20 @@ See ${log.build.report} for a summary of benchmark build status.
<target name="xalan" depends="init,harness">
<build.bm benchmark="xalan"/>
</target>
<target name="update-externdata-info" depends="set-build-version-info">
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
<echo message="setting DaCapo-DL-URL-RAW and DaCapo-DL-Commit in manifest"/>
<jar destfile="${build.target-jar}" update="true">
<manifest>
<attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/>
<attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/>
</manifest>
<metainf dir="${dacapo.build.dir}">
<include name="huge-data-md5s.list"/>
<include name="dlfiles.list"/>
</metainf>
</jar>
</target>
</project>
......@@ -2,6 +2,10 @@
dacapo.version=9.12-MR1
dacapo.nickname=uncalibrated
# External data
dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
# dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
dacapo-dl.commit=HEAD
#
# Common properties for the dacapo benchmarks
#
......
......@@ -110,6 +110,11 @@ public abstract class Benchmark {
*/
protected final File scratch;
/**
* The data directory
*/
protected final File data;
/**
* Parsed version of the configuration file for this benchmark
*/
......@@ -204,14 +209,15 @@ public abstract class Benchmark {
*
* @param scratch Scratch directory
*/
public Benchmark(Config config, File scratch) throws Exception {
this(config, scratch, true);
public Benchmark(Config config, File scratch, File data) throws Exception {
this(config, scratch, data, true);
}
public Benchmark(Config config, File scratch, boolean silent) throws Exception {
public Benchmark(Config config, File scratch, File data, boolean silent) throws Exception {
// TODO this is very ugly
Benchmark.silent = silent;
this.scratch = scratch;
this.data = data;
this.config = config;
initialize();
}
......@@ -233,7 +239,7 @@ public abstract class Benchmark {
}
}
prepareJars();
loader = DacapoClassLoader.create(config, scratch);
loader = DacapoClassLoader.create(config, scratch, data);
prepare();
}
......@@ -248,7 +254,11 @@ public abstract class Benchmark {
file.mkdir();
if (config.jars != null) {
for (int i = 0; i < config.jars.length; i++) {
extractFileResource("jar/" + config.jars[i], scratch);
try {
extractFileResource("jar/" + config.jars[i], scratch);
} catch (DacapoException e) {
// ignore
}
}
}
}
......@@ -258,7 +268,12 @@ public abstract class Benchmark {
* <code>data/<i>name</i>.zip</code> into the scratch directory.
*/
protected void prepare() throws Exception {
unpackZipFileResource("dat/" + config.name + ".zip", scratch);
// the data zip may not exist, if data is packaged externally
try {
unpackZipFileResource("dat/" + config.name + ".zip", scratch);
} catch (DacapoException e) {
e.printStackTrace();
}
}
/**
......@@ -281,7 +296,7 @@ public abstract class Benchmark {
*/
public void preIteration(String size) throws Exception {
if (verbose) {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
System.out.print("Benchmark parameters: ");
for (int i = 0; i < args.length; i++)
System.out.print(args[i] + " ");
......@@ -602,7 +617,7 @@ public abstract class Benchmark {
* @param destination
* @throws IOException
*/
private static void unpackZipStream(BufferedInputStream inputStream, File destination) throws IOException {
public static void unpackZipStream(BufferedInputStream inputStream, File destination) throws IOException {
ZipInputStream input = new ZipInputStream(inputStream);
ZipEntry entry;
while ((entry = input.getNextEntry()) != null) {
......
......@@ -74,6 +74,8 @@ public class CommandLineArgs {
private static final String OPT_SIZE = "size";
private static final String OPT_SIZES = "sizes";
private static final String OPT_SCRATCH_DIRECTORY = "scratch-directory";
private static final String OPT_EXTDATA_INSTALL = "extdata-install";
private static final String OPT_EXTDATA_SETLOC = "extdata-set-location";
private static final String OPT_CONVERGE = "converge";
private static final String OPT_MAX_ITERATIONS = "max-iterations";
private static final String OPT_VARIANCE = "variance";
......@@ -111,7 +113,9 @@ public class CommandLineArgs {
makeOption("t", OPT_THREAD_COUNT, "Set the thread count to drive the workload (mutually exclusive -k)", "thread_count"),
makeOption("k", OPT_THREAD_FACTOR, "Set the number of threads per CPU to drive the workload (mutually exclusive with -t)", "thread_per_cpu"),
makeOption("f", OPT_TIMEOUT_DIALATION, "Set the time dialation of the timeouts in the benchmark by the given integer factor.", "timeout_dialation"),
makeOption("v", OPT_VERBOSE, "Verbose output", null)
makeOption("v", OPT_VERBOSE, "Verbose output", null),
makeOption(null, OPT_EXTDATA_INSTALL, "Download and install external data.", "intall_path"),
makeOption(null, OPT_EXTDATA_SETLOC, "Path to external data location. Note: this directory should contain \"data\" and \"jar\" sub-directories.", "ext_data_loc")
};
private static CommandLineParser parser = new PosixParser();
......@@ -152,6 +156,18 @@ public class CommandLineArgs {
printUsage();
reportAndExitOk = true;
}
if (line.hasOption(OPT_EXTDATA_INSTALL)) {
String impver = TestHarness.getManifestAttribute("Implementation-Version");
File pathVersioned = new File(line.getOptionValue(OPT_EXTDATA_INSTALL), impver);
ExternData.downloadAndInstall(pathVersioned);
reportAndExitOk = true;
}
if (line.hasOption(OPT_EXTDATA_SETLOC)) {
String impver = TestHarness.getManifestAttribute("Implementation-Version");
File pathVersioned = new File(line.getOptionValue(OPT_EXTDATA_SETLOC), impver);
ExternData.setLocation(pathVersioned, true);
reportAndExitOk = true;
}
if (line.hasOption(OPT_THREAD_FACTOR) && line.hasOption(OPT_THREAD_COUNT)) {
throw new ParseException("Cannot specify -t and -k options.");
}
......
......@@ -35,14 +35,15 @@ public class DacapoClassLoader extends URLClassLoader {
*
* @param config The config file, which contains information about the jars
* this benchmark depends on
* @param scratch The scratch directory (in which the jars will be located)
* @param scratch The scratch directory (in which some of the jars may be located)
* @param extdata The external data directory (in which externally packaged jars will be located)
* @return The class loader in which this benchmark's iterations should
* execute.
*/
public static DacapoClassLoader create(Config config, File scratch) {
public static DacapoClassLoader create(Config config, File scratch, File extdata) {
DacapoClassLoader rtn = null;
try {
URL[] urls = getJars(config, scratch);
URL[] urls = getJars(config, scratch, extdata);
if (Benchmark.getVerbose()) {
System.out.println("Benchmark classpath:");
for (URL url : urls) {
......@@ -87,17 +88,26 @@ public class DacapoClassLoader extends URLClassLoader {
* benchmark
*
* @param config The config file for this benchmark, which lists the jars
* @param scratch The scratch directory, in which the jars will be located
* @param scratch The scratch directory (in which some of the jars may be located)
* @param extdata The external data directory (in which externally packaged jars will be located)
* @return An array of URLs, one URL for each jar
* @throws MalformedURLException
*/
private static URL[] getJars(Config config, File scratch) throws MalformedURLException {
private static URL[] getJars(Config config, File scratch, File extdata) throws MalformedURLException {
List<URL> jars = new ArrayList<URL>();
File jardir = new File(scratch, "jar");
File jardirScratch = new File(scratch, "jar");
File jardirExtData = new File(extdata, "jar");
if (config.jars != null) {
for (int i = 0; i < config.jars.length; i++) {
File jar = new File(jardir, config.jars[i]);
jars.add(jar.toURL());
// first try external data directory, then try scratch if not found.
File jar = new File(jardirExtData, config.jars[i]);
if (!jar.exists()) {
jar = new File(jardirScratch, config.jars[i]);
if (!jar.exists())
ExternData.failExtJarNotFound(jar, extdata);
}
jars.add(jar.toURI().toURL()); // first convert to URI to avoid deprecation
}
}
return jars.toArray(new URL[jars.size()]);
......
/*
* Copyright (c) 2006, 2009 The Australian National University.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Apache License v2.0.
* You may obtain the license at
*
* http://www.opensource.org/licenses/apache2.0.php
*/
package org.dacapo.harness;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.FileOutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.channels.ReadableByteChannel;
import java.nio.charset.Charset;
import java.nio.channels.Channels;
import java.util.Properties;
import javax.xml.bind.DatatypeConverter;
import java.security.MessageDigest;
public class ExternData {
/**
* Helper functions managing big data sets.
* Using a "data installation directory" to keep
* large data sets.
*/
public static final Path DEFAULT_LOCAL_DACAPO_CONFIG = Paths.get(System.getProperty("user.home"), ".dacapo-config.properties");
public static final String CONFIG_KEY_EXTERN_DATA_LOC = "Extern-Data-Location";
public static final String DACAPO_DL_URL_RAW = "DaCapo-DL-URL-RAW";
public static final String DACAPO_DL_COMMIT = "DaCapo-DL-Commit";
private static String getDefaultLocation() {
try {
String parent = new File(ExternData.class.getProtectionDomain().getCodeSource().getLocation().toURI()).getParent();
return parent;
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
return null; // to satisfy the static checker
}
}
/**
* Retrieve data installation directory from $HOME/.dacapo-config.properties
* If doesn't exist, return the default path (/path/to/dacapo.jar/../)
*/
public static String getLocation() {
File fileProperties = new File(DEFAULT_LOCAL_DACAPO_CONFIG.toString());
Properties props = new Properties();
try {
props.load(new FileReader(fileProperties));
return props.getProperty(CONFIG_KEY_EXTERN_DATA_LOC, getDefaultLocation());
} catch (IOException ioe) {
return getDefaultLocation();
}
}
/**
* Fail function when external data not found.
*/
public static void failExtDataNotFound(String size, File extdata) {
System.err.printf("ERROR: failed to find external data for size '%s'.\n", size);
System.err.printf("Please check that you have installed the external data properly (current: %s)\n", extdata.getAbsolutePath());
System.err.println("Please do one of the following:");
System.err.println(" 1) If you have not installed the large data, run DaCapo with --extdata-install");
System.err.println(" 2) If you have already installed the large data, run DaCapo with --extdata-set-location to correctly identify the location of the external data.");
System.exit(-1);
}
public static void failExtJarNotFound(File extjar, File extdata) {
System.err.printf("ERROR: failed to find jar: %s.\n", extjar.getName());
System.err.printf("Please check that you have installed the external data properly (current: %s)\n", extdata.getAbsolutePath());
System.err.println("Please do one of the following:");
System.err.println(" 1) If you have not installed the large data, run DaCapo with --extdata-install");
System.err.println(" 2) If you have already installed the large data, run DaCapo with --extdata-set-location to correctly identify the location of the external data.");
System.exit(-1);
}
public static void setLocation(File path, boolean md5Check) {
if (md5Check) {
// MD5 check
System.out.printf("Checking MD5 at %s...", path.toString());
if (!checkExtDataDirMD5(path)) {
System.out.println("failed!");
System.err.println("WARNING: MD5 checking failed. Your huge data does not match expected release.");
System.err.println("Please download and install the latest huge data using --extdata-install flag;");
System.err.println("otherwise please note the changes in research publication.");
} else
System.out.println("done!");
}
File fileProperties = new File(DEFAULT_LOCAL_DACAPO_CONFIG.toString());
Properties props = new Properties();
try {
fileProperties.createNewFile(); // create new if does not exist
props.load(new FileReader(fileProperties));
props.setProperty(CONFIG_KEY_EXTERN_DATA_LOC, path.getAbsolutePath());
props.store(new FileWriter(fileProperties), "");
System.out.printf("External data location has been set at %s.\n", path.getAbsolutePath());
} catch (IOException e) {
System.err.printf("IOException when creating/reading file %s: %s\n",
fileProperties.toString(), e.toString());
System.exit(-1);
}
}
private static boolean checkExtDataDirMD5(File dir) {
InputStream in = ClassLoader.getSystemResourceAsStream("META-INF/huge-data-md5s.list");
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
File datDir = new File(dir, "dat");
return reader.lines().map(l -> {
String [] fields = l.split(" ");
String md5Expected = fields[0];
String filePath = fields[1];
try {
String md5 = getMD5(new File(datDir, filePath));
if (!md5.toUpperCase().equals(md5Expected.toUpperCase())) {
return false;
}
} catch (Exception e) {
return false;
}
return true;
}).reduce(true, (a, b) -> a & b);
}
/**
* Download and install external data
*/
public static void downloadAndInstall(File path) {
try {
// create directory if not exist already
if (!path.exists())
path.mkdirs();
// download
URL urlDLRawRoot = appendURL(new URL(TestHarness.getManifestAttribute(DACAPO_DL_URL_RAW)),
TestHarness.getManifestAttribute(DACAPO_DL_COMMIT));
BufferedReader dllistReader = new BufferedReader(new InputStreamReader(
ClassLoader.getSystemResourceAsStream("META-INF/dlfiles.list")));
dllistReader.lines().forEach(s -> {
try {
downloadAndExtractItem(s, urlDLRawRoot, path);
} catch (Exception e) {
System.err.println("Download external data failed.");
System.err.println("Please try again.");
System.exit(-1);
}
});
setLocation(path, false);
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
}
private static URL appendURL(URL url, String relPath) throws MalformedURLException{
return new URL(url.toString() + "/" + relPath);
}
private static void downloadAndExtractItem(String itemRelPath, URL urlDLRawRoot, File localDataPath) throws Exception {
URL urlItem = appendURL(urlDLRawRoot, itemRelPath);
File fileLocalItem = new File(localDataPath, itemRelPath);
if (!fileLocalItem.getParentFile().exists())
fileLocalItem.getParentFile().mkdirs();
System.out.printf("Downloading %s to %s...", urlItem.toString(), fileLocalItem.toString());
try {
ReadableByteChannel rbc = Channels.newChannel(urlItem.openStream());
FileOutputStream fos = new FileOutputStream(fileLocalItem);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
System.out.println("Done.");
System.out.printf("Checking %s MD5...", fileLocalItem.toString());
ReadableByteChannel rbcMD5 = Channels.newChannel(appendURL(urlDLRawRoot, itemRelPath + ".MD5").openStream());
ByteBuffer buf = ByteBuffer.allocate(32); // MD5 is always 32 characters long
rbcMD5.read(buf);
String md5Expect = new String(buf.array(), Charset.forName("ASCII")).toLowerCase();
String md5Actual = getMD5(fileLocalItem).toLowerCase();
if (!md5Expect.equals(md5Actual)) {
System.out.println("Failed!");
System.err.printf("MD5 checking of %s failed: expect %s, got %s\n", fileLocalItem.toString(),
md5Expect, md5Actual);
System.exit(-1);
}
System.out.println("Done.");
if (fileLocalItem.getName().endsWith(".zip")) {
System.out.printf("Extracting %s...", fileLocalItem.toString());
Benchmark.unpackZipStream(new BufferedInputStream(new FileInputStream(fileLocalItem)),
fileLocalItem.getParentFile());
System.out.println("Done.");
}
} catch (Exception e) {
System.out.println("Failed!");
e.printStackTrace();
throw e;
}
}
private static String getMD5(File file) throws Exception{
MessageDigest md = MessageDigest.getInstance("MD5");
byte [] buffer = new byte [1024 * 1024 * 32]; // 32 MB buffer
InputStream stream = Files.newInputStream(file.toPath());
int bytesRead = 0;
while ((bytesRead = stream.read(buffer)) != -1) {
md.update(buffer, 0, bytesRead);
}
return DatatypeConverter.printHexBinary(md.digest());
}
}
......@@ -11,6 +11,7 @@ package org.dacapo.harness;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
......@@ -90,6 +91,7 @@ public class TestHarness {
File scratch = new File(commandLineArgs.getScratchDir());
makeCleanScratch(scratch);
File data = new File(ExternData.getLocation());
// this is not right
Benchmark.setCommandLineOptions(commandLineArgs);
......@@ -153,7 +155,16 @@ public class TestHarness {
}
harness.dump(commandLineArgs.getVerbose());
runBenchmark(scratch, bm, harness);
try {
runBenchmark(scratch, data, bm, harness);
} catch (FileNotFoundException e) {
System.err.printf("ERROR: The following file used by size '%s' could not be found: %s\n", size, e.getMessage());
System.err.printf("Please check that you have downloaded the required data for this size, " +
"and have installed it correctly under %s/%s\n", data.getAbsolutePath(), bm);
System.err.printf("Note: the directory for big data is currently set to: %s\n", data.getAbsolutePath());
System.err.printf("To change this path, please run `java -jar %s -i <new_data_dir>`.\n", args[0]);
System.exit(-1);
}
}
}
} catch (Exception e) {
......@@ -178,6 +189,7 @@ public class TestHarness {
/**
* @param scratch
* @param data
* @param bm
* @param harness
* @param c
......@@ -187,13 +199,13 @@ public class TestHarness {
* @throws InvocationTargetException
* @throws Exception
*/
private static void runBenchmark(File scratch, String bm, TestHarness harness) throws NoSuchMethodException, InstantiationException, IllegalAccessException,
private static void runBenchmark(File scratch, File data, String bm, TestHarness harness) throws NoSuchMethodException, InstantiationException, IllegalAccessException,
InvocationTargetException, Exception {
harness.config.printThreadModel(System.out, commandLineArgs.getSize(), commandLineArgs.getVerbose());
Constructor<?> cons = harness.findClass().getConstructor(new Class[] { Config.class, File.class });
Constructor<?> cons = harness.findClass().getConstructor(new Class[] { Config.class, File.class, File.class });
Benchmark b = (Benchmark) cons.newInstance(new Object[] { harness.config, scratch });
Benchmark b = (Benchmark) cons.newInstance(new Object[] { harness.config, scratch, data });
boolean valid = true;
Callback callback = commandLineArgs.getCallback();
......@@ -257,16 +269,19 @@ public class TestHarness {
return null; // not reached
}
}
private static void setBuildInfo() {
try {
String url = TestHarness.class.getProtectionDomain().getCodeSource().getLocation().getFile();
JarFile jarFile = new JarFile(url.replace("!/harness", "").replace("file:", ""));
Manifest manifest = jarFile.getManifest();
Attributes attributes = manifest.getMainAttributes();
public static String getManifestAttribute(String key) throws IOException {
String url = TestHarness.class.getProtectionDomain().getCodeSource().getLocation().getFile();
JarFile jarFile = new JarFile(url.replace("!/harness", "").replace("file:", ""));
Manifest manifest = jarFile.getManifest();
Attributes attributes = manifest.getMainAttributes();
return attributes.get(new Attributes.Name(key)).toString();
}
String nickname = attributes.get(new Attributes.Name(BUILD_NICKNAME)).toString();
String version = attributes.get(new Attributes.Name(BUILD_VERSION)).toString();
private static void setBuildInfo() {
try {
String nickname = getManifestAttribute(BUILD_NICKNAME);
String version = getManifestAttribute(BUILD_VERSION);
BuildNickName = nickname;
BuildVersion = version;
......
......@@ -24,6 +24,8 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.Vector;
import org.dacapo.harness.ExternData;
/**
* Container class for all options specified in a benchmark's configuration
* file.
......@@ -750,17 +752,25 @@ public class Config {
* <ul>
* <li>${SCRATCH} - replaced with the absolute path name of the scratch
* directory
* <li>${DATA} - replaced with the absolute path name of the big data directory
* <li>${THREADS} - replaced with the specified thread count for the benchmark
* size
* </ul>
*/
public String[] preprocessArgs(String size, File scratch) {
public String[] preprocessArgs(String size, File scratch, File extdata) throws FileNotFoundException {
String[] raw = getArgs(size);
String[] cooked = new String[raw.length];
String dataPath = new File(extdata, "dat").getAbsolutePath();
for (int i = 0; i < raw.length; i++) {
String tmp = raw[i];
tmp = tmp.replace("${SCRATCH}", scratch.getAbsolutePath());
tmp = tmp.replace("${THREADS}", Integer.toString(getThreadCount(size)));
if (tmp.contains("${DATA}")) {
tmp = tmp.replace("${DATA}", dataPath);
File argFile = new File(tmp);
if (!argFile.exists())
ExternData.failExtDataNotFound(size, extdata);
}
cooked[i] = tmp;
}
return cooked;
......