Commit 51c85e1d authored by John Zhang's avatar John Zhang

refactoring huge data to work with dacapo-dl repository

parent b4d26c06
Pipeline #2945 passed with stage
in 84 minutes and 30 seconds
......@@ -6,6 +6,10 @@ RELEASE PROCEDURES
dacapo.version=3.27
dacapo.nickname=uncalibrated
Also determine the dacapo-dl remote git repository raw data access URL and commit:
dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
dacapo-dl.commit=HEAD
2. Make fresh git clone
git clone https://github.com/dacapobench/dacapobench.git
......
......@@ -22,8 +22,8 @@ public class Avrora extends Benchmark {
String[] args;
public Avrora(Config config, File scratch) throws Exception {
super(config, scratch);
public Avrora(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("avrora.Main", true, loader);
this.method = clazz.getMethod("main", String[].class);
}
......@@ -31,7 +31,7 @@ public class Avrora extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
}
@Override
......
......@@ -27,21 +27,19 @@ size large args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/simplex
"mapVASR.png" digest 0x3086b028662d865e0913602823c0b339dc599b0b,
"bavaria.png" digest 0xf9e58383e2418b8a065e895cbef8132d9099a599;
/** NOTE: huge size not runnable.
* requires later huge data refactoring changes
*/
size huge args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/usMetro.svg", "${SCRATCH}/batik/mapVASR.svg",
"${SCRATCH}/batik/bavaria.svg",
"${SCRATCH}/batik/lakeTauca.svg", "${SCRATCH}/batik/river.svg", "${SCRATCH}/batik/tenerife.svg",
"${SCRATCH}/batik/france.svg", "${SCRATCH}/batik/nrwpTopography.svg",
"${SCRATCH}/batik/velhop.svg", "${SCRATCH}/batik/polytopeCube.svg", "${SCRATCH}/batik/ukRef.svg",
"${SCRATCH}/batik/btw.svg", "${SCRATCH}/batik/lyon.svg", "${SCRATCH}/batik/ohridTopographic.svg",
"${SCRATCH}/batik/dcStreet.svg", "${SCRATCH}/batik/chicoutimi.svg",
"${SCRATCH}/batik/usElection.svg", "${SCRATCH}/batik/moldova.svg", "${SCRATCH}/batik/mapPolitical.svg",
"${SCRATCH}/batik/hague.svg", "${SCRATCH}/batik/locGermany.svg",
"${SCRATCH}/batik/mowbrayOSM.svg", "${SCRATCH}/batik/mapWorld.svg", "${SCRATCH}/batik/sierpinski.svg",
"${SCRATCH}/batik/phi.svg", "${SCRATCH}/batik/simplex.svg", "${SCRATCH}/batik/mapLakes.svg",
"${SCRATCH}/batik/Tettstedskart.svg", "${SCRATCH}/batik/mapSanFran.svg", "${SCRATCH}/batik/serbia.svg"
size huge args "-d","${SCRATCH}","-scriptSecurityOff","${DATA}/batik/usMetro.svg", "${DATA}/batik/mapVASR.svg",
"${DATA}/batik/bavaria.svg",
"${DATA}/batik/lakeTauca.svg", "${DATA}/batik/river.svg", "${DATA}/batik/tenerife.svg",
"${DATA}/batik/france.svg", "${DATA}/batik/nrwpTopography.svg",
"${DATA}/batik/velhop.svg", "${DATA}/batik/polytopeCube.svg", "${DATA}/batik/ukRef.svg",
"${DATA}/batik/btw.svg", "${DATA}/batik/lyon.svg", "${DATA}/batik/ohridTopographic.svg",
"${DATA}/batik/dcStreet.svg", "${DATA}/batik/chicoutimi.svg",
"${DATA}/batik/usElection.svg", "${DATA}/batik/moldova.svg", "${DATA}/batik/mapPolitical.svg",
"${DATA}/batik/hague.svg", "${DATA}/batik/locGermany.svg",
"${DATA}/batik/mowbrayOSM.svg", "${DATA}/batik/mapWorld.svg", "${DATA}/batik/sierpinski.svg",
"${DATA}/batik/phi.svg", "${DATA}/batik/simplex.svg", "${DATA}/batik/mapLakes.svg",
"${DATA}/batik/Tettstedskart.svg", "${DATA}/batik/mapSanFran.svg", "${DATA}/batik/serbia.svg"
output stdout digest 0x3ebb7954cd8d418aad42f3ad6d71a1b512892178,
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"usMetro.png" digest 0xc8c2e79878cbce8ef4ea2e4b617096a9a53b5fe7,
......
......@@ -30,7 +30,7 @@
</target>
<target name="data">
<copy file="${bm-data}/batik.zip" todir="${bm-dat}"/>
<copy file="${bm-data}/${bm-name}.zip" todir="${bm-dat}"/>
</target>
<target name="jar" depends="deps">
......
......@@ -23,8 +23,8 @@ public class Batik extends Benchmark {
private String[] args;
private final Constructor<?> constructor;
public Batik(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Batik(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.apache.batik.apps.rasterizer.Main", true, loader);
this.method = clazz.getMethod("execute");
this.constructor = clazz.getConstructor(String[].class);
......@@ -33,7 +33,7 @@ public class Batik extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
}
public void iterate(String size) throws Exception {
......
......@@ -203,5 +203,4 @@
<delete file="${temp.file}"/>
<jar jarfile="${build.target-jar}" basedir="${bm-output-dir}" update="true"/>
</target>
</project>
......@@ -33,8 +33,8 @@ public class Eclipse extends Benchmark {
private final Method run;
private final Method shutdown;
public Eclipse(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Eclipse(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.eclipse.core.runtime.adaptor.EclipseStarter", true, loader);
this.method = clazz.getMethod("startup", String[].class, Runnable.class);
this.isRunning = clazz.getMethod("isRunning");
......
......@@ -24,8 +24,8 @@ public class Fop extends Benchmark {
private String[] inputs;
private String[] outputs;
public Fop(Config config, File scratch) throws Exception {
super(config, scratch);
public Fop(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("org.apache.fop.cli.Main", true, loader);
this.method = clazz.getMethod("startFOP", new Class[] { String[].class });
}
......@@ -33,7 +33,7 @@ public class Fop extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
collectInputs();
}
......
......@@ -35,8 +35,8 @@ public class H2 extends Benchmark {
private Method iterationTPCC;
private Method postIterationTPCC;
public H2(Config config, File scratch) throws Exception {
super(config, scratch, false);
public H2(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
}
@Override
......@@ -54,7 +54,7 @@ public class H2 extends Benchmark {
TreeMap<String,String> threadMap = new TreeMap<String,String>();
TreeMap<String,String[]> argMap = new TreeMap<String,String[]>();
for(String size: config.getSizes()) {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
argMap.put(size,args);
threadMap.put(size, Integer.toString(config.getThreadCount(size)));
......
......@@ -22,8 +22,8 @@ public class Jython extends Benchmark {
private Method pySetArgsMethod;
public Jython(Config config, File scratch) throws Exception {
super(config, scratch);
public Jython(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("org.python.util.jython", true, loader);
this.method = clazz.getMethod("main", String[].class);
Class<?> pyClass = Class.forName("org.python.core.PySystemState", true, loader);
......@@ -46,7 +46,7 @@ public class Jython extends Benchmark {
* script sees. Hence the Py.setArgv call, followed by the jython.main call.
*/
public void iterate(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
pySetArgsMethod.invoke(null, (Object) args);
method.invoke(null, (Object) args);
}
......
......@@ -43,7 +43,7 @@
</target>
<target name="data" depends="lucene-enwiki-bench-data">
<target name="data">
<mkdir dir="${bm-data}/luindex"/>
<mkdir dir="${bm-data}/luindex/kjv"/>
......@@ -57,7 +57,7 @@
</zip>
</target>
<target name="lucene-enwiki-bench-data-check">
<!-- <target name="lucene-enwiki-bench-data-check">
<available property="lucene-enwiki-bench-data-exists" file="${bm-data}/luindex/enwiki/enwiki.txt"/>
<condition property="lucene-enwiki-bench-data-valid-md5">
<and>
......@@ -74,6 +74,6 @@
</exec>
<mkdir dir="${bm-data}/luindex/enwiki"/>
<move file="${lucene-build-dir}/benchmark/work/enwiki.txt" todir="${bm-data}/luindex/enwiki"/>
</target>
</target> -->
</project>
......@@ -25,13 +25,13 @@ public class Luindex extends Benchmark {
private final Object benchmark;
private final Class<?> clazz;
public Luindex(Config config, File scratch) throws Exception {
super(config, scratch);
public Luindex(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
this.clazz = Class.forName("org.dacapo.luindex.Index", true, loader);
Constructor<?> cons = clazz.getConstructor(File.class);
Constructor<?> cons = clazz.getConstructor(File.class, File.class);
useBenchmarkClassLoader();
try {
benchmark = cons.newInstance(scratch);
benchmark = cons.newInstance(scratch, data);
} finally {
revertClassLoader();
}
......@@ -55,7 +55,7 @@ public class Luindex extends Benchmark {
public void iterate(String size) throws Exception {
if (getVerbose())
System.out.println("luindex benchmark starting");
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
final File INDEX_DIR = new File(scratch, "index");
......
......@@ -13,10 +13,7 @@ size default args "--dirwalk", "${SCRATCH}/luindex/shakespeare","${SCRATCH}/luin
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"index/segments_1" bytes 136;
/* NOTE: not including enwiki.txt in the dacapo.jar
* When the mechanism for dealing with large dataset is in place,
* this should be modified. */
size huge args "--linedoc", "${SCRATCH}/../bms/luindex/data/luindex/enwiki/enwiki.txt"
size huge args "--linedoc", "${DATA}/luindex/enwiki/enwiki-1.txt", "${DATA}/luindex/enwiki/enwiki-2.txt"
output stdout digest 0x3067fa0a00d9311e95cf963dab1dfea1fca4e993,
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"index/segments_1" bytes 137;
......
......@@ -64,9 +64,11 @@ import org.apache.lucene.index.IndexWriterConfig;
public class Index {
private final File scratch;
private final File data;
public Index(File scratch) {
public Index(File scratch, File data) {
this.scratch = scratch;
this.data = data;
}
/**
......@@ -83,8 +85,8 @@ public class Index {
System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory");
}
indexDocs(writer, docDir);
File prefix = docDir.getAbsolutePath().contains(scratch.getAbsolutePath()) ? scratch : data;
indexDocs(writer, docDir, prefix);
System.out.println("Optimizing...");
writer.forceMerge(1);
}
......@@ -100,23 +102,24 @@ public class Index {
IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
File txtFile = new File(args[0]);
for (int idx = 0; idx < args.length; idx ++) {
File txtFile = new File(args[idx]);
if (!txtFile.exists() || !txtFile.canRead()) {
System.out.println("Document directory '" + txtFile.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory");
}
if (!txtFile.exists() || !txtFile.canRead()) {
System.out.println("Document directory '" + txtFile.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory");
}
BufferedReader reader = new BufferedReader(new FileReader(txtFile));
reader.readLine(); // skip header line
int nLines = (args.length > 1) ? Integer.parseInt(args[1]) : Integer.MAX_VALUE;
String line = reader.readLine();
int n = 0;
while (line != null && n < nLines) {
System.out.println("adding " + line.substring(0, line.indexOf(SEP)));
writer.addDocument(getLuceneDocFromLine(line));
line = reader.readLine();
n ++;
BufferedReader reader = new BufferedReader(new FileReader(txtFile));
reader.readLine(); // skip header line
String line = reader.readLine();
int n = 0;
while (line != null) {
System.out.println("adding " + line.substring(0, line.indexOf(SEP)));
writer.addDocument(getLuceneDocFromLine(line));
line = reader.readLine();
n ++;
}
}
System.out.println("Optimizing...");
......@@ -164,10 +167,10 @@ public class Index {
* @param file
* @throws IOException
*/
void indexDocs(IndexWriter writer, File file) throws IOException {
void indexDocs(IndexWriter writer, File file, File prefix) throws IOException {
/* Strip the absolute part of the path name from file name output */
int scratchP = scratch.getCanonicalPath().length() + 1;
int prefixIdx = prefix.getCanonicalPath().length() + 1;
/* do not try to index files that cannot be read */
if (file.canRead()) {
......@@ -177,11 +180,11 @@ public class Index {
if (files != null) {
Arrays.sort(files);
for (int i = 0; i < files.length; i++) {
indexDocs(writer, new File(file, files[i]));
indexDocs(writer, new File(file, files[i]), prefix);
}
}
} else {
System.out.println("adding " + file.getCanonicalPath().substring(scratchP));
System.out.println("adding " + file.getCanonicalPath().substring(prefixIdx));
try {
Document doc = new Document();
FieldType docFT = new FieldType();
......
......@@ -20,8 +20,8 @@ import org.dacapo.parser.Config;
public class Lusearch extends org.dacapo.harness.Benchmark {
private final Object benchmark;
public Lusearch(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Lusearch(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.lusearch.Search", true, loader);
this.method = clazz.getMethod("main", String[].class);
Constructor<?> cons = clazz.getConstructor();
......@@ -35,6 +35,6 @@ public class Lusearch extends org.dacapo.harness.Benchmark {
@Override
public void iterate(String size) throws Exception {
method.invoke(benchmark, (Object) (config.preprocessArgs(size, scratch)));
method.invoke(benchmark, (Object) (config.preprocessArgs(size, scratch, data)));
}
}
......@@ -4,7 +4,7 @@ benchmark lusearch
jars "dacapo-lusearch.jar", "lucene-core-7.1.0-SNAPSHOT.jar", "lucene-demo-7.1.0-SNAPSHOT.jar", "lucene-queryparser-7.1.0-SNAPSHOT.jar";
size small args "-index","${SCRATCH}/lusearch/index-default",
"-queries","${SCRATCH}/lusearch/queries-default/query",
"-queries","${SCRATCH}/lusearch/queries-default/",
"-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "8",
"-threads", "${THREADS}"
......@@ -23,7 +23,7 @@ size small args "-index","${SCRATCH}/lusearch/index-default",
"lusearch.out7" digest(text) 0xa4ab2577aa78ead6fec346ee1d8ad1edcec0a55d;
size default args "-index","${SCRATCH}/lusearch/index-default",
"-queries", "${SCRATCH}/lusearch/queries-default/query",
"-queries", "${SCRATCH}/lusearch/queries-default/",
"-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "128",
"-threads", "${THREADS}"
......@@ -161,9 +161,9 @@ size default args "-index","${SCRATCH}/lusearch/index-default",
"lusearch.out126" digest(text) 0x22e7c16878292805cae3579bf735b6a6acac0c3b,
"lusearch.out127" digest(text) 0x90b1433771874e342bb28ed9cc68b5b738197c66;
size huge args "-index","${SCRATCH}/lusearch/index-enwiki",
size huge args "-index","${DATA}/lusearch/index-enwiki",
"-field", "body",
"-queries", "${SCRATCH}/lusearch/queries-enwiki/query",
"-queries", "${DATA}/lusearch/queries-enwiki/",
"-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "2048",
"-threads", "${THREADS}"
......
......@@ -214,7 +214,7 @@ public class Search {
reader = new OneNormsReader(reader, normsField);*/
searcher = new IndexSearcher(reader);
String query = queryBase + (id < 10 ? "00" : (id < 100 ? "0" : "")) + id + ".txt";
String query = queryBase + "query" + (id < 10 ? "00" : (id < 100 ? "0" : "")) + id + ".txt";
in = new BufferedReader(new FileReader(query));
out = new PrintWriter(new BufferedWriter(new FileWriter(outBase + id)));
......
......@@ -32,8 +32,8 @@ public class Pmd extends Benchmark {
String[] args;
public Pmd(Config config, File scratch) throws Exception {
super(config, scratch);
public Pmd(Config config, File scratch, File data) throws Exception {
super(config, scratch, data);
Class<?> clazz = Class.forName("net.sourceforge.pmd.PMD", true, loader);
this.method = clazz.getMethod("main", String[].class);
......@@ -55,7 +55,7 @@ public class Pmd extends Benchmark {
protected void prepare(String size) throws Exception {
super.prepare(size);
args = config.preprocessArgs(size, scratch);
args = config.preprocessArgs(size, scratch, data);
}
......
......@@ -25,8 +25,8 @@ public class Sunflow extends org.dacapo.harness.Benchmark {
private final Method beginMethod;
private final Method endMethod;
public Sunflow(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Sunflow(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.sunflow.Benchmark", true, loader);
this.method = clazz.getMethod("kernelMain");
this.beginMethod = clazz.getMethod("kernelBegin");
......@@ -46,7 +46,7 @@ public class Sunflow extends org.dacapo.harness.Benchmark {
* @param size The "size" of the iteration (small, default, large)
*/
public void preIteration(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
useBenchmarkClassLoader();
try {
sunflow = constructor.newInstance(Integer.parseInt(args[0]),
......
......@@ -35,8 +35,8 @@ public class Tomcat extends Benchmark {
* @param scratch Scratch directory
* @throws Exception When something goes wrong
*/
public Tomcat(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Tomcat(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
this.clazz = Class.forName("org.dacapo.tomcat.Control", true, loader);
this.method = clazz.getMethod("exec", String.class);
......
......@@ -26,8 +26,8 @@ public class Tradebeans extends Benchmark {
private Method initializeMethod;
private Method shutdownMethod;
public Tradebeans(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Tradebeans(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
// Find the launcher
Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader);
......@@ -43,7 +43,7 @@ public class Tradebeans extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
String dtSize = "medium";
if (args.length == 1)
dtSize = args[0];
......
......@@ -24,8 +24,8 @@ public class Tradesoap extends Benchmark {
private Method initializeMethod;
private Method shutdownMethod;
public Tradesoap(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Tradesoap(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader);
this.initializeMethod = clazz.getMethod("initialize", new Class[] { File.class, Integer.TYPE, String.class, Boolean.TYPE });
this.method = clazz.getMethod("performIteration", new Class[] {});
......@@ -39,7 +39,7 @@ public class Tradesoap extends Benchmark {
@Override
protected void prepare(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
String dtSize = "medium";
if (args.length == 1)
dtSize = args[0];
......
......@@ -28,8 +28,8 @@ public class Xalan extends Benchmark {
Object benchmark;
Method createWorkersMethod;
public Xalan(Config config, File scratch) throws Exception {
super(config, scratch, false);
public Xalan(Config config, File scratch, File data) throws Exception {
super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.xalan.XSLTBench", true, loader);
this.method = clazz.getMethod("doWork", int.class);
createWorkersMethod = clazz.getMethod("createWorkers", int.class);
......
......@@ -127,7 +127,7 @@ BUILDING:
<isset property="build.is-official"/>
</condition>
<!-- set target jar name -->
<property name="build.target-jar" value="dacapo.jar"/>
<property name="build.target-jar" value="dacapo-${build.version}.jar"/>
</target>
<target name="init" depends="set-build-version-info">
......@@ -238,14 +238,9 @@ BUILDING:
<!-- =================================
target: dist
================================= -->
<target name="dist" depends="harness,compile">
<move file="${build.target-jar}" tofile="dacapo-${build.version}.jar"/>
</target>
<target name="dist" depends="harness,compile,update-externdata-info"/>
<target name="doc" depends="init,bootstrap,harness" description="make the top level documentation">
<jar />
</target>
<target name="doc" depends="init,bootstrap,harness" description="make the top level documentation"/>
<!-- =================================
......@@ -392,6 +387,7 @@ See ${log.build.report} for a summary of benchmark build status.
</and>
</condition>
</fail>
<!-- <antcall target="update-externdata-md5"/> -->
</sequential>
</macrodef>
......@@ -438,4 +434,20 @@ See ${log.build.report} for a summary of benchmark build status.
<target name="xalan" depends="init,harness">
<build.bm benchmark="xalan"/>
</target>
<target name="update-externdata-info" depends="set-build-version-info">
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/huge-data-md5s.list" dest="${dacapo.build.dir}"/>
<get src="${dacapo-dl.url.raw}/${dacapo-dl.commit}/META-INF/dlfiles.list" dest="${dacapo.build.dir}"/>
<echo message="setting DaCapo-DL-URL-RAW and DaCapo-DL-Commit in manifest"/>
<jar destfile="${build.target-jar}" update="true">
<manifest>
<attribute name="DaCapo-DL-URL-RAW" value="${dacapo-dl.url.raw}"/>
<attribute name="DaCapo-DL-Commit" value="${dacapo-dl.commit}"/>
</manifest>
<metainf dir="${dacapo.build.dir}">
<include name="huge-data-md5s.list"/>
<include name="dlfiles.list"/>
</metainf>
</jar>
</target>
</project>
......@@ -2,6 +2,10 @@
dacapo.version=9.12-MR1
dacapo.nickname=uncalibrated
# External data
dacapo-dl.url.raw=https://gitlab.anu.edu.au/dacapo/anu-dev/dacapo-dl/raw
# dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
dacapo-dl.commit=HEAD
#
# Common properties for the dacapo benchmarks
#
......
......@@ -110,6 +110,11 @@ public abstract class Benchmark {
*/
protected final File scratch;
/**
* The data directory
*/
protected final File data;
/**
* Parsed version of the configuration file for this benchmark
*/
......@@ -204,14 +209,15 @@ public abstract class Benchmark {
*
* @param scratch Scratch directory
*/
public Benchmark(Config config, File scratch) throws Exception {
this(config, scratch, true);
public Benchmark(Config config, File scratch, File data) throws Exception {
this(config, scratch, data, true);
}
public Benchmark(Config config, File scratch, boolean silent) throws Exception {
public Benchmark(Config config, File scratch, File data, boolean silent) throws Exception {
// TODO this is very ugly
Benchmark.silent = silent;
this.scratch = scratch;
this.data = data;
this.config = config;
initialize();
}
......@@ -233,7 +239,7 @@ public abstract class Benchmark {
}
}
prepareJars();
loader = DacapoClassLoader.create(config, scratch);
loader = DacapoClassLoader.create(config, scratch, data);
prepare();
}
......@@ -248,7 +254,11 @@ public abstract class Benchmark {
file.mkdir();
if (config.jars != null) {
for (int i = 0; i < config.jars.length; i++) {
extractFileResource("jar/" + config.jars[i], scratch);
try {
extractFileResource("jar/" + config.jars[i], scratch);
} catch (DacapoException e) {
// ignore
}
}
}
}
......@@ -258,7 +268,12 @@ public abstract class Benchmark {
* <code>data/<i>name</i>.zip</code> into the scratch directory.
*/
protected void prepare() throws Exception {
unpackZipFileResource("dat/" + config.name + ".zip", scratch);
// the data zip may not exist, if data is packaged externally
try {
unpackZipFileResource("dat/" + config.name + ".zip", scratch);
} catch (DacapoException e) {
e.printStackTrace();
}
}
/**
......@@ -281,7 +296,7 @@ public abstract class Benchmark {
*/
public void preIteration(String size) throws Exception {
if (verbose) {
String[] args = config.preprocessArgs(size, scratch);
String[] args = config.preprocessArgs(size, scratch, data);
System.out.print("Benchmark parameters: ");
for (int i = 0; i < args.length; i++)
System.out.print(args[i] + " ");
......@@ -602,7 +617,7 @@ public abstract class Benchmark {
* @param destination
* @throws IOException
*/
private static void unpackZipStream(BufferedInputStream inputStream, File destination) throws IOException {
public static void unpackZipStream(BufferedInputStream inputStream, File destination) throws IOException {
ZipInputStream input = new ZipInputStream(inputStream);
ZipEntry entry;
while ((entry = input.getNextEntry()) != null) {
......
......@@ -74,6 +74,8 @@ public class CommandLineArgs {
private static final String OPT_SIZE = "size";
private static final String OPT_SIZES = "sizes";
private static final String OPT_SCRATCH_DIRECTORY = "scratch-directory";
private static final String OPT_EXTDATA_INSTALL = "extdata-install";
private static final String OPT_EXTDATA_SETLOC = "extdata-set-location";
private static final String OPT_CONVERGE = "converge";
private static final String OPT_MAX_ITERATIONS = "max-iterations";
private static final String OPT_VARIANCE = "variance";
......@@ -111,7 +113,9 @@ public class CommandLineArgs {
makeOption("t", OPT_THREAD_COUNT, "Set the thread count to drive the workload (mutually exclusive -k)", "thread_count"),
makeOption("k", OPT_THREAD_FACTOR, "Set the number of threads per CPU to drive the workload (mutually exclusive with -t)", "thread_per_cpu"),
makeOption("f", OPT_TIMEOUT_DIALATION, "Set the time dialation of the timeouts in the benchmark by the given integer factor.", "timeout_dialation"),
makeOption("v", OPT_VERBOSE, "Verbose output", null)
makeOption("v", OPT_VERBOSE, "Verbose output", null),
makeOption(null, OPT_EXTDATA_INSTALL, "Download and install external data.", "intall_path"),
makeOption(null, OPT_EXTDATA_SETLOC, "Path to external data location. Note: this directory should contain \"data\" and \"jar\" sub-directories.", "ext_data_loc")
};
private static CommandLineParser parser = new PosixParser();
......@@ -152,6 +156,18 @@ public class CommandLineArgs {
printUsage();
reportAndExitOk = true;
}
if (line.hasOption(OPT_EXTDATA_INSTALL)) {
String impver = TestHarness.getManifestAttribute("Implementation-Version");
File pathVersioned = new File(line.getOptionValue(OPT_EXTDATA_INSTALL), impver);
ExternData.downloadAndInstall(pathVersioned);
reportAndExitOk = true;
}
if (line.hasOption(OPT_EXTDATA_SETLOC)) {
String impver = TestHarness.getManifestAttribute("Implementation-Version");
File pathVersioned = new File(line.getOptionValue(OPT_EXTDATA_SETLOC), impver);