Commit 51c85e1d authored by John Zhang's avatar John Zhang

refactoring huge data to work with dacapo-dl repository

parent b4d26c06
Pipeline #2945 passed with stage
in 84 minutes and 30 seconds
...@@ -6,6 +6,10 @@ RELEASE PROCEDURES ...@@ -6,6 +6,10 @@ RELEASE PROCEDURES
dacapo.version=3.27 dacapo.version=3.27
dacapo.nickname=uncalibrated dacapo.nickname=uncalibrated
Also determine the dacapo-dl remote git repository raw data access URL and commit:
dacapo-dl.url.raw=https://raw.githubusercontent.com/dacapobench/dacapo-dl/
dacapo-dl.commit=HEAD
2. Make fresh git clone 2. Make fresh git clone
git clone https://github.com/dacapobench/dacapobench.git git clone https://github.com/dacapobench/dacapobench.git
......
...@@ -22,8 +22,8 @@ public class Avrora extends Benchmark { ...@@ -22,8 +22,8 @@ public class Avrora extends Benchmark {
String[] args; String[] args;
public Avrora(Config config, File scratch) throws Exception { public Avrora(Config config, File scratch, File data) throws Exception {
super(config, scratch); super(config, scratch, data);
Class<?> clazz = Class.forName("avrora.Main", true, loader); Class<?> clazz = Class.forName("avrora.Main", true, loader);
this.method = clazz.getMethod("main", String[].class); this.method = clazz.getMethod("main", String[].class);
} }
...@@ -31,7 +31,7 @@ public class Avrora extends Benchmark { ...@@ -31,7 +31,7 @@ public class Avrora extends Benchmark {
@Override @Override
protected void prepare(String size) throws Exception { protected void prepare(String size) throws Exception {
super.prepare(size); super.prepare(size);
args = config.preprocessArgs(size, scratch); args = config.preprocessArgs(size, scratch, data);
} }
@Override @Override
......
...@@ -27,21 +27,19 @@ size large args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/simplex ...@@ -27,21 +27,19 @@ size large args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/simplex
"mapVASR.png" digest 0x3086b028662d865e0913602823c0b339dc599b0b, "mapVASR.png" digest 0x3086b028662d865e0913602823c0b339dc599b0b,
"bavaria.png" digest 0xf9e58383e2418b8a065e895cbef8132d9099a599; "bavaria.png" digest 0xf9e58383e2418b8a065e895cbef8132d9099a599;
/** NOTE: huge size not runnable.
* requires later huge data refactoring changes size huge args "-d","${SCRATCH}","-scriptSecurityOff","${DATA}/batik/usMetro.svg", "${DATA}/batik/mapVASR.svg",
*/ "${DATA}/batik/bavaria.svg",
size huge args "-d","${SCRATCH}","-scriptSecurityOff","${SCRATCH}/batik/usMetro.svg", "${SCRATCH}/batik/mapVASR.svg", "${DATA}/batik/lakeTauca.svg", "${DATA}/batik/river.svg", "${DATA}/batik/tenerife.svg",
"${SCRATCH}/batik/bavaria.svg", "${DATA}/batik/france.svg", "${DATA}/batik/nrwpTopography.svg",
"${SCRATCH}/batik/lakeTauca.svg", "${SCRATCH}/batik/river.svg", "${SCRATCH}/batik/tenerife.svg", "${DATA}/batik/velhop.svg", "${DATA}/batik/polytopeCube.svg", "${DATA}/batik/ukRef.svg",
"${SCRATCH}/batik/france.svg", "${SCRATCH}/batik/nrwpTopography.svg", "${DATA}/batik/btw.svg", "${DATA}/batik/lyon.svg", "${DATA}/batik/ohridTopographic.svg",
"${SCRATCH}/batik/velhop.svg", "${SCRATCH}/batik/polytopeCube.svg", "${SCRATCH}/batik/ukRef.svg", "${DATA}/batik/dcStreet.svg", "${DATA}/batik/chicoutimi.svg",
"${SCRATCH}/batik/btw.svg", "${SCRATCH}/batik/lyon.svg", "${SCRATCH}/batik/ohridTopographic.svg", "${DATA}/batik/usElection.svg", "${DATA}/batik/moldova.svg", "${DATA}/batik/mapPolitical.svg",
"${SCRATCH}/batik/dcStreet.svg", "${SCRATCH}/batik/chicoutimi.svg", "${DATA}/batik/hague.svg", "${DATA}/batik/locGermany.svg",
"${SCRATCH}/batik/usElection.svg", "${SCRATCH}/batik/moldova.svg", "${SCRATCH}/batik/mapPolitical.svg", "${DATA}/batik/mowbrayOSM.svg", "${DATA}/batik/mapWorld.svg", "${DATA}/batik/sierpinski.svg",
"${SCRATCH}/batik/hague.svg", "${SCRATCH}/batik/locGermany.svg", "${DATA}/batik/phi.svg", "${DATA}/batik/simplex.svg", "${DATA}/batik/mapLakes.svg",
"${SCRATCH}/batik/mowbrayOSM.svg", "${SCRATCH}/batik/mapWorld.svg", "${SCRATCH}/batik/sierpinski.svg", "${DATA}/batik/Tettstedskart.svg", "${DATA}/batik/mapSanFran.svg", "${DATA}/batik/serbia.svg"
"${SCRATCH}/batik/phi.svg", "${SCRATCH}/batik/simplex.svg", "${SCRATCH}/batik/mapLakes.svg",
"${SCRATCH}/batik/Tettstedskart.svg", "${SCRATCH}/batik/mapSanFran.svg", "${SCRATCH}/batik/serbia.svg"
output stdout digest 0x3ebb7954cd8d418aad42f3ad6d71a1b512892178, output stdout digest 0x3ebb7954cd8d418aad42f3ad6d71a1b512892178,
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709, stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"usMetro.png" digest 0xc8c2e79878cbce8ef4ea2e4b617096a9a53b5fe7, "usMetro.png" digest 0xc8c2e79878cbce8ef4ea2e4b617096a9a53b5fe7,
......
...@@ -30,7 +30,7 @@ ...@@ -30,7 +30,7 @@
</target> </target>
<target name="data"> <target name="data">
<copy file="${bm-data}/batik.zip" todir="${bm-dat}"/> <copy file="${bm-data}/${bm-name}.zip" todir="${bm-dat}"/>
</target> </target>
<target name="jar" depends="deps"> <target name="jar" depends="deps">
......
...@@ -23,8 +23,8 @@ public class Batik extends Benchmark { ...@@ -23,8 +23,8 @@ public class Batik extends Benchmark {
private String[] args; private String[] args;
private final Constructor<?> constructor; private final Constructor<?> constructor;
public Batik(Config config, File scratch) throws Exception { public Batik(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.apache.batik.apps.rasterizer.Main", true, loader); Class<?> clazz = Class.forName("org.apache.batik.apps.rasterizer.Main", true, loader);
this.method = clazz.getMethod("execute"); this.method = clazz.getMethod("execute");
this.constructor = clazz.getConstructor(String[].class); this.constructor = clazz.getConstructor(String[].class);
...@@ -33,7 +33,7 @@ public class Batik extends Benchmark { ...@@ -33,7 +33,7 @@ public class Batik extends Benchmark {
@Override @Override
protected void prepare(String size) throws Exception { protected void prepare(String size) throws Exception {
super.prepare(size); super.prepare(size);
args = config.preprocessArgs(size, scratch); args = config.preprocessArgs(size, scratch, data);
} }
public void iterate(String size) throws Exception { public void iterate(String size) throws Exception {
......
...@@ -203,5 +203,4 @@ ...@@ -203,5 +203,4 @@
<delete file="${temp.file}"/> <delete file="${temp.file}"/>
<jar jarfile="${build.target-jar}" basedir="${bm-output-dir}" update="true"/> <jar jarfile="${build.target-jar}" basedir="${bm-output-dir}" update="true"/>
</target> </target>
</project> </project>
...@@ -33,8 +33,8 @@ public class Eclipse extends Benchmark { ...@@ -33,8 +33,8 @@ public class Eclipse extends Benchmark {
private final Method run; private final Method run;
private final Method shutdown; private final Method shutdown;
public Eclipse(Config config, File scratch) throws Exception { public Eclipse(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.eclipse.core.runtime.adaptor.EclipseStarter", true, loader); Class<?> clazz = Class.forName("org.eclipse.core.runtime.adaptor.EclipseStarter", true, loader);
this.method = clazz.getMethod("startup", String[].class, Runnable.class); this.method = clazz.getMethod("startup", String[].class, Runnable.class);
this.isRunning = clazz.getMethod("isRunning"); this.isRunning = clazz.getMethod("isRunning");
......
...@@ -24,8 +24,8 @@ public class Fop extends Benchmark { ...@@ -24,8 +24,8 @@ public class Fop extends Benchmark {
private String[] inputs; private String[] inputs;
private String[] outputs; private String[] outputs;
public Fop(Config config, File scratch) throws Exception { public Fop(Config config, File scratch, File data) throws Exception {
super(config, scratch); super(config, scratch, data);
Class<?> clazz = Class.forName("org.apache.fop.cli.Main", true, loader); Class<?> clazz = Class.forName("org.apache.fop.cli.Main", true, loader);
this.method = clazz.getMethod("startFOP", new Class[] { String[].class }); this.method = clazz.getMethod("startFOP", new Class[] { String[].class });
} }
...@@ -33,7 +33,7 @@ public class Fop extends Benchmark { ...@@ -33,7 +33,7 @@ public class Fop extends Benchmark {
@Override @Override
protected void prepare(String size) throws Exception { protected void prepare(String size) throws Exception {
super.prepare(size); super.prepare(size);
args = config.preprocessArgs(size, scratch); args = config.preprocessArgs(size, scratch, data);
collectInputs(); collectInputs();
} }
......
...@@ -35,8 +35,8 @@ public class H2 extends Benchmark { ...@@ -35,8 +35,8 @@ public class H2 extends Benchmark {
private Method iterationTPCC; private Method iterationTPCC;
private Method postIterationTPCC; private Method postIterationTPCC;
public H2(Config config, File scratch) throws Exception { public H2(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
} }
@Override @Override
...@@ -54,7 +54,7 @@ public class H2 extends Benchmark { ...@@ -54,7 +54,7 @@ public class H2 extends Benchmark {
TreeMap<String,String> threadMap = new TreeMap<String,String>(); TreeMap<String,String> threadMap = new TreeMap<String,String>();
TreeMap<String,String[]> argMap = new TreeMap<String,String[]>(); TreeMap<String,String[]> argMap = new TreeMap<String,String[]>();
for(String size: config.getSizes()) { for(String size: config.getSizes()) {
String[] args = config.preprocessArgs(size, scratch); String[] args = config.preprocessArgs(size, scratch, data);
argMap.put(size,args); argMap.put(size,args);
threadMap.put(size, Integer.toString(config.getThreadCount(size))); threadMap.put(size, Integer.toString(config.getThreadCount(size)));
......
...@@ -22,8 +22,8 @@ public class Jython extends Benchmark { ...@@ -22,8 +22,8 @@ public class Jython extends Benchmark {
private Method pySetArgsMethod; private Method pySetArgsMethod;
public Jython(Config config, File scratch) throws Exception { public Jython(Config config, File scratch, File data) throws Exception {
super(config, scratch); super(config, scratch, data);
Class<?> clazz = Class.forName("org.python.util.jython", true, loader); Class<?> clazz = Class.forName("org.python.util.jython", true, loader);
this.method = clazz.getMethod("main", String[].class); this.method = clazz.getMethod("main", String[].class);
Class<?> pyClass = Class.forName("org.python.core.PySystemState", true, loader); Class<?> pyClass = Class.forName("org.python.core.PySystemState", true, loader);
...@@ -46,7 +46,7 @@ public class Jython extends Benchmark { ...@@ -46,7 +46,7 @@ public class Jython extends Benchmark {
* script sees. Hence the Py.setArgv call, followed by the jython.main call. * script sees. Hence the Py.setArgv call, followed by the jython.main call.
*/ */
public void iterate(String size) throws Exception { public void iterate(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch); String[] args = config.preprocessArgs(size, scratch, data);
pySetArgsMethod.invoke(null, (Object) args); pySetArgsMethod.invoke(null, (Object) args);
method.invoke(null, (Object) args); method.invoke(null, (Object) args);
} }
......
...@@ -43,7 +43,7 @@ ...@@ -43,7 +43,7 @@
</target> </target>
<target name="data" depends="lucene-enwiki-bench-data"> <target name="data">
<mkdir dir="${bm-data}/luindex"/> <mkdir dir="${bm-data}/luindex"/>
<mkdir dir="${bm-data}/luindex/kjv"/> <mkdir dir="${bm-data}/luindex/kjv"/>
...@@ -57,7 +57,7 @@ ...@@ -57,7 +57,7 @@
</zip> </zip>
</target> </target>
<target name="lucene-enwiki-bench-data-check"> <!-- <target name="lucene-enwiki-bench-data-check">
<available property="lucene-enwiki-bench-data-exists" file="${bm-data}/luindex/enwiki/enwiki.txt"/> <available property="lucene-enwiki-bench-data-exists" file="${bm-data}/luindex/enwiki/enwiki.txt"/>
<condition property="lucene-enwiki-bench-data-valid-md5"> <condition property="lucene-enwiki-bench-data-valid-md5">
<and> <and>
...@@ -74,6 +74,6 @@ ...@@ -74,6 +74,6 @@
</exec> </exec>
<mkdir dir="${bm-data}/luindex/enwiki"/> <mkdir dir="${bm-data}/luindex/enwiki"/>
<move file="${lucene-build-dir}/benchmark/work/enwiki.txt" todir="${bm-data}/luindex/enwiki"/> <move file="${lucene-build-dir}/benchmark/work/enwiki.txt" todir="${bm-data}/luindex/enwiki"/>
</target> </target> -->
</project> </project>
...@@ -25,13 +25,13 @@ public class Luindex extends Benchmark { ...@@ -25,13 +25,13 @@ public class Luindex extends Benchmark {
private final Object benchmark; private final Object benchmark;
private final Class<?> clazz; private final Class<?> clazz;
public Luindex(Config config, File scratch) throws Exception { public Luindex(Config config, File scratch, File data) throws Exception {
super(config, scratch); super(config, scratch, data, false);
this.clazz = Class.forName("org.dacapo.luindex.Index", true, loader); this.clazz = Class.forName("org.dacapo.luindex.Index", true, loader);
Constructor<?> cons = clazz.getConstructor(File.class); Constructor<?> cons = clazz.getConstructor(File.class, File.class);
useBenchmarkClassLoader(); useBenchmarkClassLoader();
try { try {
benchmark = cons.newInstance(scratch); benchmark = cons.newInstance(scratch, data);
} finally { } finally {
revertClassLoader(); revertClassLoader();
} }
...@@ -55,7 +55,7 @@ public class Luindex extends Benchmark { ...@@ -55,7 +55,7 @@ public class Luindex extends Benchmark {
public void iterate(String size) throws Exception { public void iterate(String size) throws Exception {
if (getVerbose()) if (getVerbose())
System.out.println("luindex benchmark starting"); System.out.println("luindex benchmark starting");
String[] args = config.preprocessArgs(size, scratch); String[] args = config.preprocessArgs(size, scratch, data);
final File INDEX_DIR = new File(scratch, "index"); final File INDEX_DIR = new File(scratch, "index");
......
...@@ -13,10 +13,7 @@ size default args "--dirwalk", "${SCRATCH}/luindex/shakespeare","${SCRATCH}/luin ...@@ -13,10 +13,7 @@ size default args "--dirwalk", "${SCRATCH}/luindex/shakespeare","${SCRATCH}/luin
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709, stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"index/segments_1" bytes 136; "index/segments_1" bytes 136;
/* NOTE: not including enwiki.txt in the dacapo.jar size huge args "--linedoc", "${DATA}/luindex/enwiki/enwiki-1.txt", "${DATA}/luindex/enwiki/enwiki-2.txt"
* When the mechanism for dealing with large dataset is in place,
* this should be modified. */
size huge args "--linedoc", "${SCRATCH}/../bms/luindex/data/luindex/enwiki/enwiki.txt"
output stdout digest 0x3067fa0a00d9311e95cf963dab1dfea1fca4e993, output stdout digest 0x3067fa0a00d9311e95cf963dab1dfea1fca4e993,
stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709, stderr digest 0xda39a3ee5e6b4b0d3255bfef95601890afd80709,
"index/segments_1" bytes 137; "index/segments_1" bytes 137;
......
...@@ -64,9 +64,11 @@ import org.apache.lucene.index.IndexWriterConfig; ...@@ -64,9 +64,11 @@ import org.apache.lucene.index.IndexWriterConfig;
public class Index { public class Index {
private final File scratch; private final File scratch;
private final File data;
public Index(File scratch) { public Index(File scratch, File data) {
this.scratch = scratch; this.scratch = scratch;
this.data = data;
} }
/** /**
...@@ -83,8 +85,8 @@ public class Index { ...@@ -83,8 +85,8 @@ public class Index {
System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path"); System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory"); throw new IOException("Cannot read from document directory");
} }
File prefix = docDir.getAbsolutePath().contains(scratch.getAbsolutePath()) ? scratch : data;
indexDocs(writer, docDir); indexDocs(writer, docDir, prefix);
System.out.println("Optimizing..."); System.out.println("Optimizing...");
writer.forceMerge(1); writer.forceMerge(1);
} }
...@@ -100,23 +102,24 @@ public class Index { ...@@ -100,23 +102,24 @@ public class Index {
IWConfig.setMergePolicy (new LogByteSizeMergePolicy()); IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig); IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
File txtFile = new File(args[0]); for (int idx = 0; idx < args.length; idx ++) {
File txtFile = new File(args[idx]);
if (!txtFile.exists() || !txtFile.canRead()) { if (!txtFile.exists() || !txtFile.canRead()) {
System.out.println("Document directory '" + txtFile.getAbsolutePath() + "' does not exist or is not readable, please check the path"); System.out.println("Document directory '" + txtFile.getAbsolutePath() + "' does not exist or is not readable, please check the path");
throw new IOException("Cannot read from document directory"); throw new IOException("Cannot read from document directory");
} }
BufferedReader reader = new BufferedReader(new FileReader(txtFile)); BufferedReader reader = new BufferedReader(new FileReader(txtFile));
reader.readLine(); // skip header line reader.readLine(); // skip header line
int nLines = (args.length > 1) ? Integer.parseInt(args[1]) : Integer.MAX_VALUE; String line = reader.readLine();
String line = reader.readLine(); int n = 0;
int n = 0; while (line != null) {
while (line != null && n < nLines) { System.out.println("adding " + line.substring(0, line.indexOf(SEP)));
System.out.println("adding " + line.substring(0, line.indexOf(SEP))); writer.addDocument(getLuceneDocFromLine(line));
writer.addDocument(getLuceneDocFromLine(line)); line = reader.readLine();
line = reader.readLine(); n ++;
n ++; }
} }
System.out.println("Optimizing..."); System.out.println("Optimizing...");
...@@ -164,10 +167,10 @@ public class Index { ...@@ -164,10 +167,10 @@ public class Index {
* @param file * @param file
* @throws IOException * @throws IOException
*/ */
void indexDocs(IndexWriter writer, File file) throws IOException { void indexDocs(IndexWriter writer, File file, File prefix) throws IOException {
/* Strip the absolute part of the path name from file name output */ /* Strip the absolute part of the path name from file name output */
int scratchP = scratch.getCanonicalPath().length() + 1; int prefixIdx = prefix.getCanonicalPath().length() + 1;
/* do not try to index files that cannot be read */ /* do not try to index files that cannot be read */
if (file.canRead()) { if (file.canRead()) {
...@@ -177,11 +180,11 @@ public class Index { ...@@ -177,11 +180,11 @@ public class Index {
if (files != null) { if (files != null) {
Arrays.sort(files); Arrays.sort(files);
for (int i = 0; i < files.length; i++) { for (int i = 0; i < files.length; i++) {
indexDocs(writer, new File(file, files[i])); indexDocs(writer, new File(file, files[i]), prefix);
} }
} }
} else { } else {
System.out.println("adding " + file.getCanonicalPath().substring(scratchP)); System.out.println("adding " + file.getCanonicalPath().substring(prefixIdx));
try { try {
Document doc = new Document(); Document doc = new Document();
FieldType docFT = new FieldType(); FieldType docFT = new FieldType();
......
...@@ -20,8 +20,8 @@ import org.dacapo.parser.Config; ...@@ -20,8 +20,8 @@ import org.dacapo.parser.Config;
public class Lusearch extends org.dacapo.harness.Benchmark { public class Lusearch extends org.dacapo.harness.Benchmark {
private final Object benchmark; private final Object benchmark;
public Lusearch(Config config, File scratch) throws Exception { public Lusearch(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.lusearch.Search", true, loader); Class<?> clazz = Class.forName("org.dacapo.lusearch.Search", true, loader);
this.method = clazz.getMethod("main", String[].class); this.method = clazz.getMethod("main", String[].class);
Constructor<?> cons = clazz.getConstructor(); Constructor<?> cons = clazz.getConstructor();
...@@ -35,6 +35,6 @@ public class Lusearch extends org.dacapo.harness.Benchmark { ...@@ -35,6 +35,6 @@ public class Lusearch extends org.dacapo.harness.Benchmark {
@Override @Override
public void iterate(String size) throws Exception { public void iterate(String size) throws Exception {
method.invoke(benchmark, (Object) (config.preprocessArgs(size, scratch))); method.invoke(benchmark, (Object) (config.preprocessArgs(size, scratch, data)));
} }
} }
...@@ -4,7 +4,7 @@ benchmark lusearch ...@@ -4,7 +4,7 @@ benchmark lusearch
jars "dacapo-lusearch.jar", "lucene-core-7.1.0-SNAPSHOT.jar", "lucene-demo-7.1.0-SNAPSHOT.jar", "lucene-queryparser-7.1.0-SNAPSHOT.jar"; jars "dacapo-lusearch.jar", "lucene-core-7.1.0-SNAPSHOT.jar", "lucene-demo-7.1.0-SNAPSHOT.jar", "lucene-queryparser-7.1.0-SNAPSHOT.jar";
size small args "-index","${SCRATCH}/lusearch/index-default", size small args "-index","${SCRATCH}/lusearch/index-default",
"-queries","${SCRATCH}/lusearch/queries-default/query", "-queries","${SCRATCH}/lusearch/queries-default/",
"-output", "${SCRATCH}/lusearch.out", "-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "8", "-totalqueries", "8",
"-threads", "${THREADS}" "-threads", "${THREADS}"
...@@ -23,7 +23,7 @@ size small args "-index","${SCRATCH}/lusearch/index-default", ...@@ -23,7 +23,7 @@ size small args "-index","${SCRATCH}/lusearch/index-default",
"lusearch.out7" digest(text) 0xa4ab2577aa78ead6fec346ee1d8ad1edcec0a55d; "lusearch.out7" digest(text) 0xa4ab2577aa78ead6fec346ee1d8ad1edcec0a55d;
size default args "-index","${SCRATCH}/lusearch/index-default", size default args "-index","${SCRATCH}/lusearch/index-default",
"-queries", "${SCRATCH}/lusearch/queries-default/query", "-queries", "${SCRATCH}/lusearch/queries-default/",
"-output", "${SCRATCH}/lusearch.out", "-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "128", "-totalqueries", "128",
"-threads", "${THREADS}" "-threads", "${THREADS}"
...@@ -161,9 +161,9 @@ size default args "-index","${SCRATCH}/lusearch/index-default", ...@@ -161,9 +161,9 @@ size default args "-index","${SCRATCH}/lusearch/index-default",
"lusearch.out126" digest(text) 0x22e7c16878292805cae3579bf735b6a6acac0c3b, "lusearch.out126" digest(text) 0x22e7c16878292805cae3579bf735b6a6acac0c3b,
"lusearch.out127" digest(text) 0x90b1433771874e342bb28ed9cc68b5b738197c66; "lusearch.out127" digest(text) 0x90b1433771874e342bb28ed9cc68b5b738197c66;
size huge args "-index","${SCRATCH}/lusearch/index-enwiki", size huge args "-index","${DATA}/lusearch/index-enwiki",
"-field", "body", "-field", "body",
"-queries", "${SCRATCH}/lusearch/queries-enwiki/query", "-queries", "${DATA}/lusearch/queries-enwiki/",
"-output", "${SCRATCH}/lusearch.out", "-output", "${SCRATCH}/lusearch.out",
"-totalqueries", "2048", "-totalqueries", "2048",
"-threads", "${THREADS}" "-threads", "${THREADS}"
......
...@@ -214,7 +214,7 @@ public class Search { ...@@ -214,7 +214,7 @@ public class Search {
reader = new OneNormsReader(reader, normsField);*/ reader = new OneNormsReader(reader, normsField);*/
searcher = new IndexSearcher(reader); searcher = new IndexSearcher(reader);
String query = queryBase + (id < 10 ? "00" : (id < 100 ? "0" : "")) + id + ".txt"; String query = queryBase + "query" + (id < 10 ? "00" : (id < 100 ? "0" : "")) + id + ".txt";
in = new BufferedReader(new FileReader(query)); in = new BufferedReader(new FileReader(query));
out = new PrintWriter(new BufferedWriter(new FileWriter(outBase + id))); out = new PrintWriter(new BufferedWriter(new FileWriter(outBase + id)));
......
...@@ -32,8 +32,8 @@ public class Pmd extends Benchmark { ...@@ -32,8 +32,8 @@ public class Pmd extends Benchmark {
String[] args; String[] args;
public Pmd(Config config, File scratch) throws Exception { public Pmd(Config config, File scratch, File data) throws Exception {
super(config, scratch); super(config, scratch, data);
Class<?> clazz = Class.forName("net.sourceforge.pmd.PMD", true, loader); Class<?> clazz = Class.forName("net.sourceforge.pmd.PMD", true, loader);
this.method = clazz.getMethod("main", String[].class); this.method = clazz.getMethod("main", String[].class);
...@@ -55,7 +55,7 @@ public class Pmd extends Benchmark { ...@@ -55,7 +55,7 @@ public class Pmd extends Benchmark {
protected void prepare(String size) throws Exception { protected void prepare(String size) throws Exception {
super.prepare(size); super.prepare(size);
args = config.preprocessArgs(size, scratch); args = config.preprocessArgs(size, scratch, data);
} }
......
...@@ -25,8 +25,8 @@ public class Sunflow extends org.dacapo.harness.Benchmark { ...@@ -25,8 +25,8 @@ public class Sunflow extends org.dacapo.harness.Benchmark {
private final Method beginMethod; private final Method beginMethod;
private final Method endMethod; private final Method endMethod;
public Sunflow(Config config, File scratch) throws Exception { public Sunflow(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.sunflow.Benchmark", true, loader); Class<?> clazz = Class.forName("org.sunflow.Benchmark", true, loader);
this.method = clazz.getMethod("kernelMain"); this.method = clazz.getMethod("kernelMain");
this.beginMethod = clazz.getMethod("kernelBegin"); this.beginMethod = clazz.getMethod("kernelBegin");
...@@ -46,7 +46,7 @@ public class Sunflow extends org.dacapo.harness.Benchmark { ...@@ -46,7 +46,7 @@ public class Sunflow extends org.dacapo.harness.Benchmark {
* @param size The "size" of the iteration (small, default, large) * @param size The "size" of the iteration (small, default, large)
*/ */
public void preIteration(String size) throws Exception { public void preIteration(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch); String[] args = config.preprocessArgs(size, scratch, data);
useBenchmarkClassLoader(); useBenchmarkClassLoader();
try { try {
sunflow = constructor.newInstance(Integer.parseInt(args[0]), sunflow = constructor.newInstance(Integer.parseInt(args[0]),
......
...@@ -35,8 +35,8 @@ public class Tomcat extends Benchmark { ...@@ -35,8 +35,8 @@ public class Tomcat extends Benchmark {
* @param scratch Scratch directory * @param scratch Scratch directory
* @throws Exception When something goes wrong * @throws Exception When something goes wrong
*/ */
public Tomcat(Config config, File scratch) throws Exception { public Tomcat(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
this.clazz = Class.forName("org.dacapo.tomcat.Control", true, loader); this.clazz = Class.forName("org.dacapo.tomcat.Control", true, loader);
this.method = clazz.getMethod("exec", String.class); this.method = clazz.getMethod("exec", String.class);
......
...@@ -26,8 +26,8 @@ public class Tradebeans extends Benchmark { ...@@ -26,8 +26,8 @@ public class Tradebeans extends Benchmark {
private Method initializeMethod; private Method initializeMethod;
private Method shutdownMethod; private Method shutdownMethod;
public Tradebeans(Config config, File scratch) throws Exception { public Tradebeans(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
// Find the launcher // Find the launcher
Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader); Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader);
...@@ -43,7 +43,7 @@ public class Tradebeans extends Benchmark { ...@@ -43,7 +43,7 @@ public class Tradebeans extends Benchmark {
@Override @Override
protected void prepare(String size) throws Exception { protected void prepare(String size) throws Exception {
String[] args = config.preprocessArgs(size, scratch); String[] args = config.preprocessArgs(size, scratch, data);
String dtSize = "medium"; String dtSize = "medium";
if (args.length == 1) if (args.length == 1)
dtSize = args[0]; dtSize = args[0];
......
...@@ -24,8 +24,8 @@ public class Tradesoap extends Benchmark { ...@@ -24,8 +24,8 @@ public class Tradesoap extends Benchmark {
private Method initializeMethod; private Method initializeMethod;
private Method shutdownMethod; private Method shutdownMethod;
public Tradesoap(Config config, File scratch) throws Exception { public Tradesoap(Config config, File scratch, File data) throws Exception {
super(config, scratch, false); super(config, scratch, data, false);
Class<?> clazz = Class.forName("org.dacapo.daytrader.Launcher", true, loader);