Commit e5fd171e authored by John Zhang's avatar John Zhang

add ycsb workload ontop of cassandra

parent 72bad216
......@@ -18,16 +18,34 @@
<import file="../common.xml"/>
<property name="mvn-home" location="${toolsdir}/apache-maven-${mvn.version}/bin"/>
<property name="mvn-exe" value="${mvn-home}/mvn"/>
<condition property="mvn-exe" value="${mvn-home}/mvn.bat">
<os family="windows"/>
</condition>
<property name="ycsb-version" value="0.15.0"/>
<property name="ycsb-url" value="https://github.com/brianfrankcooper/YCSB/releases/download/${ycsb-version}"/>
<property name="ycsb-git-url" value="https://github.com/brianfrankcooper/YCSB.git"/>
<property name="ycsb-git-commit" value="${ycsb-version}"/>
<property name="ycsb-tgz" value="ycsb-${ycsb-version}.tar.gz"/>
<property name="datastax-driver-version" value="3.6.0"/>
<property name="datastax-driver-url" value="https://repo1.maven.org/maven2/com/datastax/cassandra/cassandra-driver-core/${datastax-driver-version}"/>
<property name="datastax-driver-jar" value="cassandra-driver-core-${datastax-driver-version}.jar"/>
<target name="source">
<antcall target="check-source">
<param name="target-dir" value="${bm-downloads}"/>
<param name="target-url" value="${ycsb-url}"/>
<param name="target-file" value="${ycsb-tgz}"/>
<param name="target-url" value="${datastax-driver-url}"/>
<param name="target-file" value="${datastax-driver-jar}"/>
</antcall>
<antcall target="check-git-source">
<param name="target-dir" value="${bm-downloads}"/>
<param name="git-url" value="${ycsb-git-url}"/>
<param name="git-commit" value="${ycsb-git-commit}"/>
<param name="target-file" value="${ycsb-tgz}"/>
<param name="fail-on-error" value="true"/>
</antcall>
</target>
<property name="cassandra-build-top" value="${bm-build-dir}/cassandra-${bm-version}"/>
......@@ -38,8 +56,18 @@
<exec executable="ant" dir="${cassandra-build-top}"/>
</target>
<target name="bm-build" depends="cassandra-build">
<property name="bm-harness-classpath" value="${cassandra-build-top}/build/classes/main"/>
<target name="ycsb-build" depends="get-slf4j-simple-jar">
<property name="ycsb-dir" value="${bm-build-dir}/ycsb-${ycsb-version}"/>
<mkdir dir="${ycsb-dir}"/>
<untar src="${bm-downloads}/${ycsb-tgz}" dest="${ycsb-dir}" compression="gzip"/>
<patch patchfile="${bm-files}/ycsb.patch" dir="${ycsb-dir}" strip="0" ignorewhitespace="yes"/>
<exec executable="${mvn-exe}" dir="${ycsb-dir}">
<arg line="-pl com.yahoo.ycsb:cassandra-binding dependency:copy-dependencies -am package -DskipTests"/>
</exec>
</target>
<target name="bm-build" depends="cassandra-build, ycsb-build">
<property name="bm-harness-classpath" value="${cassandra-build-top}/build/classes/main:${bm-downloads}/${datastax-driver-jar}"/>
</target>
<property name="slf4j-version" value="1.7.25"/>
......@@ -57,29 +85,32 @@
<property name="slf4j-simple-jar" value="${slf4j-dir}/slf4j-simple-${slf4j-version}.jar"/>
</target>
<target name="ycsb" depends="get-slf4j-simple-jar">
<untar src="${bm-downloads}/${ycsb-tgz}" dest="${bm-build-dir}" compression="gzip"/>
<property name="ycsb-dir" value="${bm-build-dir}/ycsb-${ycsb-version}"/>
</target>
<target name="libs" depends="ycsb">
<!-- cassandra libs -->
<target name="libs">
<copy todir="${bm-jars}">
<fileset dir="${cassandra-build-top}/lib" includes="*.jar"/>
<resources>
<!-- cassandra libs -->
<fileset dir="${cassandra-build-top}/lib" includes="*.jar"/>
<!-- datastax cassandra driver -->
<file file="${bm-downloads}/${datastax-driver-jar}"/>
<!-- ycsb libs -->
<file file="${slf4j-simple-jar}"/>
<file file="${ycsb-dir}/cassandra/target/cassandra-binding-${ycsb-version}.jar"/>
<fileset dir="${ycsb-dir}/cassandra/target/dependency" includes="*.jar"/>
<file file="${ycsb-dir}/core/target/core-${ycsb-version}.jar"/>
<fileset dir="${ycsb-dir}/core/target/dependency">
<filename name="HdrHistogram-2.1.4.jar"/>
<filename name="htrace-core4-4.1.0-incubating.jar"/>
<filename name="jackson-core-asl-1.9.4.jar"/>
<filename name="jackson-mapper-asl-1.9.4.jar"/>
</fileset>
</resources>
</copy>
<!-- cassandra libsigar dependency -->
<zip destfile="${bm-dat}/libsigar.zip" basedir="${cassandra-build-top}/lib/sigar-bin"/>
<!-- cassandra config (XXX: maybe put it in another target/directory) -->
<zip destfile="${bm-dat}/cassandra-conf.zip" basedir="${cassandra-build-top}/conf"/>
<!-- ycsb libs -->
<echo message="ycsb-dir = ${ycsb-dir}"/>
<copy file="${slf4j-simple-jar}" todir="${bm-jars}"/>
<copy todir="${bm-jars}">
<fileset dir="${ycsb-dir}/lib" includes="*.jar"/>
</copy>
<copy todir="${bm-jars}">
<fileset dir="${ycsb-dir}/cassandra-binding/lib" includes="*.jar"/>
</copy>
<!-- ycsb workloads -->
<zip destfile="${bm-dat}/ycsb-workloads.zip" basedir="${ycsb-dir}/workloads"/>
</target>
<target name="jar" depends="libs">
......
benchmark cassandra
class org.dacapo.harness.Cassandra
class org.dacapo.harness.CassandraBench
thread-model per_cpu
jars "cassandra-3.11.3.jar", "cassandra-thrift-3.11.3.jar", "HdrHistogram-2.1.9.jar", "ST4-4.0.8.jar", "airline-0.6.jar", "antlr-runtime-3.5.2.jar", "asm-5.0.4.jar", "caffeine-2.2.6.jar", "cassandra-driver-core-3.0.1-shaded.jar", "commons-cli-1.1.jar", "commons-codec-1.9.jar", "commons-lang3-3.1.jar", "commons-math3-3.2.jar", "compress-lzf-0.8.4.jar", "concurrent-trees-2.4.0.jar", "concurrentlinkedhashmap-lru-1.4.jar", "disruptor-3.0.1.jar", "ecj-4.4.2.jar", "guava-18.0.jar", "high-scale-lib-1.0.6.jar", "hppc-0.5.4.jar", "jackson-core-asl-1.9.13.jar", "jackson-mapper-asl-1.9.13.jar", "jamm-0.3.0.jar", "javax.inject.jar", "jbcrypt-0.3m.jar", "jcl-over-slf4j-1.7.7.jar", "jctools-core-1.2.1.jar", "jflex-1.6.0.jar", "jna-4.2.2.jar", "joda-time-2.4.jar", "json-simple-1.1.jar", "jstackjunit-0.0.1.jar", "libthrift-0.9.2.jar", "log4j-over-slf4j-1.7.7.jar", "logback-classic-1.1.3.jar", "logback-core-1.1.3.jar", "lz4-1.3.0.jar", "metrics-core-3.1.5.jar", "metrics-jvm-3.1.5.jar", "metrics-logback-3.1.5.jar", "netty-all-4.0.44.Final.jar", "ohc-core-0.4.4.jar", "ohc-core-j8-0.4.4.jar", "reporter-config-base-3.0.3.jar", "reporter-config3-3.0.3.jar", "sigar-1.6.4.jar", "slf4j-api-1.7.7.jar", "snakeyaml-1.11.jar", "snappy-java-1.1.1.7.jar", "snowball-stemmer-1.3.0.581.1.jar", "stream-2.5.2.jar", "thrift-server-0.3.7.jar", "cassandra-binding-0.15.0.jar", "cassandra-driver-core-3.0.0.jar", "core-0.15.0.jar", "guava-16.0.1.jar", "HdrHistogram-2.1.4.jar", "htrace-core4-4.1.0-incubating.jar", "jackson-core-asl-1.9.4.jar", "jackson-mapper-asl-1.9.4.jar", "metrics-core-3.1.2.jar", "netty-buffer-4.0.33.Final.jar", "netty-codec-4.0.33.Final.jar", "netty-common-4.0.33.Final.jar", "netty-handler-4.0.33.Final.jar", "netty-transport-4.0.33.Final.jar", "slf4j-api-1.7.25.jar", "slf4j-simple-1.7.25.jar";
jars "cassandra-3.11.3.jar", "cassandra-thrift-3.11.3.jar", "cassandra-driver-core-3.6.0.jar", "ST4-4.0.8.jar", "airline-0.6.jar", "antlr-runtime-3.5.2.jar", "asm-5.0.4.jar", "caffeine-2.2.6.jar", "cassandra-driver-core-3.0.1-shaded.jar", "commons-cli-1.1.jar", "commons-codec-1.9.jar", "commons-lang3-3.1.jar", "commons-math3-3.2.jar", "compress-lzf-0.8.4.jar", "concurrent-trees-2.4.0.jar", "concurrentlinkedhashmap-lru-1.4.jar", "disruptor-3.0.1.jar", "ecj-4.4.2.jar", "guava-18.0.jar", "high-scale-lib-1.0.6.jar", "hppc-0.5.4.jar", "jackson-core-asl-1.9.13.jar", "jackson-mapper-asl-1.9.13.jar", "jamm-0.3.0.jar", "javax.inject.jar", "jbcrypt-0.3m.jar", "jcl-over-slf4j-1.7.7.jar", "jctools-core-1.2.1.jar", "jflex-1.6.0.jar", "jna-4.2.2.jar", "joda-time-2.4.jar", "json-simple-1.1.jar", "jstackjunit-0.0.1.jar", "libthrift-0.9.2.jar", "log4j-over-slf4j-1.7.7.jar", "logback-classic-1.1.3.jar", "logback-core-1.1.3.jar", "lz4-1.3.0.jar", "metrics-core-3.1.5.jar", "metrics-jvm-3.1.5.jar", "metrics-logback-3.1.5.jar", "netty-all-4.0.44.Final.jar", "ohc-core-0.4.4.jar", "ohc-core-j8-0.4.4.jar", "reporter-config-base-3.0.3.jar", "reporter-config3-3.0.3.jar", "sigar-1.6.4.jar", "slf4j-api-1.7.7.jar", "snakeyaml-1.11.jar", "snappy-java-1.1.1.7.jar", "snowball-stemmer-1.3.0.581.1.jar", "stream-2.5.2.jar", "thrift-server-0.3.7.jar", "cassandra-binding-0.15.0.jar", "cassandra-driver-core-3.0.0.jar", "core-0.15.0.jar", "guava-16.0.1.jar", "metrics-core-3.1.2.jar", "netty-buffer-4.0.33.Final.jar", "netty-codec-4.0.33.Final.jar", "netty-common-4.0.33.Final.jar", "netty-handler-4.0.33.Final.jar", "netty-transport-4.0.33.Final.jar", "slf4j-api-1.7.25.jar", "slf4j-simple-1.7.25.jar", "HdrHistogram-2.1.4.jar", "htrace-core4-4.1.0-incubating.jar", "jackson-core-asl-1.9.4.jar", "jackson-mapper-asl-1.9.4.jar";
size test args;
size test args "workloada";
\ No newline at end of file
b1a03860b38448c4db94126d416f1fcd
c2404e811972f30f42a6479db3dd59cb
\ No newline at end of file
......@@ -9,7 +9,6 @@
package org.dacapo.harness;
import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URL;
......@@ -21,12 +20,14 @@ import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.dacapo.harness.Benchmark;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.dacapo.parser.Config;
import org.apache.cassandra.service.EmbeddedCassandraService;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.Session;
public class Cassandra extends Benchmark {
public class CassandraBench extends Benchmark {
private String [] CASSANDRA_JARS = {
"cassandra-3.11.3.jar",
"cassandra-thrift-3.11.3.jar",
......@@ -81,6 +82,7 @@ public class Cassandra extends Benchmark {
"snowball-stemmer-1.3.0.581.1.jar",
"stream-2.5.2.jar",
"thrift-server-0.3.7.jar",
"cassandra-driver-core-3.6.0.jar"
};
private String[] YCSB_JARS = {
......@@ -111,11 +113,16 @@ public class Cassandra extends Benchmark {
private File dirCassandraConf;
private File dirCassandraStorage;
private File dirCassandraLog;
private File dirYCSBWorkloads;
private File ymlConf;
private File xmlLogback;
private EmbeddedCassandraService cassandra;
private String[] ycsbWorkloadArgs;
public Cassandra(Config config, File scratch) throws Exception {
Class<?> clsYCSBClient;
Method mtdYCSBClientMain;
public CassandraBench(Config config, File scratch) throws Exception {
super(config, scratch, false);
}
......@@ -138,14 +145,13 @@ public class Cassandra extends Benchmark {
throw e;
}
}
private void setupCassandra() {
clOriginal = Thread.currentThread().getContextClassLoader();
private void setupScratch() {
dirScratchJar = new File(scratch, "jar");
dirLibSigar = new File(scratch, "libsigar");
dirCassandraConf = new File(scratch, "cassandra-conf");
dirCassandraStorage = new File(scratch, "cassandra-storage");
dirCassandraLog = new File(scratch, "cassandra-log");
dirYCSBWorkloads = new File(scratch, "ycsb-workloads");
ymlConf = new File(dirCassandraConf, "cassandra.yaml");
xmlLogback = new File(dirCassandraConf, "logback.xml");
......@@ -154,10 +160,18 @@ public class Cassandra extends Benchmark {
dirCassandraConf.mkdir();
dirCassandraStorage.mkdir();
dirCassandraLog.mkdir();
dirYCSBWorkloads.mkdir();
try {
Benchmark.unpackZipFileResource("dat/libsigar.zip", dirLibSigar);
Benchmark.unpackZipFileResource("dat/cassandra-conf.zip", dirCassandraConf);
Benchmark.unpackZipFileResource("dat/ycsb-workloads.zip", dirYCSBWorkloads);
} catch (Exception e) {
e.printStackTrace();
System.exit(-1);
}
}
private void setupCassandra() {
try {
// XXX: requires Java <= 1.8 to work; for later versions, use reflection API.
addToSystemClassLoader(findJars(dirScratchJar, Arrays.asList(CASSANDRA_JARS)));
......@@ -179,13 +193,67 @@ public class Cassandra extends Benchmark {
protected void prepare(String size) throws Exception {
super.prepare(size);
// String[] cassandraArgs = config.getArgs(size);
clOriginal = Thread.currentThread().getContextClassLoader();
setupScratch();
setupCassandra();
cassandra.start();
clYCSB = new URLClassLoader(findJars(dirScratchJar, Arrays.asList(YCSB_JARS))
.toArray(new URL[0]), clOriginal);
clsYCSBClient = clYCSB.loadClass("com.yahoo.ycsb.Client");
mtdYCSBClientMain = clsYCSBClient.getMethod("main", String[].class);
prepareYCSBArgs(size);
prepareYCSBCQL();
}
private void prepareYCSBCQL() {
Cluster cluster = Cluster.builder()
.addContactPoint("localhost")
.withPort(DatabaseDescriptor.getNativeTransportPort())
.build();
Session session = cluster.connect();
session.execute("CREATE KEYSPACE ycsb WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};");
session.execute("USE ycsb;");
session.execute("CREATE TABLE usertable (" +
"y_id varchar primary key," +
"field0 varchar," +
"field1 varchar," +
"field2 varchar," +
"field3 varchar," +
"field4 varchar," +
"field5 varchar," +
"field6 varchar," +
"field7 varchar," +
"field8 varchar," +
"field9 varchar);");
session.close();
}
private void prepareYCSBArgs(String size) {
ArrayList<String> baseArgs = new ArrayList<String>(Arrays.asList(
"-db", "com.yahoo.ycsb.db.CassandraCQLClient",
"-p", "hosts=localhost"));
List<String> sizeArgs = Arrays.asList(config.getArgs(size));
File workload = new File(dirYCSBWorkloads, sizeArgs.get(0));
baseArgs.addAll(Arrays.asList("-P", workload.toString()));
baseArgs.addAll(sizeArgs.subList(1, sizeArgs.size()));
baseArgs.add(baseArgs.size(), "-t");
ycsbWorkloadArgs = baseArgs.toArray(new String[0]);
}
public void iterate(String size) throws Exception {
Thread.currentThread().setContextClassLoader(clYCSB);
// load workload
ycsbWorkloadArgs[ycsbWorkloadArgs.length - 1] = "-load";
mtdYCSBClientMain.invoke(null, (Object)ycsbWorkloadArgs);
// run transactions
ycsbWorkloadArgs[ycsbWorkloadArgs.length - 1] = "-t";
mtdYCSBClientMain.invoke(null, (Object)ycsbWorkloadArgs);
System.out.println("DaCapo: finished iteration");
}
private static List<URL> findJars(File dir, List<String> jarNames) {
......
diff --git orig/core/src/main/java/com/yahoo/ycsb/Client.java core/src/main/java/com/yahoo/ycsb/Client.java
index 00b1e378..9b55a30f 100644
--- orig/core/src/main/java/com/yahoo/ycsb/Client.java
+++ core/src/main/java/com/yahoo/ycsb/Client.java
@@ -715,9 +715,10 @@ public final class Client {
Measurements.getMeasurements().exportMeasurements(exporter);
} finally {
- if (exporter != null) {
- exporter.close();
- }
+ // DaCapo XXX: closing the exporter will also close the DaCapo TeePrintStream
+ // if (exporter != null) {
+ // exporter.close();
+ // }
}
}
@@ -844,8 +845,8 @@ public final class Client {
e.printStackTrace();
System.exit(-1);
}
-
- System.exit(0);
+ // DaCapo: don't bomb the system/process when finish
+ // System.exit(0);
}
private static List<ClientThread> initDb(String dbname, Properties props, int threadcount,
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment