Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Thesis/mongodb and redis modubft #1723

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .mvn/local-settings.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
<settings xmlns="http://maven.apache.org/SETTINGS/1.2.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 http://maven.apache.org/xsd/settings-1.2.0.xsd">
<mirrors>
<mirror>
<id>my-repository-http-unblocker</id>
<mirrorOf>allanbank</mirrorOf>
<name></name>
<url>http://www.allanbank.com/repo/</url>
</mirror>
</mirrors>
</settings>
1 change: 1 addition & 0 deletions .mvn/maven.config
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
--settings=./.mvn/local-settings.xml
6 changes: 4 additions & 2 deletions bin/ycsb
Original file line number Diff line number Diff line change
@@ -103,7 +103,9 @@ DATABASES = {
"solr7" : "site.ycsb.db.solr7.SolrClient",
"tarantool" : "site.ycsb.db.TarantoolClient",
"tablestore" : "site.ycsb.db.tablestore.TableStoreClient",
"zookeeper" : "site.ycsb.db.zookeeper.ZKClient"
"zookeeper" : "site.ycsb.db.zookeeper.ZKClient",
"mongodb-modubft" : "site.ycsb.db.MongoDbModubftClient",
"redis-modubft" : "site.ycsb.db.RedisModubftClient"
}

OPTIONS = {
@@ -216,7 +218,7 @@ def get_classpath_from_maven(module):
"-am", "package", "-DskipTests",
"dependency:build-classpath",
"-DincludeScope=compile",
"-Dmdep.outputFilterFile=true"])
"-Dmdep.outputFilterFile=true", "-Dcheckstyle.skip"])
# the above outputs a "classpath=/path/tojar:/path/to/other/jar" for each module
# the last module will be the datastore binding
line = [x for x in mvn_output.splitlines() if x.startswith("classpath=")][-1:]
30 changes: 30 additions & 0 deletions copy_jars.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash
db=$1

if [ -z $1 ]; then
echo "db - redis or mongodb not provided"
exit 1
fi

DEST_DIR="./jars/"
mkdir -p "$DEST_DIR"

if [ $1 = "redis"]; then
CLASSPATH="/home/snayyer/repos/non-work/studium/thesis/YCSB/redis/conf:/home/snayyer/repos/non-work/studium/thesis/YCSB/redis/target/redis-binding-0.18.0-SNAPSHOT.jar:/home/snayyer/.m2/repository/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/home/snayyer/.m2/repository/com/google/protobuf/protobuf-java/3.19.6/protobuf-java-3.19.6.jar:/home/snayyer/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.4/HdrHistogram-2.1.4.jar:/home/snayyer/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.4/jackson-mapper-asl-1.9.4.jar:/home/snayyer/.m2/repository/redis/clients/jedis/2.9.0/jedis-2.9.0.jar:/home/snayyer/.m2/repository/org/apache/commons/commons-pool2/2.4.2/commons-pool2-2.4.2.jar:/home/snayyer/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.4/jackson-core-asl-1.9.4.jar:/home/snayyer/repos/non-work/studium/thesis/YCSB/core/target/core-0.18.0-SNAPSHOT.jar"
elif [ $1 = "mongodb" ]; then
CLASSPATH="/home/snayyer/repos/non-work/studium/thesis/YCSB/mongodb/conf:/home/snayyer/repos/non-work/studium/thesis/YCSB/mongodb/target/mongodb-binding-0.18.0-SNAPSHOT.jar:/home/snayyer/.m2/repository/org/apache/htrace/htrace-core4/4.1.0-incubating/htrace-core4-4.1.0-incubating.jar:/home/snayyer/.m2/repository/com/google/protobuf/protobuf-java/3.19.6/protobuf-java-3.19.6.jar:/home/snayyer/.m2/repository/org/xerial/snappy/snappy-java/1.1.7.1/snappy-java-1.1.7.1.jar:/home/snayyer/.m2/repository/org/hdrhistogram/HdrHistogram/2.1.4/HdrHistogram-2.1.4.jar:/home/snayyer/.m2/repository/org/mongodb/mongo-java-driver/3.11.0/mongo-java-driver-3.11.0.jar:/home/snayyer/.m2/repository/org/codehaus/jackson/jackson-mapper-asl/1.9.4/jackson-mapper-asl-1.9.4.jar:/home/snayyer/.m2/repository/org/codehaus/jackson/jackson-core-asl/1.9.4/jackson-core-asl-1.9.4.jar:/home/snayyer/repos/non-work/studium/thesis/YCSB/core/target/core-0.18.0-SNAPSHOT.jar:/home/snayyer/.m2/repository/com/allanbank/mongodb-async-driver/2.0.1/mongodb-async-driver-2.0.1.jar"
else
echo "Unknown db - neither redis nor mongodb"
exit 1
fi

IFS=':' read -ra JARS <<< "$CLASSPATH"
for JAR in "${JARS[@]}"; do
if [ -e "$JAR" ]; then
cp "$JAR" "$DEST_DIR"
else
echo "Skipping $JAR as it does not exist."
fi
done

scp ./jars/* snayyer@zs01:/home/snayyer/YCSB/jars
46 changes: 28 additions & 18 deletions core/src/main/java/site/ycsb/Client.java
Original file line number Diff line number Diff line change
@@ -75,7 +75,7 @@ public static StringBuilder format(long seconds) {
*/
public final class Client {
private Client() {
//not used
// not used
}

public static final String DEFAULT_RECORD_COUNT = "0";
@@ -119,8 +119,10 @@ private Client() {

/**
* Indicates how many inserts to do if less than recordcount.
* Useful for partitioning the load among multiple servers if the client is the bottleneck.
* Additionally workloads should support the "insertstart" property which tells them which record to start at.
* Useful for partitioning the load among multiple servers if the client is the
* bottleneck.
* Additionally workloads should support the "insertstart" property which tells
* them which record to start at.
*/
public static final String INSERT_COUNT_PROPERTY = "insertcount";

@@ -204,12 +206,12 @@ public static boolean checkRequiredProperties(Properties props) {
return true;
}


/**
* Exports the measurements to either sysout or a file using the exporter
* loaded from conf.
*
* @throws IOException Either failed to write to output stream or failed to close it.
* @throws IOException Either failed to write to output stream or failed to
* close it.
*/
private static void exportMeasurements(Properties props, int opcount, long runtime)
throws IOException {
@@ -282,12 +284,12 @@ public static void main(String[] args) {

long maxExecutionTime = Integer.parseInt(props.getProperty(MAX_EXECUTION_TIME, "0"));

//get number of threads, target and db
// get number of threads, target and db
int threadcount = Integer.parseInt(props.getProperty(THREAD_COUNT_PROPERTY, "1"));
String dbname = props.getProperty(DB_PROPERTY, "site.ycsb.BasicDB");
int target = Integer.parseInt(props.getProperty(TARGET_PROPERTY, "0"));

//compute the target throughput
// compute the target throughput
double targetperthreadperms = -1;
if (target > 0) {
double targetperthread = ((double) target) / ((double) threadcount);
@@ -371,7 +373,8 @@ public static void main(String[] args) {
if (status) {
// wake up status thread if it's asleep
statusthread.interrupt();
// at this point we assume all the monitored threads are already gone as per above join loop.
// at this point we assume all the monitored threads are already gone as per
// above join loop.
try {
statusthread.join();
} catch (InterruptedException ignored) {
@@ -401,8 +404,8 @@ public static void main(String[] args) {
}

private static List<ClientThread> initDb(String dbname, Properties props, int threadcount,
double targetperthreadperms, Workload workload, Tracer tracer,
CountDownLatch completeLatch) {
double targetperthreadperms, Workload workload, Tracer tracer,
CountDownLatch completeLatch) {
boolean initFailed = false;
boolean dotransactions = Boolean.valueOf(props.getProperty(DO_TRANSACTIONS_PROPERTY, String.valueOf(true)));

@@ -418,14 +421,17 @@ private static List<ClientThread> initDb(String dbname, Properties props, int th
opcount = Integer.parseInt(props.getProperty(RECORD_COUNT_PROPERTY, DEFAULT_RECORD_COUNT));
}
}
if (threadcount > opcount && opcount > 0){
if (threadcount > opcount && opcount > 0) {
threadcount = opcount;
System.out.println("Warning: the threadcount is bigger than recordcount, the threadcount will be recordcount!");
}
for (int threadid = 0; threadid < threadcount; threadid++) {
DB db;
try {
db = DBFactory.newDB(dbname, props, tracer);
Properties newProps = (Properties) props.clone();
newProps.setProperty("threadId", Integer.toString(threadid));
db.setProperties(newProps);
} catch (UnknownDBException e) {
System.out.println("Unknown DB " + dbname);
initFailed = true;
@@ -434,7 +440,8 @@ private static List<ClientThread> initDb(String dbname, Properties props, int th

int threadopcount = opcount / threadcount;

// ensure correct number of operations, in case opcount is not a multiple of threadcount
// ensure correct number of operations, in case opcount is not a multiple of
// threadcount
if (threadid < opcount % threadcount) {
++threadopcount;
}
@@ -484,9 +491,10 @@ private static HTraceConfiguration getHTraceConfiguration(Properties props) {
}

private static Thread setupWarningThread() {
//show a warning message that creating the workload is taking a while
//but only do so if it is taking longer than 2 seconds
//(showing the message right away if the setup wasn't taking very long was confusing people)
// show a warning message that creating the workload is taking a while
// but only do so if it is taking longer than 2 seconds
// (showing the message right away if the setup wasn't taking very long was
// confusing people)
return new Thread() {
@Override
public void run() {
@@ -610,7 +618,8 @@ private static Properties parseArguments(String[] args) {
System.exit(0);
}

//Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
// Issue #5 - remove call to stringPropertyNames to make compilable under Java
// 1.5
for (Enumeration e = myfileprops.propertyNames(); e.hasMoreElements();) {
String prop = (String) e.nextElement();

@@ -658,9 +667,10 @@ private static Properties parseArguments(String[] args) {
System.exit(0);
}

//overwrite file properties with properties from the command line
// overwrite file properties with properties from the command line

//Issue #5 - remove call to stringPropertyNames to make compilable under Java 1.5
// Issue #5 - remove call to stringPropertyNames to make compilable under Java
// 1.5
for (Enumeration e = props.propertyNames(); e.hasMoreElements();) {
String prop = (String) e.nextElement();

6 changes: 6 additions & 0 deletions mongodb/pom.xml
Original file line number Diff line number Diff line change
@@ -67,6 +67,12 @@ LICENSE file.
<type>jar</type>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.19.6</version>
</dependency>
</dependencies>
<repositories>
<repository>
Loading