AUT Benchmark

Machine

r5d.4xlarge
  - 16 virtual cores
  - 128G RAM
  - 2 x 300G ephemeral mount points
  - us-east-1

Setup

sudo mkdir /mnt/xfs

sudo zpool create au-zfs /dev/nvme1n1
sudo mkfs -t xfs

sudo chown -hR ubuntu:ubuntu /au-zfs /mnt/xfs

cd /au-zfs
mkdir -p data tmp output/auk/all-text output/auk/all-domains output/auk/gephi output/binaries/audio output/binaries/image output/binaries/pdf output/binaries/spreadsheet output/binaries/word-processor output/binaries/presentation-program output/binaries/text output/binaries/video output/df/audio output/df/image output/df/pdf output/df/spreadsheet output/df/word-processor output/df/presentation-program output/df/text output/df/video

cd /mnt/xfs
mkdir -p data tmp output/auk/all-text output/auk/all-domains output/auk/gephi output/binaries/audio output/binaries/image output/binaries/pdf output/binaries/spreadsheet output/binaries/word-processor output/binaries/presentation-program output/binaries/text output/binaries/video output/df/audio output/df/image output/df/pdf output/df/spreadsheet output/df/word-processor output/df/presentation-program output/df/text output/df/video

Notes

• ZFS
  • RAID 0
  • single device pool
• S3 buckets
  • no versions
  • no encryption
  • us-east-1

Jobs

AUK

r-xfs-w-xfs

rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-auk-01.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-auk-02.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-auk-03.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-auk-04.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-auk-05.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*

r-xfs-w-zfs

rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-auk-01.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-auk-02.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-auk-03.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-auk-04.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-auk-05.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /au-zfs/tmp

r-zfs-w-zfs

rm -rf /au-zfs/output/auk;  cd /au-zfs/output;  mkdir -p auk/all-text auk/all-domains auk/gephi;  rm -rf /au-zfs/tmp/*;  /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-auk-01.log;  rm -rf /au-zfs/output/auk;  cd /au-zfs/output;  mkdir -p auk/all-text auk/all-domains auk/gephi;  rm -rf /au-zfs/tmp/*;  /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-auk-02.log;  rm -rf /au-zfs/output/auk;  cd /au-zfs/output;  mkdir -p auk/all-text auk/all-domains auk/gephi;  rm -rf /au-zfs/tmp/*;  /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-auk-03.log;  rm -rf /au-zfs/output/auk;  cd /au-zfs/output;  mkdir -p auk/all-text auk/all-domains auk/gephi;  rm -rf /au-zfs/tmp/*;  /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-auk-04.log;  rm -rf /au-zfs/output/auk;  cd /au-zfs/output;  mkdir -p auk/all-text auk/all-domains auk/gephi;  rm -rf /au-zfs/tmp/*;  /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-auk-05.log;  rm -rf /au-zfs/output/auk;  cd /au-zfs/output;  mkdir -p auk/all-text auk/all-domains auk/gephi;  rm -rf /au-zfs/tmp/*

r-zfs-w-xfs

rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-auk-01.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-auk-02.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-auk-03.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-auk-04.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-auk-05.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; rm -rf /mnt/xfs/tmp/*

Binaries

r-xfs-w-xfs

rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-bin-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-bin-01.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-bin-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-bin-02.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-bin-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-bin-03.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-bin-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-bin-04.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-bin-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-bin-05.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*

r-xfs-w-zfs

rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-bin-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-bin-01.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-bin-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-bin-02.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-bin-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-bin-03.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-bin-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-bin-04.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-bin-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-bin-05.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*

r-zfs-w-zfs

rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-bin-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-bin-01.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-bin-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-bin-02.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-bin-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-bin-03.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-bin-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-bin-04.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-bin-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-bin-05.log; rm -rf /au-zfs/output/binaries; cd /au-zfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /au-zfs/tmp/*

r-zfs-w-xfs

rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-bin-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-bin-01.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-bin-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-bin-02.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-bin-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-bin-03.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-bin-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-bin-04.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-bin-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-bin.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-bin-05.log; rm -rf /mnt/xfs/output/binaries; cd /mnt/xfs/output; mkdir -p binaries/audio binaries/image binaries/pdf binaries/presentation-program binaries/spreadsheet binaries/text binaries/video binaries/word-processor; rm -rf /mnt/xfs/tmp/*

DataFrames

r-xfs-w-xfs

rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-df-01.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-df-02.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-df-03.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-df-04.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-xfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-xfs-df-05.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*

r-xfs-w-zfs

rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-df-01.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-df-02.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-df-03.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-df-04.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-zfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-zfs-df-05.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*

r-zfs-w-zfs

rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-df-01.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-df-02.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-df-03.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-df-04.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-zfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/au-zfs/tmp -Djava.io.tmpdir=/au-zfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-zfs-df-05.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /au-zfs/tmp/*

r-zfs-w-xfs

rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-df-01.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-df-02.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-df-03.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-df-04.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-xfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --conf spark.local.dir=/mnt/xfs/tmp -Djava.io.tmpdir=/mnt/xfs/tmp --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-xfs-df-05.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; rm -rf /mnt/xfs/tmp/*

S3 DataFrames

r-zfs-w-s3

aws s3 rb s3://au-df --force; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-s3-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-s3-df-01.log; aws s3 rb s3://au-df --force; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-s3-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-s3-df-02.log; aws s3 rb s3://au-df --force; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-s3-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-s3-df-03.log; aws s3 rb s3://au-df --force; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-s3-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-s3-df-04.log; aws s3 rb s3://au-df --force; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-zfs-w-s3-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-zfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-zfs-w-s3-df-05.log;

r-xfs-w-s3

aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-s3-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-s3-auk-01.log;  aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-s3-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-s3-auk-02.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-s3-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-s3-auk-03.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-s3-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-s3-auk-04.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-xfs-w-s3-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-xfs-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-xfs-w-s3-auk-05.log;

r-s3-w-s3

aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-s3-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-s3-df-01.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-s3-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-s3-df-02.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-s3-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-s3-df-03.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-s3-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-s3-df-04.log; aws s3 rb s3://au-df --force; sleep 60; aws s3 mb s3://au-df; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-s3-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-s3-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-s3-df-05.log;

r-s3-w-xfs

rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-xfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-xfs-df-01.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-xfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-xfs-df-02.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-xfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-xfs-df-03.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-xfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-xfs-df-04.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-xfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-xfs-df-05.log;

r-s3-w-zfs

rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-zfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-zfs-df-01.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-zfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-zfs-df-02.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-zfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-zfs-df-03.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-zfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-zfs-df-04.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-zfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-zfs-df-05.log;

Orca

r-s3-w-orca-auk

rm -rf /store/scratch/nruest/benchmark/auk; cd /store/scratch/nruest/benchmark; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-01.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-01.log; rm -rf /store/scratch/nruest/benchmark/auk; cd /store/scratch/nruest/benchmark; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-02.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-02.log; rm -rf /store/scratch/nruest/benchmark/auk; cd /store/scratch/nruest/benchmark; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-03.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-03.log; rm -rf /store/scratch/nruest/benchmark/auk; cd /store/scratch/nruest/benchmark; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-04.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-04.log; rm -rf /store/scratch/nruest/benchmark/auk; cd /store/scratch/nruest/benchmark; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-05.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-05.log; rm -rf /store/scratch/nruest/benchmark/auk; cd /store/scratch/nruest/benchmark; mkdir -p auk/all-text auk/all-domains auk/gephi;

r-s2-w-orca-df

rm -rf /store/scratch/nruest/benchmark/df; cd /store/scratch/nruest/benchmark; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-df-01.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-df-01.log; rm -rf /store/scratch/nruest/benchmark/df; cd /store/scratch/nruest/benchmark; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-df-02.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-df-02.log; rm -rf /store/scratch/nruest/benchmark/df; cd /store/scratch/nruest/benchmark; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-df-03.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-df-03.log; rm -rf /store/scratch/nruest/benchmark/df; cd /store/scratch/nruest/benchmark; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-df-04.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-df-04.log; rm -rf /store/scratch/nruest/benchmark/df; cd /store/scratch/nruest/benchmark; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-df-05.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g  --jars ~/aut-0.18.0-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-df-05.log; rm -rf /store/scratch/nruest/benchmark/df; cd /store/scratch/nruest/benchmark; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor;

Rho

df

rm -rf /mnt/vol1/derivative_data/tmp/df/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/df-01.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/df.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/df-01.log; rm -rf /mnt/vol1/derivative_data/tmp/df/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/df-02.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/df.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/df-02.log; rm -rf /mnt/vol1/derivative_data/tmp/df/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/df-03.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/df.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/df-03.log;rm -rf /mnt/vol1/derivative_data/tmp/df/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/df-04.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/df.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/df-04.log; rm -rf /mnt/vol1/derivative_data/tmp/df/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/df-05.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/df.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/df-05.log;

rdd

rm -rf /mnt/vol1/derivative_data/tmp/rdd/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/rdd-01.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/rdd.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/rdd-01.log; rm -rf /mnt/vol1/derivative_data/tmp/rdd/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/rdd-02.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/rdd.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/rdd-02.log; rm -rf /mnt/vol1/derivative_data/tmp/rdd/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/rdd-03.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/rdd.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/rdd-03.log;rm -rf /mnt/vol1/derivative_data/tmp/rdd/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/rdd-04.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/rdd.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/rdd-04.log; rm -rf /mnt/vol1/derivative_data/tmp/rdd/links-all; /usr/bin/time -o /home/nruest/jcdl-experiment/times/rdd-05.txt -v ~/bin/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local\[20\] --driver-memory 65g --jars /home/nruest/jcdl-experiment/aut-0.18.1-SNAPSHOT-fatjar.jar -i /home/nruest/jcdl-experiment/rdd.scala 2>&1 | tee /home/nruest/jcdl-experiment/logs/rdd-05.log;

EC2 + RADOS

r-RADOS-w-ec2-xfs-df

rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-df-01.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-df-02.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-df-03.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-df-04.log; rm -rf /mnt/xfs/output/df; cd /mnt/xfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-df-05.log;

r-RADOS-w-ec2-zfs-df

rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-df-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-df-01.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-df-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-df-02.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-df-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-df-03.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-df-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-df-04.log; rm -rf /au-zfs/output/df; cd /au-zfs/output; mkdir -p df/audio df/image df/pdf df/presentation-program df/spreadsheet df/text df/video df/word-processor; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-df-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-df.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-df-05.log;

r-RADOS-w-ec2-zfs-auk

rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-auk-01.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-auk-02.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-auk-03.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-auk-04.log; rm -rf /au-zfs/output/auk; cd /au-zfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-zfs-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-zfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-zfs-auk-05.log;

r-RADOS-w-ec2-xfs-auk

rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-auk-01.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-auk-01.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-auk-02.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-auk-02.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-auk-03.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-auk-03.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-auk-04.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-auk-04.log; rm -rf /mnt/xfs/output/auk; cd /mnt/xfs/output; mkdir -p auk/all-text auk/all-domains auk/gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-xfs-auk-05.txt -v ~/spark-2.4.4-bin-hadoop2.7/bin/spark-shell --master local[*] --driver-memory 125g  --jars ~/aut-hadoop-2.7.2.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-xfs-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-xfs-auk-05.log;

JCDL 2020

r-rados-w-ec2-auk

rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-ec2-auk-01.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-ec2-auk-01.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-ec2-auk-02.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-ec2-auk-02.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-ec2-auk-03.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-ec2-auk-03.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-ec2-auk-04.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-ec2-auk-04.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-ec2-auk-05.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-ec2-auk-05.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi;

r-s3-w-ec2-auk

rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-ec2-auk-01.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-ec2-auk-01.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-ec2-auk-02.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-ec2-auk-02.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-ec2-auk-03.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-ec2-auk-03.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-ec2-auk-04.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-ec2-auk-04.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-ec2-auk-05.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-ec2-auk-05.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi;

r-local-w-ec2-auk

rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-ec2-auk-01.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-ec2-auk-01.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-ec2-auk-02.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-ec2-auk-02.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-ec2-auk-03.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-ec2-auk-03.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-ec2-auk-04.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-ec2-auk-04.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-ec2-auk-05.txt -v /home/ubuntu/spark/bin/spark-shell --master local[*] --driver-memory 120g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-ec2-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-ec2-auk-05.log; rm -rf /dev/5467/gephi /dev/5467/all-text /dev/5467/all-domains; cd /dev/5467; mkdir -p all-text all-domains gephi;

r-rados-w-orca-auk

rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-orca-auk-01.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-orca-auk-01.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-orca-auk-02.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-orca-auk-02.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-orca-auk-03.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-orca-auk-03.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-orca-auk-04.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-orca-auk-04.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-rados-w-orca-auk-05.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-rados-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-rados-w-orca-auk-05.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi;

r-local-w-orca-auk

rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-orca-auk-01.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-orca-auk-01.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-orca-auk-02.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-orca-auk-02.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-orca-auk-03.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-orca-auk-03.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-orca-auk-04.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-orca-auk-04.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-local-w-orca-auk-05.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-local-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-local-w-orca-auk-05.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi;

r-s3-w-orca-auk

rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-01.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-01.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-02.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-02.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-03.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-03.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-04.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-04.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi; /usr/bin/time -o ~/aut-benchmark/times/r-s3-w-orca-auk-05.txt -v /store/share/spark/bin/spark-shell --master local\[\*\] --driver-memory 500g --jars ~/aut-0.50.1-SNAPSHOT-fatjar.jar -i ~/aut-benchmark/src/main/scala/r-s3-w-orca-auk.scala 2>&1 | tee ~/aut-benchmark/logs/r-s3-w-orca-auk-05.log; rm -rf /store/scratch/nruest/web_archives/winnipeg/5467/gephi /store/scratch/nruest/web_archives/winnipeg/5467/all-text /store/scratch/nruest/web_archives/winnipeg/5467/all-domains; cd /store/scratch/nruest/web_archives/winnipeg/5467; mkdir -p all-text all-domains gephi;

Results