Skip to content

Commit

Permalink
Merge pull request hyrise#258 from bastih/feature/concurrent_delta
Browse files Browse the repository at this point in the history
Concurrent delta
  • Loading branch information
bastih committed Nov 22, 2013
2 parents 1e60f58 + d33d07c commit 97f60d5
Show file tree
Hide file tree
Showing 26 changed files with 599 additions and 225 deletions.
98 changes: 91 additions & 7 deletions src/bin/perf_regression/InsertScan.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
#include <gtest/gtest-bench.h>
#include <gtest/gtest.h>
#include <string>

#include <chrono>
#include <access.h>
#include <storage.h>
#include <io.h>
Expand All @@ -13,6 +13,8 @@
//JoinScan Benchmark similar to TPC-C Implementation of Stock-Level Transaction
//See TPC-C Reference Chapter A.5

namespace hyrise { namespace access {

class InsertScanBase : public ::testing::Benchmark {

protected:
Expand Down Expand Up @@ -47,12 +49,94 @@ class InsertScanBase : public ::testing::Benchmark {
};

BENCHMARK_F(InsertScanBase, insert_single_tx_no_commit)
{
is.execute();
}
{
is.execute();
}

BENCHMARK_F(InsertScanBase, insert_single_tx_commit)
{
is.execute();
c.execute();
{
is.execute();
c.execute();
}


template <typename F>
void bench(F func, std::size_t threads_sz=10) {
std::vector<std::thread> threads;
for (std::size_t i=0;i<threads_sz; i++) {
threads.emplace_back(func);
}
for (auto& t: threads) {
t.join();
}
}

typedef struct { storage::atable_ptr_t base; storage::atable_ptr_t insert; std::size_t threads; } params;

class InsertTest : public ::testing::TestWithParam<params> {

};

std::vector<params> generateParams() {
std::vector<params> results;
for (std::size_t t:{1,2,4,8,16,32}) {
for (std::size_t opt=0; opt < 3; opt++) {
auto data = Loader::shortcuts::load("test/test10k_12.tbl");
storage::atable_ptr_t tbl;
// slightly hack-ish: opt distinguishes different insertion options
if (opt == 0) {
tbl = Loader::shortcuts::load("test/test10k_12.tbl", Loader::params().setReturnsMutableVerticalTable(true));
} else {
tbl = data->copy_structure_modifiable();
tbl->resize(opt);
for (auto col=0u; col<20; ++col) {
for (auto row=0u; row<opt; ++row) {
tbl->setValue<hyrise_int_t>(col, row, row+col);
}
}
}
results.push_back({data, tbl, t});
}
}
return results;
}


TEST_P(InsertTest, concurrent_writes_single_insert) {
auto param = GetParam();
auto data = param.base;
auto tbl = param.insert;
auto d_before = data->size();
auto insert_rows = tbl->size();
const auto runs = 1000000 / insert_rows / param.threads;
auto before = std::chrono::high_resolution_clock::now();
auto l = [&]() {
for (std::size_t d=0;d<runs;d++) {
auto ctx = hyrise::tx::TransactionManager::getInstance().buildContext();
InsertScan is;
is.setEvent("NO_PAPI");
is.setTXContext(ctx);
is.addInput(data);
is.setInputData(tbl);
Commit c;
c.setEvent("NO_PAPI");
c.setTXContext(ctx);
is.execute();
c.execute();
}
};
bench(l, param.threads);

auto after = std::chrono::high_resolution_clock::now();
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(after - before).count();
this->RecordProperty("threads", param.threads);
this->RecordProperty("inserts/commit", tbl->size());
this->RecordProperty("rows/ms", (data->size() - d_before) / ms);
}

INSTANTIATE_TEST_CASE_P(InsertTestInst,
InsertTest,
::testing::ValuesIn(generateParams()));


}}
2 changes: 1 addition & 1 deletion src/bin/units_access/InsertScanTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class InsertScanTests : public AccessTest {};

TEST_F(InsertScanTests, basic_insert_scan_test) {
auto row = Loader::shortcuts::load("test/insert_one.tbl");
storage::atable_ptr_t table(new storage::Store(row));
auto table = Loader::shortcuts::load("test/insert_one.tbl");

auto ctx = tx::TransactionManager::getInstance().buildContext();

Expand Down
5 changes: 1 addition & 4 deletions src/bin/units_access/ops_groupby.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -542,12 +542,9 @@ TEST_F(GroupByTests, group_by_scan_with_sum_and_one_arg) {
TEST_F(GroupByTests, group_multi_table) {

// Load raw data
hyrise::storage::atable_ptr_t t = Loader::shortcuts::load("test/10_30_group.tbl");
hyrise::storage::atable_ptr_t s = Loader::shortcuts::load("test/10_30_group.tbl");
hyrise::storage::atable_ptr_t reference = Loader::shortcuts::load("test/reference/group_by_scan_with_sum_and_one_arg.tbl");

// make it modifiable
auto s = std::make_shared<storage::Store>(t);

auto sum = new SumAggregateFun(0);

hyrise::access::GroupByScan gs;
Expand Down
7 changes: 3 additions & 4 deletions src/bin/units_access/ops_select.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -250,9 +250,8 @@ TEST_F(SelectTests, simple_projection_on_empty_table) {

TEST_F(SelectTests, select_after_insert_simple) {
auto ctx = tx::TransactionManager::getInstance().buildContext();
hyrise::storage::atable_ptr_t t = Loader::shortcuts::load("test/lin_xxs.tbl");
auto s = std::make_shared<const storage::Store>(t);

hyrise::storage::atable_ptr_t s = Loader::shortcuts::load("test/lin_xxs.tbl");
auto initial_size = s->size();
hyrise::storage::atable_ptr_t data = s->copy_structure_modifiable(nullptr, s->size());
data->resize(1);
for (std::size_t i=0; i <= 9; ++i) {
Expand All @@ -266,7 +265,7 @@ TEST_F(SelectTests, select_after_insert_simple) {
isc.execute();


ASSERT_EQ(t->size() + 1, isc.getResultTable(0)->size());
ASSERT_EQ(initial_size + 1, isc.getResultTable(0)->size());
}


Expand Down
202 changes: 39 additions & 163 deletions src/bin/units_storage/attribute_vector_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,172 +1,48 @@
// Copyright (c) 2012 Hasso-Plattner-Institut fuer Softwaresystemtechnik GmbH. All rights reserved.
#include "testing/test.h"
#include <storage.h>

#include <storage/BitCompressedVector.h>
#include <storage/FixedLengthVector.h>
#include <storage/AttributeVectorFactory.h>

template <typename T>
class AttributeVectorTests : public ::hyrise::Test {
public:

};

template<bool Compressed>
struct TestType {
static const bool compressed = Compressed;
};


TEST( FixedLengthVectorTest, increment_test ) {
size_t cols = 1;
size_t rows = 3;

auto tuples = std::dynamic_pointer_cast<FixedLengthVector<value_id_t>>(
AttributeVectorFactory::getAttributeVector2<value_id_t>(cols,rows, false));

tuples->resize(rows);

EXPECT_EQ(0u, tuples->get(0,0));
EXPECT_EQ(0u, tuples->inc(0,0));
EXPECT_EQ(1u, tuples->get(0,0));
EXPECT_EQ(1u, tuples->atomic_inc(0,0));
EXPECT_EQ(2u, tuples->get(0,0));
}

static const std::vector<uint64_t> bits = std::vector<uint64_t> {2, 3};

using testing::Types;

typedef Types <
TestType<false >,
TestType<true >
// BitCompressedVector<uint>,
// BitCompressedVector<uint, StrategizedAllocator<uint, MemalignStrategy<16> > >,
// BitCompressedVector<uint, StrategizedAllocator<uint, NumaNodeStrategy<NumaConfig> > >,
// FixedLengthVector<uint>,
// FixedLengthVector<uint, StrategizedAllocator<uint, MemalignStrategy<16> > >,
// FixedLengthVector<uint, StrategizedAllocator<uint, NumaNodeStrategy<NumaConfig> > >
> Vectors;

TYPED_TEST_CASE(AttributeVectorTests, Vectors);

TYPED_TEST(AttributeVectorTests, boundaries_test) {
size_t cols = 2;
size_t rows = 3;
auto tuples = AttributeVectorFactory::getAttributeVector2<value_id_t>(cols, rows, TypeParam::compressed, bits);
tuples->resize(rows);
#ifdef EXPENSIVE_ASSERTIONS
EXPECT_THROW(tuples->get(cols, 0), std::out_of_range);
EXPECT_THROW(tuples->get(0, rows), std::out_of_range);
EXPECT_THROW(tuples->get(cols, rows), std::out_of_range);
#endif
}

TYPED_TEST(AttributeVectorTests, base_test) {

auto tuples = AttributeVectorFactory::getAttributeVector2<value_id_t>(1, 3, TypeParam::compressed, bits);
tuples->resize(3);

tuples->set(0, 0, 2);
tuples->set(0, 1, 3);
tuples->set(0, 2, 1);

ASSERT_EQ(3u, tuples->size());

ASSERT_EQ(2u, tuples->get(0, 0));
ASSERT_EQ(3u, tuples->get(0, 1));
ASSERT_EQ(1u, tuples->get(0, 2));

tuples->clear();

ASSERT_EQ(0u, tuples->size());
}

TYPED_TEST(AttributeVectorTests, two_columns) {
auto tuples = AttributeVectorFactory::getAttributeVector2<value_id_t>(2, 3, TypeParam::compressed, bits);
tuples->resize(3);

tuples->set(0, 0, 2);
tuples->set(0, 1, 3);
tuples->set(0, 2, 1);

tuples->set(1, 0, 5);
tuples->set(1, 1, 7);

ASSERT_EQ(3u, tuples->size());

ASSERT_EQ(2u, tuples->get(0, 0));
ASSERT_EQ(3u, tuples->get(0, 1));
ASSERT_EQ(1u, tuples->get(0, 2));

ASSERT_EQ(5u, tuples->get(1, 0));
ASSERT_EQ(7u, tuples->get(1, 1));

tuples->clear();

ASSERT_EQ(0u, tuples->size());
}

TYPED_TEST(AttributeVectorTests, copy_test) {

auto tuples = AttributeVectorFactory::getAttributeVector2<value_id_t>(2, 3, TypeParam::compressed, bits);
tuples->resize(3);

tuples->set(0, 0, 2);
tuples->set(0, 1, 3);
tuples->set(0, 2, 1);

tuples->set(1, 0, 5);
tuples->set(1, 1, 7);

std::shared_ptr<BaseAttributeVector<uint32_t>> copy = tuples->copy();
tuples->clear();

ASSERT_EQ(3u, copy->size());

ASSERT_EQ(2u, copy->get(0, 0));
ASSERT_EQ(3u, copy->get(0, 1));
ASSERT_EQ(1u, copy->get(0, 2));

ASSERT_EQ(5u, copy->get(1, 0));
ASSERT_EQ(7u, copy->get(1, 1));

copy->clear();

ASSERT_EQ(0u, copy->size());
#include <limits>

#include "storage/storage_types.h"
#include "storage/BitCompressedVector.h"
#include "storage/FixedLengthVector.h"

TEST(BitCompressedTests, set_retrieve_bits) {
std::vector<uint64_t> bits {1, 2, 4, 8, 13};
BitCompressedVector<value_id_t> tuples(5, 2, bits);
tuples.resize(2);
auto col = 0;
for (auto bit: bits) {
auto maxval = (1 << bit) - 1; // Maximum value that can be stored is 2^bit-1
tuples.set(col, 0, 0);
tuples.set(col, 1, maxval);
EXPECT_EQ(0u, tuples.get(col, 0));
EXPECT_EQ(maxval, tuples.get(col, 1));
col++;
}
}

TYPED_TEST(AttributeVectorTests, empty_does_not_fail_when_reserve_test) {
auto tuples = AttributeVectorFactory::getAttributeVector2<value_id_t>(1, 0, TypeParam::compressed, bits);
tuples->resize(3);

// when adding three elements
tuples->set(0, 0, 2);
tuples->set(0, 1, 3);
tuples->set(0, 2, 1);

ASSERT_EQ(3u, tuples->size());

ASSERT_EQ(2u, tuples->get(0, 0));
ASSERT_EQ(3u, tuples->get(0, 1));
ASSERT_EQ(1u, tuples->get(0, 2));
TEST(BitCompressedTests, empty_size_does_not_change_with_reserve) {
BitCompressedVector<value_id_t> tuples(1, 1, {1});
ASSERT_EQ(0u, tuples.size());
ASSERT_EQ(64u, tuples.capacity());
tuples.reserve(3);
ASSERT_EQ(0u, tuples.size());
ASSERT_EQ(64u, tuples.capacity());
tuples.reserve(65);
ASSERT_EQ(0u, tuples.size());
ASSERT_EQ(128u, tuples.capacity());
}

TYPED_TEST(AttributeVectorTests, empty_size_does_not_change_with_reserve) {

auto tuples = AttributeVectorFactory::getAttributeVector2<value_id_t>(1, 1, TypeParam::compressed, std::vector<uint64_t> {1});
ASSERT_EQ(0u, tuples->size());
if (TypeParam::compressed)
ASSERT_EQ(64u, tuples->capacity());
else
ASSERT_EQ(1u, tuples->capacity());
tuples->reserve(3);
ASSERT_EQ(0u, tuples->size());
if (TypeParam::compressed)
ASSERT_EQ(64u, tuples->capacity());
else
ASSERT_EQ(3u, tuples->capacity());
TEST(FixedLengthVectorTest, increment_test) {
size_t cols = 1;
size_t rows = 3;

FixedLengthVector<value_id_t> tuples(cols, rows);
tuples.resize(rows);
EXPECT_EQ(0u, tuples.get(0,0));
EXPECT_EQ(0u, tuples.inc(0,0));
EXPECT_EQ(1u, tuples.get(0,0));
EXPECT_EQ(1u, tuples.atomic_inc(0,0));
EXPECT_EQ(2u, tuples.get(0,0));
}

5 changes: 3 additions & 2 deletions src/bin/units_storage/dictionary_tests.cpp
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
// Copyright (c) 2012 Hasso-Plattner-Institut fuer Softwaresystemtechnik GmbH. All rights reserved.
#include "testing/test.h"

#include <io.h>
#include <storage.h>
#include "storage/OrderIndifferentDictionary.h"
#include "storage/OrderPreservingDictionary.h"

class DictionaryTest : public ::hyrise::Test {};

Expand Down Expand Up @@ -123,3 +123,4 @@ TEST_F(DictionaryTest, order_preserving_string_exists) {
ASSERT_FALSE(dict.valueExists("321"));

}

Loading

0 comments on commit 97f60d5

Please sign in to comment.