Skip to content

Commit

Permalink
Merge latest fixes from master
Browse files Browse the repository at this point in the history
  • Loading branch information
Kai Mast committed Feb 16, 2015
2 parents bb9fd63 + d090e10 commit e77f139
Show file tree
Hide file tree
Showing 31 changed files with 534 additions and 302 deletions.
3 changes: 2 additions & 1 deletion Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -844,7 +844,7 @@ stress_wrappers += test/sh/search.combination.keytype=string,daemons=4.fault-tol
shell_wrappers += $(stress_wrappers)

python_wrappers =
#python_wrappers += test/sh/bindings.python.Admin.sh
python_wrappers += test/sh/bindings.python.Admin.sh
python_wrappers += test/sh/bindings.python.BasicSearch.sh
python_wrappers += test/sh/bindings.python.Basic.sh
python_wrappers += test/sh/bindings.python.CondPut.sh
Expand All @@ -870,6 +870,7 @@ python_wrappers += test/sh/bindings.python.DataTypeString.sh
python_wrappers += test/sh/bindings.python.GroupAtomic.sh
python_wrappers += test/sh/bindings.python.HyperMongo.sh
python_wrappers += test/sh/bindings.python.LengthString.sh
python_wrappers += test/sh/bindings.python.Microtransactions.sh
python_wrappers += test/sh/bindings.python.MultiAttribute.sh
python_wrappers += test/sh/bindings.python.RangeSearchInt.sh
python_wrappers += test/sh/bindings.python.RangeSearchString.sh
Expand Down
13 changes: 10 additions & 3 deletions client/client.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -774,14 +774,21 @@ client :: prepare_checks(const char* space, const schema& sc,
return i;
}

if (!path && datatype == HYPERDATATYPE_DOCUMENT)
{
// Document datatype always requires a path. Empty means root.
path = "";
}

if (path)
{
size_t path_sz = strlen(path) + 1;
size_t sz = path_sz + chks[i].value_sz;
size_t sz = path_sz + c.value.size();
unsigned char* tmp = NULL;
memory->allocate(path_sz + chks[i].value_sz, &tmp);
memory->allocate(path_sz + c.value.size(), &tmp);
memmove(tmp, path, path_sz);
memmove(tmp + path_sz, chks[i].value, chks[i].value_sz);
memmove(tmp + path_sz, c.value.data(), c.value.size());

c.value = e::slice(tmp, sz);
}

Expand Down
80 changes: 57 additions & 23 deletions common/datatype_document.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,15 @@
#include "common/datatype_int64.h"
#include "common/datatype_float.h"

#define IS_DOCUMENT_PRIMITIVE(X) \
((X) == HYPERDATATYPE_STRING || \
(X) == HYPERDATATYPE_INT64 || \
(X) == HYPERDATATYPE_FLOAT || \
(X) == HYPERDATATYPE_DOCUMENT)
inline bool is_numeral(const hyperdatatype& t)
{
return t == HYPERDATATYPE_FLOAT || t == HYPERDATATYPE_INT64;
}

inline bool is_document_primitive(const hyperdatatype& t)
{
return is_numeral(t) || t == HYPERDATATYPE_STRING || t == HYPERDATATYPE_DOCUMENT;
}

using hyperdex::datatype_document;

Expand Down Expand Up @@ -76,7 +80,7 @@ datatype_document :: check_args(const funcall& func) const
{
// A transformation that either sets (all/part of) the document, or
// pushes/pops an array (that is/within) the document.
if (IS_DOCUMENT_PRIMITIVE(func.arg1_datatype) &&
if (is_document_primitive(func.arg1_datatype) &&
(func.name == FUNC_SET ||
func.name == FUNC_LIST_LPUSH ||
func.name == FUNC_LIST_RPUSH))
Expand All @@ -103,7 +107,7 @@ datatype_document :: check_args(const funcall& func) const
}
// Perform a nested operation on primitives
else if (func.arg2_datatype == HYPERDATATYPE_STRING &&
IS_DOCUMENT_PRIMITIVE(func.arg1_datatype) &&
is_document_primitive(func.arg1_datatype) &&
func.arg1_datatype != HYPERDATATYPE_DOCUMENT)
{
datatype_info* di = datatype_info::lookup(func.arg1_datatype);
Expand Down Expand Up @@ -153,7 +157,7 @@ datatype_document :: apply(const e::slice& old_value,
{
const funcall& func = funcs[idx];

if (IS_DOCUMENT_PRIMITIVE(func.arg1_datatype) &&
if (is_document_primitive(func.arg1_datatype) &&
(func.name == FUNC_SET ||
func.name == FUNC_LIST_LPUSH ||
func.name == FUNC_LIST_RPUSH))
Expand Down Expand Up @@ -226,7 +230,7 @@ datatype_document :: apply(const e::slice& old_value,
}
}
else if (func.arg2_datatype == HYPERDATATYPE_STRING &&
IS_DOCUMENT_PRIMITIVE(func.arg1_datatype) &&
is_document_primitive(func.arg1_datatype) &&
func.arg1_datatype != HYPERDATATYPE_DOCUMENT)
{
unsigned char* value = NULL;
Expand All @@ -239,16 +243,19 @@ datatype_document :: apply(const e::slice& old_value,

if (treadstone_transformer_extract_value(trans, path.c_str(), &value, &value_sz) < 0)
{
// Value doesn't exist yet
type = func.arg1_datatype;
v = e::slice();
}
else
{
// Extract existing value from the document
if (!coerce_binary_to_primitive(e::slice(value, value_sz), &type, &scratch, &v))
{
return false;
}


if (type == HYPERDATATYPE_INT64 &&
func.arg1_datatype == HYPERDATATYPE_FLOAT)
{
Expand All @@ -257,6 +264,22 @@ datatype_document :: apply(const e::slice& old_value,
datatype_float::pack(d, &scratch, &v);
type = HYPERDATATYPE_FLOAT;
}

// Both types are numberal (integer or float)
const bool numeral = is_numeral(type) && is_numeral(func.arg1_datatype);

if (func.name != FUNC_SET &&
func.name != FUNC_DOC_UNSET &&
func.name != FUNC_DOC_RENAME &&
!numeral)
{
// More complex modifications (string append etc)
// only work with the same type
if(type != func.arg1_datatype)
{
return false;
}
}
}

datatype_info* di = datatype_info::lookup(type);
Expand All @@ -267,6 +290,7 @@ datatype_document :: apply(const e::slice& old_value,
return false;
}

// Pass funcall down to underlying datatype (string, integer etc...)
if (!di->apply(v, &func, 1, new_memory, &tmp_value))
{
return false;
Expand Down Expand Up @@ -348,11 +372,8 @@ bool
datatype_document :: document_check(const attribute_check& check,
const e::slice& doc) const
{
if (check.datatype == HYPERDATATYPE_DOCUMENT)
{
return check.predicate == HYPERPREDICATE_EQUALS &&
check.value == doc;
}
// We expected the follwing format:
// <path>\0\n<value>

const char* path = reinterpret_cast<const char*>(check.value.data());
size_t path_sz = strnlen(path, check.value.size());
Expand All @@ -371,13 +392,24 @@ datatype_document :: document_check(const attribute_check& check,
return false;
}

attribute_check new_check;
new_check.attr = check.attr;
new_check.value = check.value;
new_check.datatype = check.datatype;
new_check.predicate = check.predicate;
new_check.value.advance(path_sz + 1);
return passes_attribute_check(type, new_check, value);
if(type == HYPERDATATYPE_DOCUMENT)
{
// Compare two subdocuments
e::slice chk_value = check.value;
chk_value.advance(path_sz + 1);
return (value == chk_value);
}
else
{
// Pass down to underlying datatype
attribute_check new_check;
new_check.attr = check.attr;
new_check.value = check.value;
new_check.datatype = check.datatype;
new_check.predicate = check.predicate;
new_check.value.advance(path_sz + 1);
return passes_attribute_check(type, new_check, value);
}
}

bool
Expand Down Expand Up @@ -414,7 +446,7 @@ datatype_document :: coerce_primitive_to_binary(hyperdatatype type,
std::vector<char>* scratch,
e::slice* value) const
{
assert(IS_DOCUMENT_PRIMITIVE(type));
assert(is_document_primitive(type));
unsigned char* v = NULL;
size_t v_sz = 0;
e::guard g = e::makeguard(free_if_allocated, &v);
Expand Down Expand Up @@ -483,7 +515,9 @@ datatype_document :: coerce_binary_to_primitive(const e::slice& in,
else
{
*type = HYPERDATATYPE_DOCUMENT;
*value = in;
scratch->resize(in.size());
memmove(scratch->data(), in.data(), in.size());
*value = e::slice(&(*scratch)[0], in.size());
}

return true;
Expand Down
8 changes: 4 additions & 4 deletions daemon/datalayer.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -698,8 +698,8 @@ datalayer :: make_search_iterator(snapshot snap,
// pull a set of range queries from checks
std::vector<range> ranges;
range_searches(sc, checks, &ranges);
index_encoding* key_ie = index_encoding::lookup(sc.attrs[0].type);
index_info* key_ii = index_info::lookup(sc.attrs[0].type);
const index_encoding* key_ie = index_encoding::lookup(sc.attrs[0].type);
const index_info* key_ii = index_info::lookup(sc.attrs[0].type);

// for each range query, construct an iterator
for (size_t i = 0; i < ranges.size(); ++i)
Expand All @@ -719,7 +719,7 @@ datalayer :: make_search_iterator(snapshot snap,
{
assert(indices[j]->attr == ranges[i].attr);
const index* idx = indices[j];
index_info* ii = index_info::lookup(ranges[i].type);
const index_info* ii = index_info::lookup(ranges[i].type);

if (!ii)
{
Expand Down Expand Up @@ -749,7 +749,7 @@ datalayer :: make_search_iterator(snapshot snap,
for (size_t j = 0; j < indices.size(); ++j)
{
const index* idx = indices[j];
index_info* ii = index_info::lookup(sc.attrs[checks[i].attr].type);
const index_info* ii = index_info::lookup(sc.attrs[checks[i].attr].type);

if (!ii)
{
Expand Down
6 changes: 3 additions & 3 deletions daemon/datalayer_encodings.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ hyperdex :: encode_key(const region_id& ri,
std::vector<char>* scratch,
leveldb::Slice* out)
{
index_encoding* ie(index_encoding::lookup(key_type));
const index_encoding* ie(index_encoding::lookup(key_type));
size_t sz = object_prefix_sz(ri) + ie->encoded_size(key);

if (scratch->size() < sz)
Expand Down Expand Up @@ -291,7 +291,7 @@ hyperdex :: create_index_changes(const schema& sc,
assert(!old_value || !new_value || old_value->size() == new_value->size());
assert(!old_value || old_value->size() + 1 == sc.attrs_sz);
assert(!new_value || new_value->size() + 1 == sc.attrs_sz);
index_encoding* key_ie = index_encoding::lookup(sc.attrs[0].type);
const index_encoding* key_ie = index_encoding::lookup(sc.attrs[0].type);

for (size_t i = 0; i < indices.size(); ++i)
{
Expand All @@ -300,7 +300,7 @@ hyperdex :: create_index_changes(const schema& sc,
assert(idx->attr > 0);
assert(idx->attr < sc.attrs_sz);

index_info* ai = index_info::lookup(sc.attrs[idx->attr].type);
const index_info* ai = index_info::lookup(sc.attrs[idx->attr].type);
assert(ai);

const e::slice* old_attr = NULL;
Expand Down
2 changes: 1 addition & 1 deletion daemon/datalayer_indexer_thread.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ datalayer :: indexer_thread :: play(const region_id& ri, const schema* sc)
leveldb::ReadOptions ro;
ro.fill_cache = false;
iip.reset(leveldb_snapshot_ptr(db, NULL), db->NewIterator(ro));
index_encoding* ie = index_encoding::lookup(sc->attrs[0].type);
const index_encoding* ie = index_encoding::lookup(sc->attrs[0].type);
return new region_iterator(iip, ri, ie);
}

Expand Down
8 changes: 4 additions & 4 deletions daemon/datalayer_iterator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ datalayer :: iterator :: ~iterator() throw ()

datalayer :: replay_iterator :: replay_iterator(const region_id& ri,
leveldb_replay_iterator_ptr ptr,
index_encoding* ie)
const index_encoding* ie)
: m_ri(ri)
, m_iter(ptr.get())
, m_ptr(ptr)
Expand Down Expand Up @@ -218,7 +218,7 @@ datalayer :: dummy_iterator :: ~dummy_iterator() throw ()

datalayer :: region_iterator :: region_iterator(leveldb_iterator_ptr iter,
const region_id& ri,
index_encoding* ie)
const index_encoding* ie)
: iterator(iter.snap())
, m_iter(iter)
, m_ri(ri)
Expand Down Expand Up @@ -337,8 +337,8 @@ datalayer :: range_index_iterator :: range_index_iterator(leveldb_snapshot_ptr s
const e::slice& range_upper,
bool has_value_lower,
bool has_value_upper,
index_encoding* val_ie,
index_encoding* key_ie)
const index_encoding* val_ie,
const index_encoding* key_ie)
: index_iterator(s)
, m_iter()
, m_val_ie(val_ie)
Expand Down
16 changes: 8 additions & 8 deletions daemon/datalayer_iterator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class datalayer::iterator
class datalayer::replay_iterator
{
public:
replay_iterator(const region_id& ri, leveldb_replay_iterator_ptr ptr, index_encoding* ie);
replay_iterator(const region_id& ri, leveldb_replay_iterator_ptr ptr, const index_encoding* ie);

public:
bool valid();
Expand All @@ -86,7 +86,7 @@ class datalayer::replay_iterator
leveldb::ReplayIterator* m_iter;
leveldb_replay_iterator_ptr m_ptr;
std::vector<char> m_decoded;
index_encoding* m_ie;
const index_encoding *const m_ie;

private:
replay_iterator(const replay_iterator&);
Expand Down Expand Up @@ -114,7 +114,7 @@ class datalayer::region_iterator : public iterator
public:
region_iterator(leveldb_iterator_ptr iter,
const region_id& ri,
index_encoding* ie);
const index_encoding* ie);
virtual ~region_iterator() throw ();

public:
Expand All @@ -132,7 +132,7 @@ class datalayer::region_iterator : public iterator
leveldb_iterator_ptr m_iter;
region_id m_ri;
std::vector<char> m_decoded;
index_encoding* m_ie;
const index_encoding *const m_ie;
};

class datalayer::index_iterator : public iterator
Expand All @@ -159,8 +159,8 @@ class datalayer::range_index_iterator : public index_iterator
const e::slice& range_upper,
bool has_value_lower,
bool has_value_upper,
index_encoding* val_ie,
index_encoding* key_ie);
const index_encoding* val_ie,
const index_encoding* key_ie);
virtual ~range_index_iterator() throw ();

public:
Expand All @@ -187,8 +187,8 @@ class datalayer::range_index_iterator : public index_iterator

private:
leveldb_iterator_ptr m_iter;
index_encoding* m_val_ie;
index_encoding* m_key_ie;
const index_encoding *const m_val_ie;
const index_encoding *const m_key_ie;
e::slice m_range_prefix;
e::slice m_range_lower;
e::slice m_range_upper;
Expand Down
Loading

0 comments on commit e77f139

Please sign in to comment.