Add some basic tests

test/jython/dbtest/hbase/test_entity_io.py

@@ -0,0 +1,156 @@
+#
+# Tests for reading and writing entities to/from HBase
+#
+
+from jyunit.util import *
+
+import java.lang
+from java.util import ArrayList, HashSet, HashMap
+from meetup.beeno import EntityMetadata, EntityService, HBaseException, MappingException
+from com.meetup.db.hbase import TestEntities
+
+
+def setup():
+	# create a dummy HBase table for testing
+	import db.hbase
+	admin = db.hbase.Admin()
+
+	if not admin.exists("test_simple"):
+		admin.create("test_simple", {"props:": {}})
+	if not admin.exists("test_complex"):
+		admin.create("test_complex", {"props:": {db.hbase.VERSIONS: 10}, "extended:": {db.hbase.VERSIONS: 10}})
+
+def teardown():
+	# clean up the dummy table
+	import db.hbase
+	admin = db.hbase.Admin()
+
+	#if admin.exists("test_simple"):
+	#	admin.disable("test_simple")
+	#	admin.drop("test_simple")
+
+	#if admin.exists("test_complex"):
+	#	admin.disable("test_complex")
+	#	admin.drop("test_complex")
+
+
+def save_and_get():
+	entity1 = TestEntities.SimpleEntity()
+	entity1.setId("entity1")
+	entity1.setStringProperty("all my words")
+	entity1.setIntProperty(123)
+	entity1.setFloatProperty(1.1)
+	entity1.setDoubleProperty(123.456789)
+	entity1.setLongProperty(444444444444)
+
+	service = EntityService(TestEntities.SimpleEntity)
+	service.save(entity1)
+
+	entity2 = service.get("entity1")
+	assertNotNull(entity2)
+	assertEquals(entity2.getId(), entity1.getId())
+	assertEquals(entity2.getStringProperty(), entity1.getStringProperty())
+	assertEquals(entity2.getIntProperty(), entity1.getIntProperty())
+	assertEquals(entity2.getFloatProperty(), entity1.getFloatProperty(), None, 0.0001)
+	assertEquals(entity2.getDoubleProperty(), entity1.getDoubleProperty(), None, 0.0001)
+	assertEquals(entity2.getLongProperty(), entity1.getLongProperty())
+
+	# update the retrieved entity and check for the changes
+	entity2.setStringProperty("new stuff")
+	entity2.setDoubleProperty(9.8765432101)
+	entity2.setIntProperty(-17)
+	service.save(entity2)
+
+	entity3 = service.get(entity2.getId())
+	assertEquals(entity3.getId(), "entity1")
+	assertEquals(entity3.getStringProperty(), "new stuff")
+	assertEquals(entity3.getIntProperty(), -17)
+	assertEquals(entity3.getFloatProperty(), 1.1, None, 0.0001)
+	assertEquals(entity3.getDoubleProperty(), 9.8765432101, None, 0.0001)
+	assertEquals(entity3.getLongProperty(), 444444444444)
+
+
+def save_and_get_complex():
+	'''Test saving entities containing mapped collection properties'''
+	entity1 = TestEntities.ComplexEntity()
+	entity1.setId("complex1")
+	strings = ArrayList()
+	strings.add("one")
+	strings.add("two")
+	entity1.setStringList(strings)
+	ints = HashSet()
+	ints.add(1)
+	ints.add(2)
+	entity1.setIntSet(ints)
+	extended = HashMap()
+	extended.put("prop1", "one")
+	extended.put("prop2", "two")
+	entity1.setExtendedProps(extended)
+
+	service = EntityService(TestEntities.ComplexEntity)
+	service.save(entity1)
+
+	entity2 = service.get("complex1")
+	assertNotNull(entity2)
+	assertEquals(entity2.getId(), entity1.getId())
+	assertTrue(entity2.getStringList().contains("one"))
+	assertTrue(entity2.getStringList().contains("two"))
+	assertTrue(entity2.getIntSet().contains(java.lang.Long(1)))
+	assertTrue(entity2.getIntSet().contains(java.lang.Long(2)))
+	assertNotNull(entity2.getExtendedProps())
+	assertEquals(entity2.getExtendedProps().get("prop1"), "one")
+	assertEquals(entity2.getExtendedProps().get("prop2"), "two")
+
+
+def save_multiple():
+	'''Test saving multiple entities as a batch'''
+	entities = [ TestEntities.SimpleEntity("e1", "string1", 1, 1.1, 1.1, 1),
+				 TestEntities.SimpleEntity("e2", "string2", 2, 2.2, 2.2, 2),
+				 TestEntities.SimpleEntity("e3", "string3", 3, 3.3, 3.3, 3),
+				 TestEntities.SimpleEntity("e4", "string4", 4, 4.4, 4.4, 4),
+				 TestEntities.SimpleEntity("e5", "string5", 5, 5.5, 5.5, 5),
+				 TestEntities.SimpleEntity("e6", "string6", 6, 6.6, 6.6, 6) ]
+
+	srv = EntityService(TestEntities.SimpleEntity)
+	srv.saveAll(entities)
+
+	saved = []
+	for e in entities:
+		saved.append( srv.get(e.getId()) )
+
+	assertEquals(len(entities), len(saved))
+	for cnt in range(0, len(entities)):
+		assertEquals(entities[cnt].getId(), saved[cnt].getId())
+		assertEquals(entities[cnt].getStringProperty(), saved[cnt].getStringProperty())
+		assertEquals(entities[cnt].getIntProperty(), saved[cnt].getIntProperty())
+		assertEquals(entities[cnt].getFloatProperty(), saved[cnt].getFloatProperty(), None, 0.0001)
+		assertEquals(entities[cnt].getDoubleProperty(), saved[cnt].getDoubleProperty(), None, 0.0001)
+		assertEquals(entities[cnt].getLongProperty(), saved[cnt].getLongProperty())
+
+
+def save_timeout():
+	'''Test timeouts when saving recs'''
+	newent = TestEntities.SimpleEntity("ne1", "string one", 1, 1.1, 1.1, 1);
+	service = EntityService(TestEntities.SimpleEntity)
+	try:
+		# save with too short timeout
+		success = service.save(newent, 1)
+		assertFalse(success, "Expected save request to timeout")
+	except HBaseException, e:
+		fail("Save failed with unexpected exception")
+
+
+def run_test():
+	save_and_get()
+	save_multiple()
+	save_and_get_complex()
+	#save_timeout()
+
+
+if __name__ == "__main__":
+	try:
+		setup()
+		run_test()
+	finally:
+		teardown()
+

test/jython/dbtest/hbase/test_entity_metadata.py

@@ -0,0 +1,102 @@
+#
+# Tests for parsing hbase entity mapping information
+# from annotated java classes.
+
+from jyunit.util import *
+
+from com.meetup.base.db.hbase import EntityMetadata, HBaseException, MappingException
+from com.meetup.db.hbase import TestEntities
+from com.meetup.feeds.db import FeedItem
+
+import java.lang
+
+class EntityTest(SimpleTest):
+	def __init__(self, cls, tablename, fieldprops):
+		self.name = "EntityTest[%s]" % str(cls.getName())
+		self.entitycls = cls
+		self.tablename = tablename
+		self.fieldprops = fieldprops
+
+	def run(self):
+		metadata = EntityMetadata.getInstance()
+		iteminfo = metadata.getInfo(self.entitycls)
+
+		assertEquals(iteminfo.getTablename(), self.tablename)
+
+		keyprop = iteminfo.getKeyProperty()
+		assertNotNull(keyprop)
+		assertEquals(keyprop.getName(), "id")
+
+
+		for field, propname in self.fieldprops.items():
+			prop = iteminfo.getFieldProperty(field)
+			assertNotNull(prop)
+			assertEquals(prop.getName(), propname)
+
+
+def test_parsing():
+	# check feed table
+	expectedFields = {"info:actor_member": "memberId",
+					  "info:actor_chapter": "chapterId",
+					  "info:target_member": "targetMemberId",
+					  "info:item_type": "itemType",
+					  "info:pub_date": "publishTimestamp"}
+	feedtest = EntityTest(FeedItem, "FeedItem", expectedFields)
+	runTest(feedtest)
+
+	######### check the entity mapping test cases
+	# simple success case
+	expectedFields = {"props:stringcol": "stringProperty",
+					  "props:intcol": "intProperty",
+					  "props:floatcol": "floatProperty",
+					  "props:doublecol": "doubleProperty",
+					  "props:longcol": "longProperty"}
+	simpletest = EntityTest(TestEntities.SimpleEntity, "test_simple", expectedFields)
+	runTest(simpletest)
+
+	############ check basic field mapping when indexes are present ########
+	expectedFields = {"props:stringcol": "stringProperty",
+					  "props:tscol": "timestamp"}
+	idxtest = EntityTest(TestEntities.IndexedEntity, "test_indexed", expectedFields)
+	runTest(idxtest)
+
+def test_rowkey():	
+	# entity with no row key mapping
+	metadata = EntityMetadata.getInstance()
+	try:
+		nokeyinfo = metadata.getInfo(TestEntities.NoKeyEntity)
+		fail("EntityMetadata should have failed parsing NoKeyEntity due to missing HRowKey")
+	except MappingException, me:
+		assertMatches(me.getMessage(), "Missing .* row key property")
+
+	# duplicate row keys mapped -- only one is allowed
+	try:
+		dupekeyinfo = metadata.getInfo(TestEntities.DupeKeyEntity)
+		fail("EntityMetadata should have failed parsing DupeKeyEntity due to duplicate HRowKey");
+	except MappingException, me:
+		assertMatches(me.getMessage(), "Duplicate mappings .* row key")
+
+	# duplicate mappings for same field -- only one is allowed
+	try:
+		dupefieldinfo = metadata.getInfo(TestEntities.DupeFieldEntity)
+		fail("EntityMetadata should have failed parsing DupeFieldEntity due to duplicate field mappings")
+	except MappingException, me:
+		assertMatches(me.getMessage(), "Duplicate mappings .* field")
+
+
+def test_indexes():
+	metadata = EntityMetadata.getInstance()
+	idxinfo = metadata.getInfo(TestEntities.IndexedEntity)
+	idxmapping = idxinfo.getFirstPropertyIndex("stringProperty")
+	assertNotNull(idxmapping)
+	assertEquals(idxmapping.getTableName(), "test_indexed-by_stringcol")
+
+
+def run_test():
+	test_parsing()
+	test_rowkey()
+	test_indexes()
+
+
+if __name__ == "__main__":
+	run_test()

test/jython/dbtest/hbase/test_filter.py

@@ -0,0 +1,94 @@
+#
+# Test comparison operations for custom HBase filters
+#
+from com.meetup.base.db.hbase import *
+
+from org.apache.hadoop.hbase import KeyValue
+from org.apache.hadoop.hbase.util import Bytes
+
+import java.lang
+from java.util import Locale
+
+from jyunit.util import *
+
+
+def test_column_eq():
+    '''Checks column equals operation in ColumnMatchFilter'''
+    fam1 = java.lang.String('fam1')
+    col1 = java.lang.String('col1')
+    col2 = java.lang.String('col2')
+    fam1col1 = java.lang.String.format("%s:%s", [fam1, col1])
+    colval = java.lang.String('myvalue')
+    mismatchval = java.lang.String('secondvalue')
+    rowkey1 = Bytes.toBytes( java.lang.String('row1') )
+
+    colfilt = ColumnMatchFilter(Bytes.toBytes(fam1col1), ColumnMatchFilter.CompareOp.EQUAL, PBUtil.toBytes(colval), True)
+
+    row1 = KeyValue(rowkey1, Bytes.toBytes(fam1), Bytes.toBytes(col1), PBUtil.toBytes(colval))
+    row_ne = KeyValue(rowkey1, Bytes.toBytes(fam1), Bytes.toBytes(col1), PBUtil.toBytes(mismatchval))
+    row_missing = KeyValue(rowkey1, Bytes.toBytes(fam1), Bytes.toBytes(col2), PBUtil.toBytes(colval))
+
+    colfilt.filterKeyValue(row1)
+    assertFalse( colfilt.filterRow(), "Row with matching value should not be filtered" )
+    colfilt.reset()
+
+    colfilt.filterKeyValue(row_ne)
+    assertTrue( colfilt.filterRow(), "Row with mismatched value should be filtered" )
+    colfilt.reset()
+
+    colfilt.filterKeyValue(row_missing)
+    assertTrue( colfilt.filterRow(), "Row missing column should be filtered" )
+    colfilt.reset()
+
+    # test row missing column without 'filter if missing' flag
+    colfilt = ColumnMatchFilter(Bytes.toBytes(fam1col1), ColumnMatchFilter.CompareOp.EQUAL, PBUtil.toBytes(colval), False)
+
+    colfilt.filterKeyValue(row_missing)
+    assertFalse( colfilt.filterRow(), "Row missing column should not be filtered without flag" )
+    colfilt.reset()
+
+
+
+def test_column_ne():
+    '''Checks column not equal operation in ColumnMatchFilter'''
+    fam1 = java.lang.String('fam1')
+    col1 = java.lang.String('col1')
+    col2 = java.lang.String('col2')
+    fam1col1 = java.lang.String.format("%s:%s", [fam1, col1])
+    colval = java.lang.String('myvalue')
+    mismatchval = java.lang.String('secondvalue')
+    rowkey1 = Bytes.toBytes( java.lang.String('row1') )
+
+    colfilt = ColumnMatchFilter(Bytes.toBytes(fam1col1), ColumnMatchFilter.CompareOp.NOT_EQUAL, PBUtil.toBytes(colval), True)
+
+    row1 = KeyValue(rowkey1, Bytes.toBytes(fam1), Bytes.toBytes(col1), PBUtil.toBytes(colval))
+    row_ne = KeyValue(rowkey1, Bytes.toBytes(fam1), Bytes.toBytes(col1), PBUtil.toBytes(mismatchval))
+    row_missing = KeyValue(rowkey1, Bytes.toBytes(fam1), Bytes.toBytes(col2), PBUtil.toBytes(colval))
+
+    colfilt.filterKeyValue(row1)
+    assertTrue( colfilt.filterRow(), "Row with matching value should be filtered" )
+    colfilt.reset()
+
+    colfilt.filterKeyValue(row_ne)
+    assertFalse( colfilt.filterRow(), "Row with mismatched value should not be filtered" )
+    colfilt.reset()
+
+    colfilt.filterKeyValue(row_missing)
+    assertTrue( colfilt.filterRow(), "Row missing column should be filtered" )
+    colfilt.reset()
+
+    # test row missing column without 'filter if missing' flag
+    colfilt = ColumnMatchFilter(Bytes.toBytes(fam1col1), ColumnMatchFilter.CompareOp.NOT_EQUAL, PBUtil.toBytes(colval), False)
+
+    colfilt.filterKeyValue(row_missing)
+    assertFalse( colfilt.filterRow(), "Row missing column should not be filtered without flag" )
+    colfilt.reset()
+
+
+
+def run_test():
+    test_column_eq()
+    test_column_ne()
+
+if __name__ == "__main__":
+	run_test()