Get tests working with a mini-cluster

bin/env.sh

@@ -13,10 +13,10 @@ cd -P -- "$prgbase" > /dev/null && pwd -P
 
 basedir=$( cd -P -- "$prgdir/.." > /dev/null && pwd -P )
 
-for f in `ls $basedir/lib/*.jar`;
+CP=$basedir/conf
+for f in `find $basedir/lib/ -name '*.jar'`;
 do
 	CP=$CP:$f;
 done
-CP=$CP:$basedir/conf
 
 export CP

bin/jyunit

@@ -14,7 +14,8 @@ cd -P -- "$prgbase" > /dev/null && pwd -P
 basedir=$( cd -P -- "$prgdir/.." > /dev/null && pwd -P )
 
 . $prgdir/env.sh
-JYTHON_CP=$CP:$basedir/build
+
+JYTHON_CP=$basedir/build:$basedir/test/conf:$CP
 export JYTHON_CP
 
 #export LOG4J_OPTS="-Dlog4j.configuration=log4j_test.properties" # to change default log level add -Dlog4j.info

bin/run_tests.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
 
 BASEDIR=`dirname $0`
+
 $BASEDIR/jyunit $BASEDIR/../src/jython/jyunit/run.py $@

build.xml

@@ -5,6 +5,7 @@
   <property name="project.title" value="Meetup Beeno" />
   <property name="project.version" value="0.1" />
   <property name="build.dir" value="${basedir}/build" />
+  <property name="build.test" value="${basedir}/build/test" />
   <property name="dist.dir" value="${basedir}/dist" />
   <property name="lib.dir" value="${basedir}/lib" />
   <property name="src.dir" value="${basedir}/src/java" />
@@ -57,7 +58,13 @@
   <!-- ====================== Run Tests ============================== -->
   <target name="test" depends="-jyunit_prepare"
 		  description="Run all unit tests">
-    <exec executable="${basedir}/bin/run_tests.sh" failonerror="yes" inputstring="${tests.files}">
+	<mkdir dir="${build.test}/data" />
+    <exec executable="${basedir}/bin/run_tests.sh" 
+		  failonerror="yes" 
+		  inputstring="${tests.files}">
+	  <!-- system properties need for hbase test case -->
+      <env key="JAVA_OPTS" value="-Dtest.build.data=${build.test}/data -Xmx2048M -XX:MaxPermSize=128M"/>
+
       <arg value="--javasrc=${test.src.dir}" />
       <arg value="--directory=${test.jysrc.dir}" />
       <arg value="--product=Meetup.Beeno" />

conf/log4j.properties

@@ -4,3 +4,4 @@ log4j.appender.R.File=beeno.log
 log4j.appender.R.DatePattern='.'yyyy-MM-dd
 log4j.appender.R.layout=org.apache.log4j.PatternLayout 
 log4j.appender.R.layout.ConversionPattern=%d [%t] %-5p %c %x - %m%n
+log4j.logger.meetup.beeno=DEBUG, R

src/java/meetup/beeno/util/HUtil.java

@@ -6,8 +6,6 @@
 import java.util.Set;
 
 import meetup.beeno.EntityMetadata;
-import meetup.beeno.EntityMetadata.EntityInfo;
-import meetup.beeno.EntityMetadata.FieldMapping;
 
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.KeyValue;
@@ -34,6 +32,14 @@ public class HUtil {
 
 	private static HTablePool pool = new HTablePool(new HBaseConfiguration(), MAX_POOL_SIZE);
 
+	public static HTablePool getPool() {
+		return pool;
+	}
+
+	public static void setPool(HTablePool newPool) {
+		pool = newPool;
+	}
+
 	public static HTable getTable(String tablename) {
 		if (log.isDebugEnabled())
 			log.debug("Getting table "+tablename+" from pool");

src/jython/db/hbase.py

@@ -12,7 +12,7 @@
 
 import com.google.protobuf
 
-from meetup.beeno.filter import ColumnRowFilter
+from meetup.beeno.filter import ColumnMatchFilter
 from meetup.beeno.util import PBUtil
 
 # constants for dicts
@@ -366,4 +366,4 @@ def eq(colname, colvalue):
 		valbytes = colvalue.toByteArray()
 	else:
 		valbytes = PBUtil.toBytes( colvalue )
-	return ColumnRowFilter( namebytes, ColumnRowFilter.CompareOp.EQUAL, valbytes )
+	return ColumnMatchFilter( namebytes, ColumnMatchFilter.CompareOp.EQUAL, valbytes )

test/conf/hbase-site.xml

@@ -0,0 +1,137 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2007 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>hbase.regionserver.msginterval</name>
+    <value>1000</value>
+    <description>Interval between messages from the RegionServer to HMaster
+    in milliseconds.  Default is 15. Set this value low if you want unit
+    tests to be responsive.
+    </description>
+  </property>
+  <property>
+    <name>hbase.client.pause</name>
+    <value>5000</value>
+    <description>General client pause value.  Used mostly as value to wait
+    before running a retry of a failed get, region lookup, etc.</description>
+  </property>
+  <property>
+    <name>hbase.master.meta.thread.rescanfrequency</name>
+    <value>10000</value>
+    <description>How long the HMaster sleeps (in milliseconds) between scans of
+    the root and meta tables.
+    </description>
+  </property>
+  <property>
+    <name>hbase.server.thread.wakefrequency</name>
+    <value>1000</value>
+    <description>Time to sleep in between searches for work (in milliseconds).
+    Used as sleep interval by service threads such as META scanner and log roller.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>5</value>
+    <description>Count of RPC Server instances spun up on RegionServers
+    Same property is used by the HMaster for count of master handlers.
+    Default is 10.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.lease.period</name>
+    <value>6000</value>
+    <description>Length of time the master will wait before timing out a region
+    server lease. Since region servers report in every second (see above), this
+    value has been reduced so that the master will notice a dead region server
+    sooner. The default is 30 seconds.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.info.port</name>
+    <value>-1</value>
+    <description>The port for the hbase master web UI
+    Set to -1 if you do not want the info server to run.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.info.port</name>
+    <value>-1</value>
+    <description>The port for the hbase regionserver web UI
+    Set to -1 if you do not want the info server to run.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.info.port.auto</name>
+    <value>true</value>
+    <description>Info server auto port bind. Enables automatic port
+    search if hbase.regionserver.info.port is already in use.
+    Enabled for testing to run multiple tests on one machine.
+    </description>
+  </property>
+  <property>
+    <name>hbase.master.lease.thread.wakefrequency</name>
+    <value>3000</value>
+    <description>The interval between checks for expired region server leases.
+    This value has been reduced due to the other reduced values above so that
+    the master will notice a dead region server sooner. The default is 15 seconds.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.optionalcacheflushinterval</name>
+    <value>10000</value>
+    <description>
+    Amount of time to wait since the last time a region was flushed before
+    invoking an optional cache flush. Default 60,000.
+    </description>
+  </property>
+  <property>
+    <name>hbase.regionserver.safemode</name>
+    <value>false</value>
+    <description>
+    Turn on/off safe mode in region server. Always on for production, always off
+    for tests.
+    </description>
+  </property>
+  <property>
+    <name>hbase.hregion.max.filesize</name>
+    <value>67108864</value>
+    <description>
+    Maximum desired file size for an HRegion.  If filesize exceeds
+    value + (value / 2), the HRegion is split in two.  Default: 256M.
+
+    Keep the maximum filesize small so we split more often in tests.
+    </description>
+  </property>
+  <property>
+    <name>hadoop.log.dir</name>
+    <value>${user.dir}/../logs</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.property.clientPort</name>
+    <value>21810</value>
+    <description>Property from ZooKeeper's config zoo.cfg.
+    The port at which the clients will connect.
+    </description>
+  </property>
+</configuration>

test/jython/dbtest/hbase/test_entity_io.py

@@ -7,32 +7,38 @@
 import java.lang
 from java.util import ArrayList, HashSet, HashMap
 from org.apache.hadoop.hbase import HBaseClusterTestCase
+from org.apache.hadoop.hbase.client import HTablePool
 from meetup.beeno import EntityMetadata, EntityService, HBaseException, MappingException
 from meetup.beeno import TestEntities
+from meetup.beeno.util import HUtil
 
 class HBaseContext(HBaseClusterTestCase):
     def __init__(self):
         super(HBaseContext, self).__init__()
+        self.setName('beeno')
+
+        if self.conf.get('test.build.data') is None:
+            self.conf.set('test.build.data', '/tmp/beeno')
 
 hc = HBaseContext()
 
 def setup():
     hc.setUp()
-
-	# create a dummy HBase table for testing
-	import db.hbase
-	admin = db.hbase.Admin()
+    HUtil.setPool( HTablePool( hc.conf, 5 ) )
+    # create a dummy HBase table for testing
+    import db.hbase
+    admin = db.hbase.Admin(hc.conf)
 
-	if not admin.exists("test_simple"):
-		admin.create("test_simple", {"props:": {}})
-	if not admin.exists("test_complex"):
-		admin.create("test_complex", {"props:": {db.hbase.VERSIONS: 10}, "extended:": {db.hbase.VERSIONS: 10}})
+    if not admin.exists("test_simple"):
+        admin.create("test_simple", {"props:": {}})
+    if not admin.exists("test_complex"):
+        admin.create("test_complex", {"props:": {db.hbase.VERSIONS: 10}, "extended:": {db.hbase.VERSIONS: 10}})
 
 def teardown():
     try:
         # clean up the dummy table
         import db.hbase
-        admin = db.hbase.Admin()
+        admin = db.hbase.Admin(hc.conf)
 
         #if admin.exists("test_simple"):
         #	admin.disable("test_simple")