From c7365ff0a56d13a5cc7429f0c45e9553fe10bcc7 Mon Sep 17 00:00:00 2001 From: Gary Helmling Date: Sat, 13 Feb 2010 16:48:13 -0500 Subject: [PATCH] Added alternate index key generation --- src/java/meetup/beeno/EntityIndexer.java | 92 ++++++++++++++----- src/java/meetup/beeno/HIndex.java | 3 +- src/java/meetup/beeno/IndexKeyFactory.java | 6 ++ .../meetup/beeno/mapping/IndexMapping.java | 4 + 4 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 src/java/meetup/beeno/IndexKeyFactory.java diff --git a/src/java/meetup/beeno/EntityIndexer.java b/src/java/meetup/beeno/EntityIndexer.java index a447a03..d3c9eac 100644 --- a/src/java/meetup/beeno/EntityIndexer.java +++ b/src/java/meetup/beeno/EntityIndexer.java @@ -42,6 +42,7 @@ public class EntityIndexer { private HUtil.HCol dateField; private boolean invertDate = false; private List extraFields; + private IndexKeyFactory keyFactory = new DefaultKeyFactory(); public EntityIndexer(IndexMapping mapping) { this.indexTable = mapping.getTableName(); @@ -50,6 +51,15 @@ public EntityIndexer(IndexMapping mapping) { this.dateField = mapping.getDateField(); this.invertDate = mapping.isDateInverted(); this.extraFields = mapping.getExtraFields(); + + if (mapping.getKeyFactory() != null) { + try { + this.keyFactory = mapping.getKeyFactory().newInstance(); + } + catch (Exception e) { + throw new IllegalArgumentException("Unable to instantiate key factory class", e); + } + } } public String getIndexTable() { return this.indexTable; } @@ -147,30 +157,70 @@ protected byte[] getValue(byte[] family, byte[] col, * implementations */ public byte[] createIndexKey(byte[] primaryVal, Long date, byte[] origRow) { - byte[] key = new byte[0]; - HDataTypes.HField pbVal = PBUtil.readMessage(primaryVal); - // order numeric types - if (pbVal != null && pbVal.getType() == HDataTypes.HField.Type.INTEGER) { - key = Bytes.add(key, HUtil.toOrderedBytes(pbVal.getInteger())); + if (this.dateField != null && date != null) { + return this.keyFactory.createKey(primaryVal, origRow, date, this.invertDate); } else { - // just use raw bytes - key = Bytes.add(key, primaryVal); - } - - // add on date, if specified - if (this.dateField != null && date != null) { - key = Bytes.add(key, - Bytes.add(ROW_KEY_SEP, - HUtil.toOrderedBytes(date, this.invertDate)) ); + return this.keyFactory.createKey(primaryVal, origRow, null, this.invertDate); } + } + + + public static class DefaultKeyFactory implements IndexKeyFactory { + + @Override + public byte[] createKey( byte[] primaryVal, byte[] rowKey, Long date, boolean invertDate ) { + byte[] key = new byte[0]; + HDataTypes.HField pbVal = PBUtil.readMessage(primaryVal); + // order numeric types + if (pbVal != null && pbVal.getType() == HDataTypes.HField.Type.INTEGER) { + key = Bytes.add(key, HUtil.toOrderedBytes(pbVal.getInteger())); + } + else { + // just use raw bytes + key = Bytes.add(key, primaryVal); + } + + // add on date, if specified + if (date != null) { + key = Bytes.add(key, + Bytes.add(ROW_KEY_SEP, HUtil.toOrderedBytes(date, invertDate)) ); + } + + // add on the original row key to ensure uniqueness + if (rowKey != null && rowKey.length > 0) { + key = Bytes.add(key, + Bytes.add(ROW_KEY_SEP, rowKey)); + } + + return key; + } + } + + + /** + * Generates the same index keys as DefaultKeyFactory, but prefixed with the primary value mod 100 for + * better row key distribution. + * + * This is designed specifically to avoid hot regions arising from frequently used indexes based off of + * a sequentially incremented primary value. + * @author garyh + * + */ + public static class ModKeyFactory extends DefaultKeyFactory { + private static int base = 100; - // add on the original row key to ensure uniqueness - if (origRow != null && origRow.length > 0) { - key = Bytes.add(key, - Bytes.add(ROW_KEY_SEP, origRow)); + public byte[] createKey( byte[] primaryVal, byte[] rowKey, Long date, boolean invertDate) { + byte[] key = new byte[0]; + HDataTypes.HField pbVal = PBUtil.readMessage(primaryVal); + // order numeric types + if (pbVal != null && pbVal.getType() == HDataTypes.HField.Type.INTEGER) { + long val = pbVal.getInteger(); + key = Bytes.add(Bytes.toBytes(Long.toString( val % base )), ROW_KEY_SEP); + } + key = Bytes.add(key, super.createKey(primaryVal, rowKey, date, invertDate)); + + return key; } - - return key; - } + } } diff --git a/src/java/meetup/beeno/HIndex.java b/src/java/meetup/beeno/HIndex.java index 5d7e49a..d9170fa 100644 --- a/src/java/meetup/beeno/HIndex.java +++ b/src/java/meetup/beeno/HIndex.java @@ -21,5 +21,6 @@ public @interface HIndex { String date_col() default ""; boolean date_invert() default false; - String[] extra_cols(); + String[] extra_cols() default {}; + Class key_factory() default EntityIndexer.DefaultKeyFactory.class; } diff --git a/src/java/meetup/beeno/IndexKeyFactory.java b/src/java/meetup/beeno/IndexKeyFactory.java new file mode 100644 index 0000000..96b2c86 --- /dev/null +++ b/src/java/meetup/beeno/IndexKeyFactory.java @@ -0,0 +1,6 @@ +package meetup.beeno; + +public interface IndexKeyFactory { + + public byte[] createKey(byte[] primaryVal, byte[] rowKey, Long date, boolean invertDate); +} diff --git a/src/java/meetup/beeno/mapping/IndexMapping.java b/src/java/meetup/beeno/mapping/IndexMapping.java index f7e16d2..e223e43 100644 --- a/src/java/meetup/beeno/mapping/IndexMapping.java +++ b/src/java/meetup/beeno/mapping/IndexMapping.java @@ -5,6 +5,7 @@ import meetup.beeno.EntityIndexer; import meetup.beeno.HIndex; +import meetup.beeno.IndexKeyFactory; import meetup.beeno.util.HUtil; import meetup.beeno.util.HUtil.HCol; @@ -20,6 +21,7 @@ public class IndexMapping { protected boolean invertDate = false; protected List extraFields = new ArrayList(); protected EntityIndexer generator; + protected Class keyFactory; public IndexMapping(String baseTable, FieldMapping baseField, HIndex indexAnnotation) { this.indexTable = String.format("%s-by_%s", baseTable, baseField.getColumn()); @@ -33,6 +35,7 @@ public IndexMapping(String baseTable, FieldMapping baseField, HIndex indexAnnota if (indexAnnotation.date_col() != null && indexAnnotation.date_col().length() > 0) this.dateCol = HUtil.HCol.parse(indexAnnotation.date_col()); this.invertDate = indexAnnotation.date_invert(); + this.keyFactory = indexAnnotation.key_factory(); this.generator = new EntityIndexer(this); } @@ -43,4 +46,5 @@ public IndexMapping(String baseTable, FieldMapping baseField, HIndex indexAnnota public boolean isDateInverted() { return this.invertDate; } public List getExtraFields() { return this.extraFields; } public EntityIndexer getGenerator() { return this.generator; } + public Class getKeyFactory() { return this.keyFactory; } } \ No newline at end of file