diff --git a/src/main/java/emissary/core/BaseDataObject.java b/src/main/java/emissary/core/BaseDataObject.java index f78d603c43..83bee0843a 100755 --- a/src/main/java/emissary/core/BaseDataObject.java +++ b/src/main/java/emissary/core/BaseDataObject.java @@ -1,1348 +1,44 @@ package emissary.core; -import emissary.core.channels.SeekableByteChannelFactory; -import emissary.core.channels.SeekableByteChannelHelper; -import emissary.directory.DirectoryEntry; -import emissary.pickup.Priority; -import emissary.util.ByteUtil; -import emissary.util.PayloadUtil; - -import com.google.common.collect.LinkedListMultimap; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Serializable; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.SeekableByteChannel; -import java.rmi.Remote; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.UUID; -import javax.annotation.Nullable; - -/** - * Class to hold data, header, footer, and attributes - */ -public class BaseDataObject implements Serializable, Cloneable, Remote, IBaseDataObject { - protected static final Logger logger = LoggerFactory.getLogger(BaseDataObject.class); - - /* Used to limit the size of a returned byte array to avoid certain edge case scenarios */ - public static final int MAX_BYTE_ARRAY_SIZE = Integer.MAX_VALUE - 8; - - /* Including this here make serialization of this object faster. */ - private static final long serialVersionUID = 7362181964652092657L; - - /* Actual data - migrate away from this towards byte channels. */ - @Nullable - protected byte[] theData; - - /** - * Original name of the input data. Can only be set in the constructor of the DataObject. returned via the - * {@link #getFilename()} method. Also used in constructing the {@link #shortName()} of the document. - */ - protected String theFileName; - - /** - * Terminal portion of theFileName - */ - protected String shortName; - - /** - * The internal identifier, generated for each constructed object - */ - protected UUID internalId = UUID.randomUUID(); - - /** - * The currentForm is a stack of the itinerary items. The contents of the list are {@link String} and map to the - * dataType portion of the keys in the emissary.DirectoryPlace. - */ - protected List currentForm = new ArrayList<>(); - - /** - * History of processing errors. Lines of text are accumulated from String and returned in-toto as a String. - */ - protected StringBuilder procError; - - /** - * A travelogue built up as the agent moves about. Appended to by the agent as it goes from place to place. - */ - protected TransformHistory history = new TransformHistory(); - - /** - * The last determined language(characterset) of the data. - */ - @Nullable - protected String fontEncoding = null; - - /** - * Dynamic facets or metadata attributes of the data - */ - protected LinkedListMultimap parameters = LinkedListMultimap.create(100); - - /** - * If this file caused other agents to be sprouted, indicate how many - */ - protected int numChildren = 0; - - /** - * If this file has siblings that were sprouted at the same time, this will indicate how many total siblings there are. - * This can be used to navigate among siblings without needing to refer to the parent. - */ - protected int numSiblings = 0; - - /** - * What child is this in the family order - */ - protected int birthOrder = 0; - - /** - * Hash of alternate views of the data {@link String} current form is the key, byte[] is the value - */ - protected Map multipartAlternative = new TreeMap<>(); - - /** - * Any header that goes along with the data - */ - @Nullable - protected byte[] header = null; - - /** - * Any footer that goes along with the data - */ - @Nullable - protected byte[] footer = null; - - /** - * If the header has some encoding scheme record it - */ - @Nullable - protected String headerEncoding = null; - - /** - * Record the classification scheme for the document - */ - @Nullable - protected String classification = null; - - /** - * Keep track of if and how the document is broken so we can report on it later - */ - @Nullable - protected StringBuilder brokenDocument = null; - - // Filetypes that we think are equivalent to no file type at all - protected String[] emptyFileTypes = {Form.UNKNOWN}; - - /** - * The integer priority of the data object. A lower number is higher priority. - */ - protected int priority = Priority.DEFAULT; - - /** - * The timestamp for when the BaseDataObject was created. Used in data provenance tracking. - */ - protected Instant creationTimestamp; - - /** - * The extracted records, if any - */ - @Nullable - protected List extractedRecords; - - /** - * Check to see if this tree is able to be written out. - */ - protected boolean outputable = true; - - /** - * The unique identifier of this object - */ - protected String id; - - /** - * The identifier of the {@link emissary.pickup.WorkBundle} - */ - protected String workBundleId; - - /** - * The identifier used to track the object through the system - */ - protected String transactionId; - - /** - * A factory to create channels for the referenced data. - */ - @Nullable - protected SeekableByteChannelFactory seekableByteChannelFactory; - - final SafeUsageChecker safeUsageChecker = new SafeUsageChecker(); - - protected enum DataState { - NO_DATA, CHANNEL_ONLY, BYTE_ARRAY_ONLY, BYTE_ARRAY_AND_CHANNEL - } - - protected static final String INVALID_STATE_MSG = "Can't have both theData and seekableByteChannelFactory set. Object is %s"; - - /** - *

- * Determine what state we're in with respect to the byte[] of data vs a channel. - *

- * - *

- * Not exposed publicly as consumers should be moving to channels, meaning ultimately the states will be simply either a - * channel factory exists or does not exist. - *

- * - *

- * Consumers should not modify their behaviour based on the state of the BDO, if they're being modified to handle - * channels, they should only handle channels, not both channels and byte[]. - *

- * - * @return the {@link DataState} of this BDO - */ - protected DataState getDataState() { - if (theData == null) { - if (seekableByteChannelFactory == null) { - return DataState.NO_DATA; - } else { - return DataState.CHANNEL_ONLY; - } - } else { - if (seekableByteChannelFactory == null) { - return DataState.BYTE_ARRAY_ONLY; - } else { - return DataState.BYTE_ARRAY_AND_CHANNEL; - } - } - } - - /** - * {@inheritDoc} - */ - @Override - public void checkForUnsafeDataChanges() { - safeUsageChecker.checkForUnsafeDataChanges(); - - if (theData != null) { - safeUsageChecker.recordSnapshot(theData); - } - } - - /** - * Create an empty BaseDataObject. - */ - public BaseDataObject() { - this.theData = null; - setCreationTimestamp(Instant.now()); - } - - /** - * Create a new BaseDataObject with byte array and name passed in. WARNING: this implementation uses the passed in array - * directly, no copy is made so the caller should not reuse the array. - * - * @param newData the bytes to hold - * @param name the name of the data item - */ - public BaseDataObject(final byte[] newData, final String name) { - setData(newData); - setFilename(name); - setCreationTimestamp(Instant.now()); - } - - /** - * Create a new BaseDataObject with byte array, name, and initial form WARNING: this implementation uses the passed in - * array directly, no copy is made so the caller should not reuse the array. - * - * @param newData the bytes to hold - * @param name the name of the data item - * @param form the initial form of the data - */ - public BaseDataObject(final byte[] newData, final String name, @Nullable final String form) { - this(newData, name); - if (form != null) { - pushCurrentForm(form); - } - } - - public BaseDataObject(final byte[] newData, final String name, final String form, @Nullable final String fileType) { - this(newData, name, form); - if (fileType != null) { - this.setFileType(fileType); - } - } - - /** - * Set the header byte array WARNING: this implementation uses the passed in array directly, no copy is made so the - * caller should not reuse the array. - * - * @param header the byte array of header data - */ - @Override - public void setHeader(final byte[] header) { - this.header = header; - } - - /** - * Get the value of headerEncoding. Tells how to interpret the header information. - * - * @return Value of headerEncoding. - */ - @Override - public String getHeaderEncoding() { - return this.headerEncoding; - } - - /** - * Set the value of headerEncoding for proper interpretation and processing later - * - * @param v Value to assign to headerEncoding. - */ - @Override - public void setHeaderEncoding(final String v) { - this.headerEncoding = v; - } - - /** - * Set the footer byte array WARNING: this implementation uses the passed in array directly, no copy is made so the - * caller should not reuse the array. - * - * @param footer byte array of footer data - */ - @Override - public void setFooter(final byte[] footer) { - this.footer = footer; - } - - /** - * Set the filename - * - * @param f the new name of the data including path - */ - @Override - public void setFilename(final String f) { - this.theFileName = f; - this.shortName = makeShortName(); - } - - /** - * Set the byte channel factory using whichever implementation is providing access to the data. - * - * Setting this will null out {@link #theData} - */ - @Override - public void setChannelFactory(final SeekableByteChannelFactory sbcf) { - Validate.notNull(sbcf, "Required: SeekableByteChannelFactory not null"); - this.theData = null; - this.seekableByteChannelFactory = sbcf; - - // calls to setData clear the unsafe state by definition - // reset the usage checker but don't capture a snapshot until someone requests the data in byte[] form - safeUsageChecker.reset(); - } - - /** - * Returns the seekable byte channel factory containing a reference to the data, or wraps the in-memory data on the BDO - * in a new factory. - * - * @return the factory containing the data reference or the data wrapped in a new factory - */ - @Nullable - @Override - @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") - public SeekableByteChannelFactory getChannelFactory() { - switch (getDataState()) { - case BYTE_ARRAY_AND_CHANNEL: - throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); - case CHANNEL_ONLY: - return seekableByteChannelFactory; - case BYTE_ARRAY_ONLY: - return SeekableByteChannelHelper.memory(this.theData); - case NO_DATA: - default: - return null; - } - } - - /** - * {@inheritDoc} - */ - @Nullable - @Override - public InputStream newInputStream() { - final SeekableByteChannelFactory sbcf = getChannelFactory(); - - return sbcf == null ? null : Channels.newInputStream(sbcf.create()); - } - - /** - *

- * Return BaseDataObjects byte array OR as much as we can from the reference to the data up to MAX_BYTE_ARRAY_SIZE. - *

- * - *

- * Data returned from a backing Channel will be truncated at {@link BaseDataObject#MAX_BYTE_ARRAY_SIZE}. Using - * channel-related methods is now preferred to allow handling of larger objects - *

- * - *

- * WARNING: There is no way for the caller to know whether the data being returned is the direct array held in - * memory, or a copy of the data from a byte channel factory, so the returned byte array should be treated as live and - * not be modified. - *

- * - * @see #getChannelFactory() - * @return the data as a byte array - */ - @Nullable - @Override - @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") - public byte[] data() { - switch (getDataState()) { - case BYTE_ARRAY_AND_CHANNEL: - throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); - case BYTE_ARRAY_ONLY: - return theData; - case CHANNEL_ONLY: - // Max size here is slightly less than the true max size to avoid memory issues - final byte[] bytes = SeekableByteChannelHelper.getByteArrayFromBdo(this, MAX_BYTE_ARRAY_SIZE); - - // capture a reference to the returned byte[] so we can test for unsafe modifications of its contents - safeUsageChecker.recordSnapshot(bytes); - - return bytes; - case NO_DATA: - default: - return null; // NOSONAR maintains backwards compatibility - } - } - - /** - * @see #setData(byte[], int, int) - */ - @Override - public void setData(@Nullable final byte[] newData) { - this.seekableByteChannelFactory = null; - this.theData = newData == null ? new byte[0] : newData; - - // calls to setData clear the unsafe state by definition, but we need to capture a new snapshot - safeUsageChecker.resetCacheThenRecordSnapshot(theData); - } - - /** - *

- * Set new data on the BDO, using a range of the provided byte array. This will remove the reference to any byte channel - * factory that backs this BDO so be careful! - *

- * - *

- * Limited in size to 2^31. Use channel-based methods for larger data. - *

- * - * @param newData containing the source of the new data - * @param offset where to start copying from - * @param length how much to copy - * @see #setChannelFactory(SeekableByteChannelFactory) - */ - @Override - public void setData(@Nullable final byte[] newData, final int offset, final int length) { - this.seekableByteChannelFactory = null; - if (length <= 0 || newData == null) { - this.theData = new byte[0]; - } else { - this.theData = new byte[length]; - System.arraycopy(newData, offset, this.theData, 0, length); - } - - // calls to setData clear the unsafe state by definition, but we need to capture a new snapshot - safeUsageChecker.resetCacheThenRecordSnapshot(theData); - } - - /** - * Checks if the data is defined with a non-zero length. - * - * @return if data is undefined or zero length. - */ - @Override - public boolean hasContent() throws IOException { - return getChannelSize() > 0; - } - - /** - * Convenience method to get the size of the channel or byte array providing access to the data. - * - * @return the channel size - */ - @Override - @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") - public long getChannelSize() throws IOException { - switch (getDataState()) { - case BYTE_ARRAY_AND_CHANNEL: - throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); - case BYTE_ARRAY_ONLY: - return ArrayUtils.getLength(theData); - case CHANNEL_ONLY: - try (final SeekableByteChannel sbc = this.seekableByteChannelFactory.create()) { - return sbc.size(); - } - case NO_DATA: - default: - return 0; - } - } - - /** - * Fetch the size of the payload. Prefer to use: {@link #getChannelSize} - * - * @return the length of theData, or the size of the seekable byte channel up to - * {@link BaseDataObject#MAX_BYTE_ARRAY_SIZE}. - */ - @Override - @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") - public int dataLength() { - switch (getDataState()) { - case BYTE_ARRAY_AND_CHANNEL: - throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); - case BYTE_ARRAY_ONLY: - return ArrayUtils.getLength(theData); - case CHANNEL_ONLY: - try { - return (int) Math.min(getChannelSize(), MAX_BYTE_ARRAY_SIZE); - } catch (final IOException ioe) { - logger.error("Couldn't get size of channel on object {}", shortName(), ioe); - return 0; - } - case NO_DATA: - default: - return 0; - } - } - - @Override - public String shortName() { - return this.shortName; - } - - /** - * Construct the shortname - */ - private String makeShortName() { - /* - * using the file object works for most cases. It fails on the unix side if it is given a valid Windows path. - */ - // File file = new File( theFileName ); - // return file.getName(); - // so..... we'll have to perform the check ourselves ARRRRRRRRRGH!!!! - final int unixPathIndex = this.theFileName.lastIndexOf("/"); - if (unixPathIndex >= 0) { - return this.theFileName.substring(unixPathIndex + 1); - } - // check for windows path - final int windowsPathIndex = this.theFileName.lastIndexOf("\\"); - if (windowsPathIndex >= 0) { - return this.theFileName.substring(windowsPathIndex + 1); - } - - return this.theFileName; - } - - @Override - public String getFilename() { - return this.theFileName; - } - - @Override - public String currentForm() { - return currentFormAt(0); - } - - @Override - public String currentFormAt(final int i) { - if (i < this.currentForm.size()) { - return this.currentForm.get(i); - } - return ""; - } - - @Override - public int searchCurrentForm(final String value) { - return this.currentForm.indexOf(value); - } - - @Nullable - @Override - public String searchCurrentForm(final Collection values) { - for (final String value : values) { - if (this.currentForm.contains(value)) { - return value; - } - } - return null; - } - - @Override - public int currentFormSize() { - return this.currentForm.size(); - } - - @Override - public void replaceCurrentForm(@Nullable final String form) { - this.currentForm.clear(); - if (form != null) { - pushCurrentForm(form); - } - } - - /** - * Remove a form from the head of the list - * - * @return The value that was removed, or {@code null} if the list was empty. - */ - @Nullable - @Override - public String popCurrentForm() { - if (this.currentForm.isEmpty()) { - return null; - } else { - return this.currentForm.remove(0); - } - } - - @Override - public int deleteCurrentForm(final String form) { - int count = 0; - - if (this.currentForm == null || this.currentForm.isEmpty()) { - return count; - } - - // Remove all matching - for (final Iterator i = this.currentForm.iterator(); i.hasNext();) { - final String val = i.next(); - if (val.equals(form)) { - i.remove(); - count++; - } - } - return count; - } - - @Override - public int deleteCurrentFormAt(final int i) { - // Make sure its a legal position. - if ((i >= 0) && (i < this.currentForm.size())) { - this.currentForm.remove(i); - } - return this.currentForm.size(); - } - - @Override - public int addCurrentFormAt(final int i, final String newForm) { - if (newForm == null) { - throw new IllegalArgumentException("caller attempted to add a null form value at position " + i); - } - - checkForAndLogDuplicates(newForm, "addCurrentFormAt"); - if (i < this.currentForm.size()) { - this.currentForm.add(i, newForm); - } else { - this.currentForm.add(newForm); - } - return this.currentForm.size(); - } - - @Override - public int enqueueCurrentForm(final String newForm) { - if (newForm == null) { - throw new IllegalArgumentException("caller attempted to enqueue a null form value"); - } - - checkForAndLogDuplicates(newForm, "enqueueCurrentForm"); - this.currentForm.add(newForm); - return this.currentForm.size(); - } - - @Override - public int pushCurrentForm(final String newForm) { - if (newForm == null) { - throw new IllegalArgumentException("caller attempted to push a null form value"); - } else if (!PayloadUtil.isValidForm(newForm)) { - // If there is a key separator in the form, then throw an error log as this will cause issues in routing - logger.error("INVALID FORM: The form can only contain a-z, A-Z, 0-9, '-', '_', '()', '/', '+'. Given form: {}", newForm); - } - - checkForAndLogDuplicates(newForm, "pushCurrentForm"); - return addCurrentFormAt(0, newForm); - } - - @Override - public void setCurrentForm(final String newForm) { - setCurrentForm(newForm, false); - } - - @Override - public void setCurrentForm(final String newForm, final boolean clearAllForms) { - if (StringUtils.isBlank(newForm)) { - throw new IllegalArgumentException("caller attempted to set the current form to a null value"); - } - - if (clearAllForms) { - replaceCurrentForm(newForm); - } else { - popCurrentForm(); - pushCurrentForm(newForm); - } - } - - - @Override - public List getAllCurrentForms() { - return new ArrayList<>(this.currentForm); - } - - @Override - public void pullFormToTop(final String curForm) { - if (this.currentForm.size() > 1) { - // Delete it - final int count = deleteCurrentForm(curForm); - - // If deleted, add it back on top - if (count > 0) { - this.currentForm.add(0, curForm); - } - } - } - - private void checkForAndLogDuplicates(String newForm, String method) { - if (currentForm.contains(newForm)) { - logger.info("Duplicate form {} being added through BaseDataObject.{}", newForm, method); - } - } - - @Override - public String toString() { - final StringBuilder myOutput = new StringBuilder(); - final String ls = System.getProperty("line.separator"); - - myOutput.append(ls); - myOutput.append(" currentForms: ").append(getAllCurrentForms()).append(ls); - myOutput.append(" ").append(history); - - return myOutput.toString(); - } - - @Override - public String printMeta() { - return PayloadUtil.printFormattedMetadata(this); - } - - @Override - public void addProcessingError(final String err) { - if (this.procError == null) { - this.procError = new StringBuilder(); - } - this.procError.append(err).append("\n"); - } - - @Override - public String getProcessingError() { - String s = null; - if (this.procError != null) { - s = this.procError.toString(); - } - return s; - } - - @Override - public TransformHistory getTransformHistory() { - return new TransformHistory(history); - } - - @Override - public List transformHistory() { - return transformHistory(false); - } - - @Override - public List transformHistory(boolean includeCoordinated) { - return history.get(includeCoordinated); - } - - @Override - public void clearTransformHistory() { - this.history.clear(); - } - - @Override - public void appendTransformHistory(final String key) { - appendTransformHistory(key, false); - } - - @Override - public void appendTransformHistory(final String key, boolean coordinated) { - this.history.append(key, coordinated); - } - - @Override - public void setHistory(TransformHistory newHistory) { - this.history.set(newHistory); - } - - @Override - public String whereAmI() { - String host = null; - try { - host = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - host = "FAILED"; - } - return host; - } - - @Nullable - @Override - public DirectoryEntry getLastPlaceVisited() { - TransformHistory.History entry = history.lastVisit(); - return entry == null ? null : new DirectoryEntry(entry.getKey()); - } - - @Nullable - @Override - public DirectoryEntry getPenultimatePlaceVisited() { - TransformHistory.History entry = history.penultimateVisit(); - return entry == null ? null : new DirectoryEntry(entry.getKey()); - } - - @Override - public boolean hasVisited(final String pattern) { - return history.hasVisited(pattern); - } - - @Override - public boolean beforeStart() { - return history.beforeStart(); - } - - @Override - public void clearParameters() { - this.parameters.clear(); - } - - @Override - public boolean hasParameter(final String key) { - return this.parameters.containsKey(key); - } - - @Override - public void setParameters(final Map map) { - this.parameters.clear(); - putParameters(map); - } - - @Override - public void setParameter(final String key, final Object val) { - deleteParameter(key); - putParameter(key, val); - } - - @Override - public void putParameter(final String key, final Object val) { - this.parameters.removeAll(key); - - if (val instanceof Iterable) { - this.parameters.putAll(key, (Iterable) val); - } else { - this.parameters.put(key, val); - } - } - - /** - * Put a collection of parameters into the metadata map, keeping both old and new values - * - * @param m the map of new parameters - */ - @Override - public void putParameters(final Map m) { - putParameters(m, MergePolicy.KEEP_ALL); - } - - /** - * Merge in new parameters using the specified policy to determine whether to keep all values, unique values, or prefer - * existing values - * - * @param m map of new parameters - * @param policy the merge policy - */ - @Override - public void putParameters(final Map m, final MergePolicy policy) { - for (final Map.Entry entry : m.entrySet()) { - final String name = entry.getKey(); - - if ((policy == MergePolicy.KEEP_EXISTING) && this.parameters.containsKey(name)) { - continue; - } - - final Object value = entry.getValue(); - - if ((policy == MergePolicy.DROP_EXISTING)) { - // store the provided value for this key, discarding any previously-stored value - setParameter(name, value); - continue; - } - - if (value instanceof Iterable) { - for (final Object v : (Iterable) value) { - if (policy == MergePolicy.KEEP_ALL || policy == MergePolicy.KEEP_EXISTING) { - this.parameters.put(name, v); - } else if (policy == MergePolicy.DISTINCT) { - if (!this.parameters.containsEntry(name, v)) { - this.parameters.put(name, v); - } - } else { - throw new IllegalStateException("Unhandled parameter merge policy " + policy + " for " + name); - } - } - } else { - if (policy == MergePolicy.KEEP_ALL || policy == MergePolicy.KEEP_EXISTING) { - this.parameters.put(name, value); - } else if (policy == MergePolicy.DISTINCT) { - if (!this.parameters.containsEntry(name, value)) { - this.parameters.put(name, value); - } - } else { - throw new IllegalStateException("Unhandled parameter merge policy " + policy + " for " + name); - } - } - } - } - - - /** - * Put a collection of parameters into the metadata map, adding only distinct k/v pairs - * - * @param m the map of new parameters - */ - @Override - public void putUniqueParameters(final Map m) { - putParameters(m, MergePolicy.DISTINCT); - } - - /** - * Merge in parameters keeping existing keys unchanged - * - * @param m map of new parameters to consider - */ - @Override - public void mergeParameters(final Map m) { - putParameters(m, MergePolicy.KEEP_EXISTING); - } - - @Nullable - @Override - public List getParameter(final String key) { - // Try remapping - List v = this.parameters.get(key); - if (CollectionUtils.isEmpty(v)) { - return null; - } - return v; - } - - @Override - public void appendParameter(final String key, final CharSequence value) { - this.parameters.put(key, value); - } - - @Override - public void appendParameter(final String key, final Iterable values) { - this.parameters.putAll(key, values); - } - - /** - * Append data to the specified metadata element if it doesn't already exist If you expect to append a lot if things - * this way, this method might not have the performance characteristics that you expect. You can build a set and - * externally and append the values after they are uniqued. - * - * @param key name of the metadata element - * @param value the value to append - * @return true if the item is added, false if it already exists - */ - @Override - public boolean appendUniqueParameter(final String key, final CharSequence value) { - if (this.parameters.containsEntry(key, value)) { - return false; - } - - this.parameters.put(key, value); - return true; - } - - @Override - public String getStringParameter(final String key) { - return getStringParameter(key, DEFAULT_PARAM_SEPARATOR); - } - - @Nullable - @Override - public String getStringParameter(final String key, final String sep) { - final List obj = getParameter(key); - if (obj == null) { - return null; - } else if (obj.isEmpty()) { - return null; - } else if ((obj.size() == 1) && (obj.get(0) instanceof String)) { - return (String) obj.get(0); - } else if ((obj.size() == 1) && (obj.get(0) == null)) { - return null; - } else { - final StringBuilder sb = new StringBuilder(); - for (final Object item : obj) { - if (sb.length() > 0) { - sb.append(sep); - } - sb.append(item); - } - return sb.toString(); - } - } - - /** - * Retrieve all the metadata elements of this object This method returns possibly mapped metadata element names - * - * @return map of metadata elements - */ - @Override - public Map> getParameters() { - return this.parameters.asMap(); - } - - /** - * Get a processed represenation of the parameters for external use - */ - @Override - public Map getCookedParameters() { - final Map ext = new TreeMap<>(); - for (final String key : this.parameters.keySet()) { - ext.put(key.toString(), getStringParameter(key)); - } - return ext; - } - - @Override - public Set getParameterKeys() { - return this.parameters.keySet(); - } - - @Override - public List deleteParameter(final String key) { - return this.parameters.removeAll(key); - } - - @Override - public void setNumChildren(final int num) { - this.numChildren = num; - } - - @Override - public void setNumSiblings(final int num) { - this.numSiblings = num; - } - - @Override - public void setBirthOrder(final int num) { - this.birthOrder = num; - } - - @Override - public int getNumChildren() { - return this.numChildren; - } - - @Override - public int getNumSiblings() { - return this.numSiblings; - } - - @Override - public int getBirthOrder() { - return this.birthOrder; - } - - /** - * Return a reference to the header byte array. WARNING: this implementation returns the actual array directly, no copy - * is made so the caller must be aware that modifications to the returned array are live. - * - * @return byte array of header information or null if none - */ - @Override - public byte[] header() { - return this.header; - } - - @Override - @Deprecated(forRemoval = true) - public ByteBuffer headerBuffer() { - return ByteBuffer.wrap(header()); - } - - /** - * Return a reference to the footer byte array. WARNING: this implementation returns the actual array directly, no copy - * is made so the caller must be aware that modifications to the returned array are live. - * - * @return byte array of footer data or null if none - */ - @Override - public byte[] footer() { - return this.footer; - } - - - @Override - @Deprecated(forRemoval = true) - public ByteBuffer footerBuffer() { - return ByteBuffer.wrap(footer()); - } - - @Override - @Deprecated(forRemoval = true) - public ByteBuffer dataBuffer() { - return ByteBuffer.wrap(data()); - } - - @Override - public String getFontEncoding() { - return this.fontEncoding; - } - - @Override - public void setFontEncoding(final String fe) { - this.fontEncoding = fe; - } - - private static final String FILETYPE = "FILETYPE"; - - /** - * Put the FILETYPE parameter, null to clear - * - * @param v the value to store or null - */ - @Override - public void setFileType(@Nullable final String v) { - deleteParameter(FILETYPE); - if (v != null) { - setParameter(FILETYPE, v); - } - } - - @Override - @Deprecated(forRemoval = true) - public boolean setFileTypeIfEmpty(final String v, final String[] empties) { - if (isFileTypeEmpty(empties)) { - setFileType(v); - return true; - } - return false; - } - - @Override - public boolean setFileTypeIfEmpty(final String v) { - return setFileTypeIfEmpty(v, this.emptyFileTypes); - } - - @Override - public boolean isFileTypeEmpty() { - return isFileTypeEmpty(this.emptyFileTypes); - } - - /** - * Return true if the file type is null or in one of the specified set of empties - * - * @param empties a list of types that count as empty - */ - protected boolean isFileTypeEmpty(@Nullable final String[] empties) { - final String s = getFileType(); - - if (StringUtils.isEmpty(s)) { - return true; - } - - if (s.endsWith(Form.SUFFIXES_UNWRAPPED)) { - return true; - } - - for (int i = 0; empties != null && i < empties.length; i++) { - if (s.equals(empties[i])) { - return true; - } - } - return false; - } - - @Override - public String getFileType() { - return getStringParameter(FILETYPE); - } - - @Override - public int getNumAlternateViews() { - return this.multipartAlternative.size(); - } - - /** - * Return a specified multipart alternative view of the data WARNING: this implementation returns the actual array - * directly, no copy is made so the caller must be aware that modifications to the returned array are live. - * - * @param s the name of the view to retrieve - * @return byte array of alternate view data or null if none - */ - @Override - public byte[] getAlternateView(final String s) { - return this.multipartAlternative.get(s); - } - - @Override - public void appendAlternateView(final String name, final byte[] data) { - appendAlternateView(name, data, 0, data.length); - } - - @Override - public void appendAlternateView(final String name, final byte[] data, final int offset, final int length) { - final byte[] av = getAlternateView(name); - if (av != null) { - addAlternateView(name, ByteUtil.glue(av, 0, av.length - 1, data, offset, offset + length - 1)); - } else { - addAlternateView(name, data, offset, length); - } - } +import com.google.common.collect.LinkedListMultimap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - /** - * Return a specified multipart alternative view of the data in a buffer - * - * @param s the name of the view to retrieve - * @return buffer of alternate view data or null if none - */ - @Nullable - @Override - @Deprecated(forRemoval = true) - public ByteBuffer getAlternateViewBuffer(final String s) { - final byte[] viewdata = getAlternateView(s); - if (viewdata == null) { - return null; - } - return ByteBuffer.wrap(viewdata); - } +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import javax.annotation.Nullable; - /** - * Add a multipart alternative view of the data WARNING: this implementation returns the actual array directly, no copy - * is made so the caller must be aware that modifications to the returned array are live. - * - * @param name the name of the new view - * @param data the byte array of data for the view - */ - @Override - public void addAlternateView(final String name, @Nullable final byte[] data) { - if (data == null) { - this.multipartAlternative.remove(name); - } else { - this.multipartAlternative.put(name, data); - } - } +/** + * Class to hold data, header, footer, and attributes + */ +public class BaseDataObject extends BaseRecord implements Cloneable, IBaseDataObject { - @Override - public void addAlternateView(final String name, @Nullable final byte[] data, final int offset, final int length) { - if (data == null || length <= 0) { - this.multipartAlternative.remove(name); - } else { - final byte[] mpa = new byte[length]; - System.arraycopy(data, offset, mpa, 0, length); - this.multipartAlternative.put(name, mpa); - } - } + protected static final Logger logger = LoggerFactory.getLogger(BaseDataObject.class); - /** - * {@inheritDoc} - * - * @return an ordered set of alternate view names - */ - @Override - public Set getAlternateViewNames() { - return new TreeSet<>(this.multipartAlternative.keySet()); - } + /* Including this here make serialization of this object faster. */ + private static final long serialVersionUID = 7362181964652092657L; /** - * Get the alternate view map. WARNING: this implementation returns the actual map directly, no copy is made so the - * caller must be aware that modifications to the returned map are live. - * - * @return an map of alternate views ordered by name, key = String, value = byte[] + * The extracted records, if any */ - @Override - public Map getAlternateViews() { - return this.multipartAlternative; - } - - @Override - public boolean isBroken() { - return (this.brokenDocument != null); - } - - @Override - public void setBroken(@Nullable final String v) { - if (v == null) { - this.brokenDocument = null; - return; - } - - if (this.brokenDocument == null) { - this.brokenDocument = new StringBuilder(); - this.brokenDocument.append(v); - } else { - this.brokenDocument.append(", ").append(v); - } - } - @Nullable - @Override - public String getBroken() { - if (this.brokenDocument == null) { - return null; - } - return this.brokenDocument.toString(); - } + protected List extractedRecords; - @Override - public void setClassification(final String classification) { - this.classification = classification; + public BaseDataObject() { + super(); } - @Override - public String getClassification() { - return this.classification; + public BaseDataObject(final byte[] newData, final String name) { + super(newData, name); } - @Override - public int getPriority() { - return this.priority; + public BaseDataObject(final byte[] newData, final String name, @Nullable final String form) { + super(newData, name, form); } - @Override - public void setPriority(final int priority) { - this.priority = priority; + public BaseDataObject(final byte[] newData, final String name, final String form, @Nullable final String fileType) { + super(newData, name, form, fileType); } /** @@ -1378,26 +74,6 @@ public IBaseDataObject clone() throws CloneNotSupportedException { return c; } - @Override - public Instant getCreationTimestamp() { - return this.creationTimestamp; - } - - /** - * The creation timestamp is part of the provenance of the event represented by this instance. It is normally set from - * the constructor - * - * @param creationTimestamp when this item was created - */ - @Override - public void setCreationTimestamp(final Instant creationTimestamp) { - if (creationTimestamp == null) { - throw new IllegalArgumentException("Timestamp must not be null"); - } - - this.creationTimestamp = creationTimestamp; - } - @Override public List getExtractedRecords() { return this.extractedRecords; @@ -1464,49 +140,4 @@ public void clearExtractedRecords() { public int getExtractedRecordCount() { return (this.extractedRecords == null) ? 0 : this.extractedRecords.size(); } - - @Override - public UUID getInternalId() { - return this.internalId; - } - - @Override - public boolean isOutputable() { - return outputable; - } - - @Override - public void setOutputable(boolean outputable) { - this.outputable = outputable; - } - - @Override - public String getId() { - return id; - } - - @Override - public void setId(String id) { - this.id = id; - } - - @Override - public String getWorkBundleId() { - return workBundleId; - } - - @Override - public void setWorkBundleId(String workBundleId) { - this.workBundleId = workBundleId; - } - - @Override - public String getTransactionId() { - return transactionId; - } - - @Override - public void setTransactionId(String transactionId) { - this.transactionId = transactionId; - } } diff --git a/src/main/java/emissary/core/BaseRecord.java b/src/main/java/emissary/core/BaseRecord.java new file mode 100755 index 0000000000..bb09c09870 --- /dev/null +++ b/src/main/java/emissary/core/BaseRecord.java @@ -0,0 +1,1402 @@ +package emissary.core; + +import emissary.core.channels.SeekableByteChannelFactory; +import emissary.core.channels.SeekableByteChannelHelper; +import emissary.directory.DirectoryEntry; +import emissary.pickup.Priority; +import emissary.util.ByteUtil; +import emissary.util.PayloadUtil; + +import com.google.common.collect.LinkedListMultimap; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Serializable; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.rmi.Remote; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.UUID; +import javax.annotation.Nullable; + +/** + * Class to hold data, header, footer, and attributes + */ +public abstract class BaseRecord implements Serializable, Remote, IBaseRecord { + protected static final Logger logger = LoggerFactory.getLogger(BaseRecord.class); + + /* Used to limit the size of a returned byte array to avoid certain edge case scenarios */ + public static final int MAX_BYTE_ARRAY_SIZE = Integer.MAX_VALUE - 8; + private static final long serialVersionUID = -7482609339026723563L; + + /* Actual data - migrate away from this towards byte channels. */ + @Nullable + protected byte[] theData; + + /** + * Original name of the input data. Can only be set in the constructor of the DataObject. returned via the + * {@link #getFilename()} method. Also used in constructing the {@link #shortName()} of the document. + */ + protected String theFileName; + + /** + * Terminal portion of theFileName + */ + protected String shortName; + + /** + * The internal identifier, generated for each constructed object + */ + protected UUID internalId = UUID.randomUUID(); + + /** + * The currentForm is a stack of the itinerary items. The contents of the list are {@link String} and map to the + * dataType portion of the keys in the emissary.DirectoryPlace. + */ + protected List currentForm = new ArrayList<>(); + + /** + * History of processing errors. Lines of text are accumulated from String and returned in-toto as a String. + */ + protected StringBuilder procError; + + /** + * A travelogue built up as the agent moves about. Appended to by the agent as it goes from place to place. + */ + protected TransformHistory history = new TransformHistory(); + + /** + * The last determined language(characterset) of the data. + */ + @Nullable + protected String fontEncoding = null; + + /** + * Dynamic facets or metadata attributes of the data + */ + protected LinkedListMultimap parameters = LinkedListMultimap.create(100); + + /** + * If this file caused other agents to be sprouted, indicate how many + */ + protected int numChildren = 0; + + /** + * If this file has siblings that were sprouted at the same time, this will indicate how many total siblings there are. + * This can be used to navigate among siblings without needing to refer to the parent. + */ + protected int numSiblings = 0; + + /** + * What child is this in the family order + */ + protected int birthOrder = 0; + + /** + * Hash of alternate views of the data {@link String} current form is the key, byte[] is the value + */ + protected Map multipartAlternative = new TreeMap<>(); + + /** + * Any header that goes along with the data + */ + @Nullable + protected byte[] header = null; + + /** + * Any footer that goes along with the data + */ + @Nullable + protected byte[] footer = null; + + /** + * If the header has some encoding scheme record it + */ + @Nullable + protected String headerEncoding = null; + + /** + * Record the classification scheme for the document + */ + @Nullable + protected String classification = null; + + /** + * Keep track of if and how the document is broken so we can report on it later + */ + @Nullable + protected StringBuilder brokenDocument = null; + + // Filetypes that we think are equivalent to no file type at all + protected String[] emptyFileTypes = {Form.UNKNOWN}; + + /** + * The integer priority of the data object. A lower number is higher priority. + */ + protected int priority = Priority.DEFAULT; + + /** + * The timestamp for when the BaseDataObject was created. Used in data provenance tracking. + */ + protected Instant creationTimestamp; + + /** + * Check to see if this tree is able to be written out. + */ + protected boolean outputable = true; + + /** + * The unique identifier of this object + */ + protected String id; + + /** + * The identifier of the {@link emissary.pickup.WorkBundle} + */ + protected String workBundleId; + + /** + * The identifier used to track the object through the system + */ + protected String transactionId; + + /** + * A factory to create channels for the referenced data. + */ + @Nullable + protected SeekableByteChannelFactory seekableByteChannelFactory; + + final SafeUsageChecker safeUsageChecker = new SafeUsageChecker(); + + protected enum DataState { + NO_DATA, CHANNEL_ONLY, BYTE_ARRAY_ONLY, BYTE_ARRAY_AND_CHANNEL + } + + protected static final String INVALID_STATE_MSG = "Can't have both theData and seekableByteChannelFactory set. Object is %s"; + + /** + *

+ * Determine what state we're in with respect to the byte[] of data vs a channel. + *

+ * + *

+ * Not exposed publicly as consumers should be moving to channels, meaning ultimately the states will be simply either a + * channel factory exists or does not exist. + *

+ * + *

+ * Consumers should not modify their behaviour based on the state of the BDO, if they're being modified to handle + * channels, they should only handle channels, not both channels and byte[]. + *

+ * + * @return the {@link DataState} of this BDO + */ + protected DataState getDataState() { + if (theData == null) { + if (seekableByteChannelFactory == null) { + return DataState.NO_DATA; + } else { + return DataState.CHANNEL_ONLY; + } + } else { + if (seekableByteChannelFactory == null) { + return DataState.BYTE_ARRAY_ONLY; + } else { + return DataState.BYTE_ARRAY_AND_CHANNEL; + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void checkForUnsafeDataChanges() { + safeUsageChecker.checkForUnsafeDataChanges(); + + if (theData != null) { + safeUsageChecker.recordSnapshot(theData); + } + } + + /** + * Create an empty BaseDataObject. + */ + public BaseRecord() { + this.theData = null; + setCreationTimestamp(Instant.now()); + } + + /** + * Create a new BaseDataObject with byte array and name passed in. WARNING: this implementation uses the passed in array + * directly, no copy is made so the caller should not reuse the array. + * + * @param newData the bytes to hold + * @param name the name of the data item + */ + public BaseRecord(final byte[] newData, final String name) { + setData(newData); + setFilename(name); + setCreationTimestamp(Instant.now()); + } + + /** + * Create a new BaseDataObject with byte array, name, and initial form WARNING: this implementation uses the passed in + * array directly, no copy is made so the caller should not reuse the array. + * + * @param newData the bytes to hold + * @param name the name of the data item + * @param form the initial form of the data + */ + public BaseRecord(final byte[] newData, final String name, @Nullable final String form) { + this(newData, name); + if (form != null) { + pushCurrentForm(form); + } + } + + public BaseRecord(final byte[] newData, final String name, final String form, @Nullable final String fileType) { + this(newData, name, form); + if (fileType != null) { + this.setFileType(fileType); + } + } + + /** + * Set the header byte array WARNING: this implementation uses the passed in array directly, no copy is made so the + * caller should not reuse the array. + * + * @param header the byte array of header data + */ + @Override + public void setHeader(final byte[] header) { + this.header = header; + } + + /** + * Get the value of headerEncoding. Tells how to interpret the header information. + * + * @return Value of headerEncoding. + */ + @Override + public String getHeaderEncoding() { + return this.headerEncoding; + } + + /** + * Set the value of headerEncoding for proper interpretation and processing later + * + * @param v Value to assign to headerEncoding. + */ + @Override + public void setHeaderEncoding(final String v) { + this.headerEncoding = v; + } + + /** + * Set the footer byte array WARNING: this implementation uses the passed in array directly, no copy is made so the + * caller should not reuse the array. + * + * @param footer byte array of footer data + */ + @Override + public void setFooter(final byte[] footer) { + this.footer = footer; + } + + /** + * Set the filename + * + * @param f the new name of the data including path + */ + @Override + public void setFilename(final String f) { + this.theFileName = f; + this.shortName = makeShortName(); + } + + /** + * Set the byte channel factory using whichever implementation is providing access to the data. + * + * Setting this will null out {@link #theData} + */ + @Override + public void setChannelFactory(final SeekableByteChannelFactory sbcf) { + Validate.notNull(sbcf, "Required: SeekableByteChannelFactory not null"); + this.theData = null; + this.seekableByteChannelFactory = sbcf; + + // calls to setData clear the unsafe state by definition + // reset the usage checker but don't capture a snapshot until someone requests the data in byte[] form + safeUsageChecker.reset(); + } + + /** + * Returns the seekable byte channel factory containing a reference to the data, or wraps the in-memory data on the BDO + * in a new factory. + * + * @return the factory containing the data reference or the data wrapped in a new factory + */ + @Nullable + @Override + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") + public SeekableByteChannelFactory getChannelFactory() { + switch (getDataState()) { + case BYTE_ARRAY_AND_CHANNEL: + throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); + case CHANNEL_ONLY: + return seekableByteChannelFactory; + case BYTE_ARRAY_ONLY: + return SeekableByteChannelHelper.memory(this.theData); + case NO_DATA: + default: + return null; + } + } + + /** + * {@inheritDoc} + */ + @Nullable + @Override + public InputStream newInputStream() { + final SeekableByteChannelFactory sbcf = getChannelFactory(); + + return sbcf == null ? null : Channels.newInputStream(sbcf.create()); + } + + /** + *

+ * Return BaseDataObjects byte array OR as much as we can from the reference to the data up to MAX_BYTE_ARRAY_SIZE. + *

+ * + *

+ * Data returned from a backing Channel will be truncated at {@link BaseRecord#MAX_BYTE_ARRAY_SIZE}. Using + * channel-related methods is now preferred to allow handling of larger objects + *

+ * + *

+ * WARNING: There is no way for the caller to know whether the data being returned is the direct array held in + * memory, or a copy of the data from a byte channel factory, so the returned byte array should be treated as live and + * not be modified. + *

+ * + * @see #getChannelFactory() + * @return the data as a byte array + */ + @Nullable + @Override + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") + public byte[] data() { + switch (getDataState()) { + case BYTE_ARRAY_AND_CHANNEL: + throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); + case BYTE_ARRAY_ONLY: + return theData; + case CHANNEL_ONLY: + // Max size here is slightly less than the true max size to avoid memory issues + final byte[] bytes = SeekableByteChannelHelper.getByteArrayFromBdo(this, MAX_BYTE_ARRAY_SIZE); + + // capture a reference to the returned byte[] so we can test for unsafe modifications of its contents + safeUsageChecker.recordSnapshot(bytes); + + return bytes; + case NO_DATA: + default: + return null; // NOSONAR maintains backwards compatibility + } + } + + /** + * @see #setData(byte[], int, int) + */ + @Override + public void setData(@Nullable final byte[] newData) { + this.seekableByteChannelFactory = null; + this.theData = newData == null ? new byte[0] : newData; + + // calls to setData clear the unsafe state by definition, but we need to capture a new snapshot + safeUsageChecker.resetCacheThenRecordSnapshot(theData); + } + + /** + *

+ * Set new data on the BDO, using a range of the provided byte array. This will remove the reference to any byte channel + * factory that backs this BDO so be careful! + *

+ * + *

+ * Limited in size to 2^31. Use channel-based methods for larger data. + *

+ * + * @param newData containing the source of the new data + * @param offset where to start copying from + * @param length how much to copy + * @see #setChannelFactory(SeekableByteChannelFactory) + */ + @Override + public void setData(@Nullable final byte[] newData, final int offset, final int length) { + this.seekableByteChannelFactory = null; + if (length <= 0 || newData == null) { + this.theData = new byte[0]; + } else { + this.theData = new byte[length]; + System.arraycopy(newData, offset, this.theData, 0, length); + } + + // calls to setData clear the unsafe state by definition, but we need to capture a new snapshot + safeUsageChecker.resetCacheThenRecordSnapshot(theData); + } + + /** + * Checks if the data is defined with a non-zero length. + * + * @return if data is undefined or zero length. + */ + @Override + public boolean hasContent() throws IOException { + return getChannelSize() > 0; + } + + /** + * Convenience method to get the size of the channel or byte array providing access to the data. + * + * @return the channel size + */ + @Override + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") + public long getChannelSize() throws IOException { + switch (getDataState()) { + case BYTE_ARRAY_AND_CHANNEL: + throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); + case BYTE_ARRAY_ONLY: + return ArrayUtils.getLength(theData); + case CHANNEL_ONLY: + try (final SeekableByteChannel sbc = this.seekableByteChannelFactory.create()) { + return sbc.size(); + } + case NO_DATA: + default: + return 0; + } + } + + /** + * Fetch the size of the payload. Prefer to use: {@link #getChannelSize} + * + * @return the length of theData, or the size of the seekable byte channel up to {@link BaseRecord#MAX_BYTE_ARRAY_SIZE}. + */ + @Override + @SuppressWarnings("UnnecessaryDefaultInEnumSwitch") + public int dataLength() { + switch (getDataState()) { + case BYTE_ARRAY_AND_CHANNEL: + throw new IllegalStateException(String.format(INVALID_STATE_MSG, shortName())); + case BYTE_ARRAY_ONLY: + return ArrayUtils.getLength(theData); + case CHANNEL_ONLY: + try { + return (int) Math.min(getChannelSize(), MAX_BYTE_ARRAY_SIZE); + } catch (final IOException ioe) { + logger.error("Couldn't get size of channel on object {}", shortName(), ioe); + return 0; + } + case NO_DATA: + default: + return 0; + } + } + + @Override + public String shortName() { + return this.shortName; + } + + /** + * Construct the shortname + */ + private String makeShortName() { + /* + * using the file object works for most cases. It fails on the unix side if it is given a valid Windows path. + */ + // File file = new File( theFileName ); + // return file.getName(); + // so..... we'll have to perform the check ourselves ARRRRRRRRRGH!!!! + final int unixPathIndex = this.theFileName.lastIndexOf("/"); + if (unixPathIndex >= 0) { + return this.theFileName.substring(unixPathIndex + 1); + } + // check for windows path + final int windowsPathIndex = this.theFileName.lastIndexOf("\\"); + if (windowsPathIndex >= 0) { + return this.theFileName.substring(windowsPathIndex + 1); + } + + return this.theFileName; + } + + @Override + public String getFilename() { + return this.theFileName; + } + + @Override + public String currentForm() { + return currentFormAt(0); + } + + @Override + public String currentFormAt(final int i) { + if (i < this.currentForm.size()) { + return this.currentForm.get(i); + } + return ""; + } + + @Override + public int searchCurrentForm(final String value) { + return this.currentForm.indexOf(value); + } + + @Nullable + @Override + public String searchCurrentForm(final Collection values) { + for (final String value : values) { + if (this.currentForm.contains(value)) { + return value; + } + } + return null; + } + + @Override + public int currentFormSize() { + return this.currentForm.size(); + } + + @Override + public void replaceCurrentForm(@Nullable final String form) { + this.currentForm.clear(); + if (form != null) { + pushCurrentForm(form); + } + } + + /** + * Remove a form from the head of the list + * + * @return The value that was removed, or {@code null} if the list was empty. + */ + @Nullable + @Override + public String popCurrentForm() { + if (this.currentForm.isEmpty()) { + return null; + } else { + return this.currentForm.remove(0); + } + } + + @Override + public int deleteCurrentForm(final String form) { + int count = 0; + + if (this.currentForm == null || this.currentForm.isEmpty()) { + return count; + } + + // Remove all matching + for (final Iterator i = this.currentForm.iterator(); i.hasNext();) { + final String val = i.next(); + if (val.equals(form)) { + i.remove(); + count++; + } + } + return count; + } + + @Override + public int deleteCurrentFormAt(final int i) { + // Make sure its a legal position. + if ((i >= 0) && (i < this.currentForm.size())) { + this.currentForm.remove(i); + } + return this.currentForm.size(); + } + + @Override + public int addCurrentFormAt(final int i, final String newForm) { + if (newForm == null) { + throw new IllegalArgumentException("caller attempted to add a null form value at position " + i); + } + + checkForAndLogDuplicates(newForm, "addCurrentFormAt"); + if (i < this.currentForm.size()) { + this.currentForm.add(i, newForm); + } else { + this.currentForm.add(newForm); + } + return this.currentForm.size(); + } + + @Override + public int enqueueCurrentForm(final String newForm) { + if (newForm == null) { + throw new IllegalArgumentException("caller attempted to enqueue a null form value"); + } + + checkForAndLogDuplicates(newForm, "enqueueCurrentForm"); + this.currentForm.add(newForm); + return this.currentForm.size(); + } + + @Override + public int pushCurrentForm(final String newForm) { + if (newForm == null) { + throw new IllegalArgumentException("caller attempted to push a null form value"); + } else if (!PayloadUtil.isValidForm(newForm)) { + // If there is a key separator in the form, then throw an error log as this will cause issues in routing + logger.error("INVALID FORM: The form can only contain a-z, A-Z, 0-9, '-', '_', '()', '/', '+'. Given form: {}", newForm); + } + + checkForAndLogDuplicates(newForm, "pushCurrentForm"); + return addCurrentFormAt(0, newForm); + } + + @Override + public void setCurrentForm(final String newForm) { + setCurrentForm(newForm, false); + } + + @Override + public void setCurrentForm(final String newForm, final boolean clearAllForms) { + if (StringUtils.isBlank(newForm)) { + throw new IllegalArgumentException("caller attempted to set the current form to a null value"); + } + + if (clearAllForms) { + replaceCurrentForm(newForm); + } else { + popCurrentForm(); + pushCurrentForm(newForm); + } + } + + + @Override + public List getAllCurrentForms() { + return new ArrayList<>(this.currentForm); + } + + @Override + public void pullFormToTop(final String curForm) { + if (this.currentForm.size() > 1) { + // Delete it + final int count = deleteCurrentForm(curForm); + + // If deleted, add it back on top + if (count > 0) { + this.currentForm.add(0, curForm); + } + } + } + + private void checkForAndLogDuplicates(String newForm, String method) { + if (currentForm.contains(newForm)) { + logger.info("Duplicate form {} being added through BaseDataObject.{}", newForm, method); + } + } + + @Override + public String toString() { + final StringBuilder myOutput = new StringBuilder(); + final String ls = System.getProperty("line.separator"); + + myOutput.append(ls); + myOutput.append(" currentForms: ").append(getAllCurrentForms()).append(ls); + myOutput.append(" ").append(history); + + return myOutput.toString(); + } + + @Override + public String printMeta() { + return PayloadUtil.printFormattedMetadata(this); + } + + @Override + public void addProcessingError(final String err) { + if (this.procError == null) { + this.procError = new StringBuilder(); + } + this.procError.append(err).append("\n"); + } + + @Override + public String getProcessingError() { + String s = null; + if (this.procError != null) { + s = this.procError.toString(); + } + return s; + } + + @Override + public TransformHistory getTransformHistory() { + return new TransformHistory(history); + } + + @Override + public List transformHistory() { + return transformHistory(false); + } + + @Override + public List transformHistory(boolean includeCoordinated) { + return history.get(includeCoordinated); + } + + @Override + public void clearTransformHistory() { + this.history.clear(); + } + + @Override + public void appendTransformHistory(final String key) { + appendTransformHistory(key, false); + } + + @Override + public void appendTransformHistory(final String key, boolean coordinated) { + this.history.append(key, coordinated); + } + + @Override + public void setHistory(TransformHistory newHistory) { + this.history.set(newHistory); + } + + @Override + public String whereAmI() { + String host = null; + try { + host = InetAddress.getLocalHost().getCanonicalHostName(); + } catch (UnknownHostException e) { + host = "FAILED"; + } + return host; + } + + @Nullable + @Override + public DirectoryEntry getLastPlaceVisited() { + TransformHistory.History entry = history.lastVisit(); + return entry == null ? null : new DirectoryEntry(entry.getKey()); + } + + @Nullable + @Override + public DirectoryEntry getPenultimatePlaceVisited() { + TransformHistory.History entry = history.penultimateVisit(); + return entry == null ? null : new DirectoryEntry(entry.getKey()); + } + + @Override + public boolean hasVisited(final String pattern) { + return history.hasVisited(pattern); + } + + @Override + public boolean beforeStart() { + return history.beforeStart(); + } + + @Override + public void clearParameters() { + this.parameters.clear(); + } + + @Override + public boolean hasParameter(final String key) { + return this.parameters.containsKey(key); + } + + @Override + public void setParameters(final Map map) { + this.parameters.clear(); + putParameters(map); + } + + @Override + public void setParameter(final String key, final Object val) { + deleteParameter(key); + putParameter(key, val); + } + + @Override + public void putParameter(final String key, final Object val) { + this.parameters.removeAll(key); + + if (val instanceof Iterable) { + this.parameters.putAll(key, (Iterable) val); + } else { + this.parameters.put(key, val); + } + } + + /** + * Put a collection of parameters into the metadata map, keeping both old and new values + * + * @param m the map of new parameters + */ + @Override + public void putParameters(final Map m) { + putParameters(m, MergePolicy.KEEP_ALL); + } + + /** + * Merge in new parameters using the specified policy to determine whether to keep all values, unique values, or prefer + * existing values + * + * @param m map of new parameters + * @param policy the merge policy + */ + @Override + public void putParameters(final Map m, final MergePolicy policy) { + for (final Map.Entry entry : m.entrySet()) { + final String name = entry.getKey(); + + if ((policy == MergePolicy.KEEP_EXISTING) && this.parameters.containsKey(name)) { + continue; + } + + final Object value = entry.getValue(); + + if ((policy == MergePolicy.DROP_EXISTING)) { + // store the provided value for this key, discarding any previously-stored value + setParameter(name, value); + continue; + } + + if (value instanceof Iterable) { + for (final Object v : (Iterable) value) { + if (policy == MergePolicy.KEEP_ALL || policy == MergePolicy.KEEP_EXISTING) { + this.parameters.put(name, v); + } else if (policy == MergePolicy.DISTINCT) { + if (!this.parameters.containsEntry(name, v)) { + this.parameters.put(name, v); + } + } else { + throw new IllegalStateException("Unhandled parameter merge policy " + policy + " for " + name); + } + } + } else { + if (policy == MergePolicy.KEEP_ALL || policy == MergePolicy.KEEP_EXISTING) { + this.parameters.put(name, value); + } else if (policy == MergePolicy.DISTINCT) { + if (!this.parameters.containsEntry(name, value)) { + this.parameters.put(name, value); + } + } else { + throw new IllegalStateException("Unhandled parameter merge policy " + policy + " for " + name); + } + } + } + } + + + /** + * Put a collection of parameters into the metadata map, adding only distinct k/v pairs + * + * @param m the map of new parameters + */ + @Override + public void putUniqueParameters(final Map m) { + putParameters(m, MergePolicy.DISTINCT); + } + + /** + * Merge in parameters keeping existing keys unchanged + * + * @param m map of new parameters to consider + */ + @Override + public void mergeParameters(final Map m) { + putParameters(m, MergePolicy.KEEP_EXISTING); + } + + @Nullable + @Override + public List getParameter(final String key) { + // Try remapping + List v = this.parameters.get(key); + if (CollectionUtils.isEmpty(v)) { + return null; + } + return v; + } + + @Override + public void appendParameter(final String key, final CharSequence value) { + this.parameters.put(key, value); + } + + @Override + public void appendParameter(final String key, final Iterable values) { + this.parameters.putAll(key, values); + } + + /** + * Append data to the specified metadata element if it doesn't already exist If you expect to append a lot if things + * this way, this method might not have the performance characteristics that you expect. You can build a set and + * externally and append the values after they are uniqued. + * + * @param key name of the metadata element + * @param value the value to append + * @return true if the item is added, false if it already exists + */ + @Override + public boolean appendUniqueParameter(final String key, final CharSequence value) { + if (this.parameters.containsEntry(key, value)) { + return false; + } + + this.parameters.put(key, value); + return true; + } + + @Override + public String getStringParameter(final String key) { + return getStringParameter(key, DEFAULT_PARAM_SEPARATOR); + } + + @Nullable + @Override + public String getStringParameter(final String key, final String sep) { + final List obj = getParameter(key); + if (obj == null) { + return null; + } else if (obj.isEmpty()) { + return null; + } else if ((obj.size() == 1) && (obj.get(0) instanceof String)) { + return (String) obj.get(0); + } else if ((obj.size() == 1) && (obj.get(0) == null)) { + return null; + } else { + final StringBuilder sb = new StringBuilder(); + for (final Object item : obj) { + if (sb.length() > 0) { + sb.append(sep); + } + sb.append(item); + } + return sb.toString(); + } + } + + /** + * Retrieve all the metadata elements of this object This method returns possibly mapped metadata element names + * + * @return map of metadata elements + */ + @Override + public Map> getParameters() { + return this.parameters.asMap(); + } + + /** + * Get a processed represenation of the parameters for external use + */ + @Override + public Map getCookedParameters() { + final Map ext = new TreeMap<>(); + for (final String key : this.parameters.keySet()) { + ext.put(key.toString(), getStringParameter(key)); + } + return ext; + } + + @Override + public Set getParameterKeys() { + return this.parameters.keySet(); + } + + @Override + public List deleteParameter(final String key) { + return this.parameters.removeAll(key); + } + + @Override + public void setNumChildren(final int num) { + this.numChildren = num; + } + + @Override + public void setNumSiblings(final int num) { + this.numSiblings = num; + } + + @Override + public void setBirthOrder(final int num) { + this.birthOrder = num; + } + + @Override + public int getNumChildren() { + return this.numChildren; + } + + @Override + public int getNumSiblings() { + return this.numSiblings; + } + + @Override + public int getBirthOrder() { + return this.birthOrder; + } + + /** + * Return a reference to the header byte array. WARNING: this implementation returns the actual array directly, no copy + * is made so the caller must be aware that modifications to the returned array are live. + * + * @return byte array of header information or null if none + */ + @Override + public byte[] header() { + return this.header; + } + + @Override + @Deprecated(forRemoval = true) + public ByteBuffer headerBuffer() { + return ByteBuffer.wrap(header()); + } + + /** + * Return a reference to the footer byte array. WARNING: this implementation returns the actual array directly, no copy + * is made so the caller must be aware that modifications to the returned array are live. + * + * @return byte array of footer data or null if none + */ + @Override + public byte[] footer() { + return this.footer; + } + + + @Override + @Deprecated(forRemoval = true) + public ByteBuffer footerBuffer() { + return ByteBuffer.wrap(footer()); + } + + @Override + @Deprecated(forRemoval = true) + public ByteBuffer dataBuffer() { + return ByteBuffer.wrap(data()); + } + + @Override + public String getFontEncoding() { + return this.fontEncoding; + } + + @Override + public void setFontEncoding(final String fe) { + this.fontEncoding = fe; + } + + private static final String FILETYPE = "FILETYPE"; + + /** + * Put the FILETYPE parameter, null to clear + * + * @param v the value to store or null + */ + @Override + public void setFileType(@Nullable final String v) { + deleteParameter(FILETYPE); + if (v != null) { + setParameter(FILETYPE, v); + } + } + + @Override + @Deprecated(forRemoval = true) + public boolean setFileTypeIfEmpty(final String v, final String[] empties) { + if (isFileTypeEmpty(empties)) { + setFileType(v); + return true; + } + return false; + } + + @Override + public boolean setFileTypeIfEmpty(final String v) { + return setFileTypeIfEmpty(v, this.emptyFileTypes); + } + + @Override + public boolean isFileTypeEmpty() { + return isFileTypeEmpty(this.emptyFileTypes); + } + + /** + * Return true if the file type is null or in one of the specified set of empties + * + * @param empties a list of types that count as empty + */ + protected boolean isFileTypeEmpty(@Nullable final String[] empties) { + final String s = getFileType(); + + if (StringUtils.isEmpty(s)) { + return true; + } + + if (s.endsWith(Form.SUFFIXES_UNWRAPPED)) { + return true; + } + + for (int i = 0; empties != null && i < empties.length; i++) { + if (s.equals(empties[i])) { + return true; + } + } + return false; + } + + @Override + public String getFileType() { + return getStringParameter(FILETYPE); + } + + @Override + public int getNumAlternateViews() { + return this.multipartAlternative.size(); + } + + /** + * Return a specified multipart alternative view of the data WARNING: this implementation returns the actual array + * directly, no copy is made so the caller must be aware that modifications to the returned array are live. + * + * @param s the name of the view to retrieve + * @return byte array of alternate view data or null if none + */ + @Override + public byte[] getAlternateView(final String s) { + return this.multipartAlternative.get(s); + } + + @Override + public void appendAlternateView(final String name, final byte[] data) { + appendAlternateView(name, data, 0, data.length); + } + + @Override + public void appendAlternateView(final String name, final byte[] data, final int offset, final int length) { + final byte[] av = getAlternateView(name); + if (av != null) { + addAlternateView(name, ByteUtil.glue(av, 0, av.length - 1, data, offset, offset + length - 1)); + } else { + addAlternateView(name, data, offset, length); + } + } + + /** + * Return a specified multipart alternative view of the data in a buffer + * + * @param s the name of the view to retrieve + * @return buffer of alternate view data or null if none + */ + @Nullable + @Override + @Deprecated(forRemoval = true) + public ByteBuffer getAlternateViewBuffer(final String s) { + final byte[] viewdata = getAlternateView(s); + if (viewdata == null) { + return null; + } + return ByteBuffer.wrap(viewdata); + } + + /** + * Add a multipart alternative view of the data WARNING: this implementation returns the actual array directly, no copy + * is made so the caller must be aware that modifications to the returned array are live. + * + * @param name the name of the new view + * @param data the byte array of data for the view + */ + @Override + public void addAlternateView(final String name, @Nullable final byte[] data) { + if (data == null) { + this.multipartAlternative.remove(name); + } else { + this.multipartAlternative.put(name, data); + } + } + + @Override + public void addAlternateView(final String name, @Nullable final byte[] data, final int offset, final int length) { + if (data == null || length <= 0) { + this.multipartAlternative.remove(name); + } else { + final byte[] mpa = new byte[length]; + System.arraycopy(data, offset, mpa, 0, length); + this.multipartAlternative.put(name, mpa); + } + } + + /** + * {@inheritDoc} + * + * @return an ordered set of alternate view names + */ + @Override + public Set getAlternateViewNames() { + return new TreeSet<>(this.multipartAlternative.keySet()); + } + + /** + * Get the alternate view map. WARNING: this implementation returns the actual map directly, no copy is made so the + * caller must be aware that modifications to the returned map are live. + * + * @return an map of alternate views ordered by name, key = String, value = byte[] + */ + @Override + public Map getAlternateViews() { + return this.multipartAlternative; + } + + @Override + public boolean isBroken() { + return (this.brokenDocument != null); + } + + @Override + public void setBroken(@Nullable final String v) { + if (v == null) { + this.brokenDocument = null; + return; + } + + if (this.brokenDocument == null) { + this.brokenDocument = new StringBuilder(); + this.brokenDocument.append(v); + } else { + this.brokenDocument.append(", ").append(v); + } + } + + @Nullable + @Override + public String getBroken() { + if (this.brokenDocument == null) { + return null; + } + return this.brokenDocument.toString(); + } + + @Override + public void setClassification(final String classification) { + this.classification = classification; + } + + @Override + public String getClassification() { + return this.classification; + } + + @Override + public int getPriority() { + return this.priority; + } + + @Override + public void setPriority(final int priority) { + this.priority = priority; + } + + @Override + public Instant getCreationTimestamp() { + return this.creationTimestamp; + } + + /** + * The creation timestamp is part of the provenance of the event represented by this instance. It is normally set from + * the constructor + * + * @param creationTimestamp when this item was created + */ + @Override + public void setCreationTimestamp(final Instant creationTimestamp) { + if (creationTimestamp == null) { + throw new IllegalArgumentException("Timestamp must not be null"); + } + + this.creationTimestamp = creationTimestamp; + } + + @Override + public UUID getInternalId() { + return this.internalId; + } + + @Override + public boolean isOutputable() { + return outputable; + } + + @Override + public void setOutputable(boolean outputable) { + this.outputable = outputable; + } + + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Override + public String getWorkBundleId() { + return workBundleId; + } + + @Override + public void setWorkBundleId(String workBundleId) { + this.workBundleId = workBundleId; + } + + @Override + public String getTransactionId() { + return transactionId; + } + + @Override + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } +} diff --git a/src/main/java/emissary/core/DataObjectFactory.java b/src/main/java/emissary/core/DataObjectFactory.java index a1f07751f6..333ceff707 100755 --- a/src/main/java/emissary/core/DataObjectFactory.java +++ b/src/main/java/emissary/core/DataObjectFactory.java @@ -64,7 +64,7 @@ public static IBaseDataObject getInstance() { /** * Get an instance of the configured DataObject impl with pretty much arbitrary arguments to the constructor - * + * * @param args the arguments to the BaseDataObject constructor */ public static IBaseDataObject getInstance(final Object... args) { @@ -74,7 +74,7 @@ public static IBaseDataObject getInstance(final Object... args) { /** * Get an instance of the configured DataObject impl with filename, form, and file type set - * + * * @param payload the payload data * @param filename the filename * @param fileTypeAndForm the form and filetype to set on the IBDO @@ -86,7 +86,7 @@ public static IBaseDataObject getInstance(final byte[] payload, final String fil /** * Get an instance of the configured DataObject impl with filename, form, and file type set - * + * * @param payload the payload data * @param filename the filename * @param form the form to set on the IBDO diff --git a/src/main/java/emissary/core/ExtractedRecord.java b/src/main/java/emissary/core/ExtractedRecord.java new file mode 100644 index 0000000000..f91838b139 --- /dev/null +++ b/src/main/java/emissary/core/ExtractedRecord.java @@ -0,0 +1,27 @@ +package emissary.core; + +import javax.annotation.Nullable; + +public class ExtractedRecord extends BaseRecord implements IExtractedRecord { + + private static final long serialVersionUID = -1298573504297726742L; + + public ExtractedRecord() { + super(); + } + + public ExtractedRecord(byte[] newData, String name) { + super(newData, name); + } + + @SuppressWarnings("unused") + public ExtractedRecord(byte[] newData, String name, @Nullable String form) { + super(newData, name, form); + } + + @SuppressWarnings("unused") + public ExtractedRecord(byte[] newData, String name, String form, @Nullable String fileType) { + super(newData, name, form, fileType); + } + +} diff --git a/src/main/java/emissary/core/IBaseDataObject.java b/src/main/java/emissary/core/IBaseDataObject.java index e63f1bf67c..70a3a066f7 100755 --- a/src/main/java/emissary/core/IBaseDataObject.java +++ b/src/main/java/emissary/core/IBaseDataObject.java @@ -1,795 +1,8 @@ package emissary.core; -import emissary.core.channels.SeekableByteChannelFactory; -import emissary.directory.DirectoryEntry; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.time.Instant; -import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -public interface IBaseDataObject { - - /** - * Define the merge policy values for parameter handling - */ - enum MergePolicy { - DISTINCT, KEEP_EXISTING, KEEP_ALL, DROP_EXISTING - } - - /** - * Default separator when stringing parameter values together - */ - String DEFAULT_PARAM_SEPARATOR = ";"; - - /** - * Checks to see if payload byte arrays visible to external classes have any changes not explicitly saved via a call to - * the {@link IBaseDataObject#setData(byte[]) setData(byte[])}, {@link IBaseDataObject#setData(byte[], int, int) - * setData(byte[], int, int)}, or {@link IBaseDataObject#setChannelFactory(SeekableByteChannelFactory) - * setChannelFactory(SeekableByteChannelFactory)} method. - */ - void checkForUnsafeDataChanges(); - - /** - * Return the data as a byte array. If using a channel to the data, calling this method will only return up to - * Integer.MAX_VALUE bytes of the original data. - * - * @return byte array of the data - * @see #getChannelFactory() as the preferred data accessor for larger data - */ - byte[] data(); - - /** - * Set BaseDataObjects data to byte array passed in. - * - * @param newData byte array to set replacing any existing data - */ - void setData(byte[] newData); - - /** - * Set BaseDataObjects data to the portion of the byte array specified - * - * @param newData array containing desired data - * @param offset the index of the first byte to use - * @param length the number of bytes to use - */ - void setData(final byte[] newData, int offset, int length); - - /** - * Checks if the data is defined with a non-zero length. - * - * @return if data is undefined or zero length. - */ - boolean hasContent() throws IOException; - - /** - * Set the byte channel factory using whichever implementation is providing access to the data. - * - * @param sbcf the new channel factory to set on this object - */ - void setChannelFactory(final SeekableByteChannelFactory sbcf); - - /** - * Returns a new InputStream to the data that this BaseDataObject contains. - *

- * NOTE 1: Mutating the data elements of this IBaseDataObject while reading from the InputStream will have indeterminate - * results. - *

- * NOTE 2: The calling code is responsible for closing the returned InputStream. - * - * @return a new stream that reads the data that this object contains, or null if this object has no data. - */ - InputStream newInputStream(); - - /** - * Returns the seekable byte channel factory containing a reference to the data - * - * @return the factory containing the data reference - */ - SeekableByteChannelFactory getChannelFactory(); - - /** - * Get the size of the channel referenced by this object - * - * @return the channel size - * @throws IOException if an error occurs with the underlying channel - */ - long getChannelSize() throws IOException; - - /** - * Return length of the data, up to Integer.MAX_VALUE if the data is in a channel. - * - * Prefer use of {@link #getChannelSize()} going forwards - * - * @return length in bytes of the data - */ - int dataLength(); - - /** - * Set the header byte array - * - * @param arg1 the byte array of header data - */ - void setHeader(byte[] arg1); - - /** - * Return a reference to the header byte array. - * - * @return byte array of header information or null if none - */ - byte[] header(); - - /** - * Set the footer byte array - * - * @param arg1 byte array of footer data - */ - void setFooter(byte[] arg1); - - /** - * Return a reference to the footer byte array. - * - * @return byte array of footer data or null if none - */ - byte[] footer(); - - - /** - * Get the value of headerEncoding. Tells how to interpret the header information. - * - * @return Value of headerEncoding. - */ - String getHeaderEncoding(); - - /** - * Set the value of headerEncoding for proper interpretation and processing later - * - * @param arg1 Value to assign to headerEncoding. - */ - void setHeaderEncoding(String arg1); - - /** - * Get the classification string for the data - * - * @return String classification value - */ - String getClassification(); - - /** - * Set the classification. - * - * @param classification string classification value - */ - void setClassification(String classification); - - - /** - * Sets the number of children that the current agents spawned. - * - * @param num the number value to set - */ - void setNumChildren(int num); - - /** - * Gets the number of children that have this as a parent - * - * @return the number of children that have this parent - */ - int getNumChildren(); - - /** - * Sets the number of siblings for this data object. - * - * @param num the number of siblings to set - */ - void setNumSiblings(int num); - - /** - * Get the number of siblings - * - * @return the number of siblings including this one - */ - int getNumSiblings(); - - /** - * What number is this sibling in the family - * - * @param num the birthorder number value to set - */ - void setBirthOrder(int num); - - /** - * Get this sibling number, count from one. - * - * @return the birth order of this sibling - */ - int getBirthOrder(); - - /** - * Return the header wrapped in a ByteBuffer class. - * - * @return buffer required by the HTML Velocity templates. - */ - @Deprecated(forRemoval = true) - ByteBuffer headerBuffer(); - - /** - * Return the footer wrapped in a ByteBuffer class. - * - * @return buffer required by the HTML Velocity templates. - */ - @Deprecated(forRemoval = true) - ByteBuffer footerBuffer(); - - /** - * Return theData wrapped in a ByteBuffer class. - * - * @return buffer required by the HTML Velocity templates. - */ - @Deprecated(forRemoval = true) - ByteBuffer dataBuffer(); - - /** - * Get the font encoding string - * - * @return string name of font encoding for the data - */ - String getFontEncoding(); - - /** - * Set the font encoding string - * - * @param arg1 string name of font encoding for the data - */ - void setFontEncoding(String arg1); - - /** - * Clear all metadata elements - */ - void clearParameters(); - - /** - * Determine if parameter is present - * - * @param key name of metadata element to check - */ - boolean hasParameter(String key); - - /** - * Replace all of the metadata elements with a new set - * - * @param map the new set - */ - void setParameters(Map map); - - /** - * Set a new parameter value, deleting an old one - * - * @param key the name of the element - * @param val the value of the element - */ - void setParameter(String key, Object val); - - /** - * Put a new metadata element into the map - * - * @param key the name of the element - * @param val the value of the element - */ - void putParameter(String key, Object val); - - /** - * Put a collection of parameters into the metadata map - * - * @param m the map of new parameters - */ - void putParameters(Map m); - /** - * Put a collection of parameters into the metadata map using the specified merge policy - * - * @param m the map of new parameters - * @param policy the merge policy - */ - void putParameters(Map m, MergePolicy policy); - - /** - * Merge a collection of parameters into the metadata map - * - * @param m the map of new parameters - */ - void mergeParameters(Map m); - - /** - * Put a collection of parameters into the metadata map uniquely - * - * @param m the map of new parameters - */ - void putUniqueParameters(Map m); - - /** - * Retrieve a specified metadata element - * - * @param key name of the metadata element - * @return the value or null if no such element - */ - List getParameter(String key); - - /** - * Append data to the specified metadata element - * - * @param key name of the metadata element - * @param value the value to append - */ - void appendParameter(String key, CharSequence value); - - /** - * Append data values to the specified metadata element - * - * @param key name of the metadata element - * @param values the values to append - */ - void appendParameter(String key, Iterable values); - - /** - * Append data to the specified metadata element if it doesn't exist - * - * @param key name of the metadata element - * @param value the value to append - * @return true if the item is added, false if it already exists - */ - boolean appendUniqueParameter(String key, CharSequence value); - - /** - * Retrieve a specified metadata element as a string value - * - * @param key name of the metadata element - * @return the string value or null if no such element - */ - String getStringParameter(String key); - - /** - * Retrieve a specified metadata element as a string value - * - * @param key name of the metadata element - * @param sep the separator for multivalued fields - * @return the string value or null if no such element - */ - String getStringParameter(String key, String sep); - - /** - * Retrieve all the metadata elements of this object - * - * @return map of metadata elements - */ - Map> getParameters(); - - /** - * Retrieve all the metadata elements of this object in a way that is processed for use external to this instance - * - * @return map of metadata elements - */ - Map getCookedParameters(); - - /** - * Retrieve all of the current metadata keys - * - * @return set of charsequence keys - */ - Set getParameterKeys(); - - /** - * Delete the specified metadata element named - * - * @param key the name of the metadata item to delete - * @return the object deleted of null if none - */ - List deleteParameter(String key); - - /** - * Put the FILETYPE parameter - * - * @param arg1 the value to store - */ - void setFileType(String arg1); - - /** - * Set FILETYPE parameter iff empty. - * - * @param arg1 the value of the filetype to set - * @param arg2 the list of things caller considers equal to being empty - * @return true if it was empty and set - * @deprecated Use {@link #setFileType(String)} instead. - */ - @Deprecated(forRemoval = true) - boolean setFileTypeIfEmpty(String arg1, String[] arg2); - - /** - * Set FILETYPE parameter iff empty using the built-in definition of empty - * - * @param arg1 the value of the filetype to set - * @return true if it was empty and set - */ - boolean setFileTypeIfEmpty(String arg1); - - /** - * Return true if the file type is null or in one of the "don't care" set - * - * @since 3.3.3 - */ - boolean isFileTypeEmpty(); - - /** - * Get the FILETYPE parameter - * - * @return the string value of the FILETYPE parameter - */ - String getFileType(); - - /** - * Disclose how many multipart alternative views of the data exist - * - * @return count of alternate views - */ - int getNumAlternateViews(); - - /** - * Return a specified multipart alternative view of the data - * - * @param arg1 the name of the view to retrieve - * @return byte array of alternate view data - */ - byte[] getAlternateView(String arg1); - - /** - * Return a specified multipart alternative view of the data in a buffer - * - * @param arg1 the name of the view to retrieve - * @return buffer of alternate view data - */ - @Deprecated(forRemoval = true) - ByteBuffer getAlternateViewBuffer(String arg1); - - /** - * Add a multipart alternative view of the data - * - * @param name the name of the new view - * @param data the byte array of data for the view - */ - void addAlternateView(String name, byte[] data); - - /** - * Add a multipart alternative view of the data - * - * @param name the name of the new view - * @param data the byte array conatining data for the view - * @param offset index of the first byte to use - * @param length number of bytes to use - */ - void addAlternateView(String name, byte[] data, int offset, int length); - - /** - * Append the specified data to the alternate view - * - * @param name the name of the new view - * @param data the byte array of data for the view - */ - void appendAlternateView(String name, byte[] data); - - /** - * Append to a multipart alternative view of the data - * - * @param name the name of the view - * @param data the byte array conatining data for the view - * @param offset index of the first byte to use - * @param length number of bytes to use - */ - void appendAlternateView(String name, byte[] data, int offset, int length); - - /** - * Get the set of alt view names for new foreach loops - * - * @return set of alternate view names - */ - Set getAlternateViewNames(); - - /** - * Get the alternate view map. - * - * @return map of alternate views, key = String, value = byte[] - */ - Map getAlternateViews(); - - /** - * Test for broken document - * - * @return true if broken - */ - boolean isBroken(); - - /** - * Set brokenness for document - * - * @param arg1 the message to record - */ - void setBroken(String arg1); - - /** - * Get brokenness indicator message - * - * @return string message of what is broken - */ - String getBroken(); - - /** - * Returns the name of the file without the path with which the file will be written. - * - * @return the short name of the file (no path) - */ - String shortName(); - - /** - * Returns the filename associated with the data. - * - * @return the string name with path - */ - String getFilename(); - - /** - * Returns the internally generated identifier used to track the object - * - * @return a String representing the internal ID - */ - UUID getInternalId(); - - /** - * Set the filename - * - * @param f the new name of the data including path - */ - void setFilename(String f); - - - /** - * Return the current form of the data (top of the stack) - * - * @return string value of current form - */ - String currentForm(); - - /** - * Return the current form at specified position of the list - * - * @param i The specified position - * @return String containing the form or empty string if illegal position - */ - String currentFormAt(int i); - - /** - * Check to see if this value is already on the stack of itinerary items - * - * @param val the string to look for - * @return the position where it was found or -1 - */ - int searchCurrentForm(String val); - - /** - * Check to see one of these values is on the stack of itinerary items - * - * @param values the List of strings to look for - * @return the String that was found out of the list sent in or null - */ - String searchCurrentForm(Collection values); - - /** - * Get the size of the itinerary stack - * - * @return size of form stack - */ - int currentFormSize(); - - /** - * Remove a form from the head of the list - * - * @return the new size of the itinerary stack - */ - String popCurrentForm(); - - /** - * Replace all current forms with specified - * - * @param form the new current form or null if none desired - */ - void replaceCurrentForm(String form); - - /** - * Remove a form where ever it appears in the stack - * - * @param form the value to remove - * @return the number of elements removed from the stack - */ - int deleteCurrentForm(String form); - - /** - * Remove a form at the specified location of the itinerary stack - * - * @param i the position to delete - * @return the new size of the itinerary stack - */ - int deleteCurrentFormAt(int i); - - /** - * Add current form newForm at idx - * - * @param i the position to do the insert - * @param val the value to insert - * @return size of the new stack - */ - int addCurrentFormAt(int i, String val); - - /** - * Add a form to the end of the list (the bottom of the stack) - * - * @param val the new value to add to the tail of the stack - * @return the new size of the itinerary stack - */ - int enqueueCurrentForm(String val); - - /** - * Push a form onto the head of the list - * - * @param val the new value to push on the stack - * @return the new size of the itinerary stack - */ - int pushCurrentForm(String val); - - /** - * Replaces the current form of the data with a new form Does a pop() followed by a push(newForm) to simulate what would - * happen in the old "one form at a time system" - * - * @param val value of the the new form of the data - */ - void setCurrentForm(String val); - - /** - * Replaces the current form of the data with a form passed and potentially clears the entire form stack - * - * @param val value of the the new form of the data - * @param clearAllForms whether or not to clear the entire form stack - */ - void setCurrentForm(String val, boolean clearAllForms); - - /** - * Return a clone the whole current form list Note this is not a reference to our private store - * - * @return ordered list of current forms - */ - List getAllCurrentForms(); - - /** - * Move curForm to the top of the stack pushing everything above it down one slot - * - * @param curForm the form to pull to the top - */ - void pullFormToTop(String curForm); - - /** - * Return BaseDataObjects info as a String. - * - * @return string value of this object - */ - @Override - String toString(); - - - /** - * Record a processing error - * - * @param val the new error message to record - */ - void addProcessingError(String val); - - /** - * Retrieve the processing error(s) - * - * @return string value of processing errors - */ - String getProcessingError(); - - /** - * Replace history with the new history - * - * @param history of new history strings to use - */ - void setHistory(TransformHistory history); - - /** - * Get the transform history - * - * @return history of places visited - */ - TransformHistory getTransformHistory(); - - /** - * List of places the data object was carried to. - * - * @return List of strings making up the history - */ - List transformHistory(); - - /** - * List of places the data object was carried to. - * - * @param includeCoordinated include the places that were coordinated - * @return List of strings making up the history - */ - List transformHistory(boolean includeCoordinated); - - /** - * Clear the transformation history - */ - void clearTransformHistory(); - - /** - * Appends the new key to the transform history. This is called by MobileAgent before moving to the new place. It - * usually adds the four-tuple of a place's key - * - * @see emissary.core.MobileAgent#agentControl - * @param key the new value to append - */ - void appendTransformHistory(String key); - - /** - * Appends the new key to the transform history. This is called by MobileAgent before moving to the new place. It - * usually adds the four-tuple of a place's key. Coordinated history keys are meant for informational purposes and have - * no bearing on the routing algorithm. It is important to list the places visited in coordination, but should not - * report as the last place visited. - * - * @see emissary.core.MobileAgent#agentControl - * @param key the new value to append - * @param coordinated true if history entries are for informational purposes only - */ - void appendTransformHistory(String key, boolean coordinated); - - /** - * Return what machine we are located on - * - * @return string local host name - */ - String whereAmI(); - - /** - * Return an SDE based on the last item in the transform history or null if empty - * - * @return last item in history - */ - DirectoryEntry getLastPlaceVisited(); - - /** - * Return an SDE based on the penultimate item in the transform history or null if empty - * - * @return penultimate item in history - */ - DirectoryEntry getPenultimatePlaceVisited(); - - /** - * Return true if the payload has been to a place matching the key passed in. - * - * @param pattern the key pattern to match - */ - boolean hasVisited(String pattern); - - /** - * True if this payload hasn't had any processing yet Does not count parent processing as being for this payload - * - * @return true if not yet started - */ - boolean beforeStart(); +public interface IBaseDataObject extends IBaseRecord { /** * Support deep copy via clone @@ -797,39 +10,6 @@ enum MergePolicy { @Deprecated IBaseDataObject clone() throws CloneNotSupportedException; - /** - * Print the parameters, nicely formatted - */ - String printMeta(); - - /** - * Get data object's priority. - * - * @return int priority (lower the number, higher the priority). - */ - int getPriority(); - - /** - * Set the data object's priority, typically based on input dir/file priority. - * - * @param priority int (lower the number, higher the priority). - */ - void setPriority(int priority); - - /** - * Get the timestamp for when the object was created. This attribute will be used for data provenance. - * - * @return date - the timestamp the object was created - */ - Instant getCreationTimestamp(); - - /** - * Set the timestamp for when the object was created - * - * @param creationTimestamp - the date the object was created - */ - void setCreationTimestamp(Instant creationTimestamp); - /** * Get the List of extracted records */ @@ -873,59 +53,4 @@ enum MergePolicy { */ int getExtractedRecordCount(); - /** - * Test if tree is outputable - * - * @return true if this tree is not able to be output, false otherwise - */ - boolean isOutputable(); - - /** - * Set whether or not the tree is able to be written out - * - * @param outputable true if this tree is not able to be output, false otherwise - */ - void setOutputable(boolean outputable); - - /** - * Get ID - * - * @return the unique identifier of the IBaseDataObject - */ - String getId(); - - /** - * Set the unique identifier of the IBaseDataObject - * - * @param id the unique identifier of the IBaseDataObject - */ - void setId(String id); - - /** - * Get the Work Bundle ID - * - * @return the unique identifier of the {@link emissary.pickup.WorkBundle} - */ - String getWorkBundleId(); - - /** - * Set the unique identifier of the {@link emissary.pickup.WorkBundle} - * - * @param workBundleId the unique identifier of the {@link emissary.pickup.WorkBundle} - */ - void setWorkBundleId(String workBundleId); - - /** - * Get the Transaction ID - * - * @return the unique identifier of the transaction - */ - String getTransactionId(); - - /** - * Set the unique identifier of the transaction - * - * @param transactionId the unique identifier of the transaction - */ - void setTransactionId(String transactionId); } diff --git a/src/main/java/emissary/core/IBaseDataObjectHelper.java b/src/main/java/emissary/core/IBaseDataObjectHelper.java index 6aa1486918..9831b28bea 100644 --- a/src/main/java/emissary/core/IBaseDataObjectHelper.java +++ b/src/main/java/emissary/core/IBaseDataObjectHelper.java @@ -50,7 +50,7 @@ private IBaseDataObjectHelper() {} public static IBaseDataObject clone(final IBaseDataObject iBaseDataObject, final boolean fullClone) { Validate.notNull(iBaseDataObject, "Required: iBaseDataObject not null"); - final BaseDataObject bdo = fullClone ? new InternalIdBaseDataObject(iBaseDataObject.getInternalId()) : new BaseDataObject(); + final IBaseDataObject bdo = fullClone ? new InternalIdBaseDataObject(iBaseDataObject.getInternalId()) : DataObjectFactory.getInstance(); final SeekableByteChannelFactory sbcf = iBaseDataObject.getChannelFactory(); if (sbcf != null) { diff --git a/src/main/java/emissary/core/IBaseDataObjectXmlCodecs.java b/src/main/java/emissary/core/IBaseDataObjectXmlCodecs.java index 7e4fa4f79d..daf6e69518 100644 --- a/src/main/java/emissary/core/IBaseDataObjectXmlCodecs.java +++ b/src/main/java/emissary/core/IBaseDataObjectXmlCodecs.java @@ -86,12 +86,12 @@ public final class IBaseDataObjectXmlCodecs { */ public static final Map PRIMITVE_NAME_DEFAULT_MAP = Collections .unmodifiableMap(new ConcurrentHashMap<>(Stream.of( - new AbstractMap.SimpleEntry<>(BIRTH_ORDER, new BaseDataObject().getBirthOrder()), - new AbstractMap.SimpleEntry<>(BROKEN, new BaseDataObject().isBroken()), - new AbstractMap.SimpleEntry<>(NUM_CHILDREN, new BaseDataObject().getNumChildren()), - new AbstractMap.SimpleEntry<>(NUM_SIBLINGS, new BaseDataObject().getNumSiblings()), - new AbstractMap.SimpleEntry<>(OUTPUTABLE, new BaseDataObject().isOutputable()), - new AbstractMap.SimpleEntry<>(PRIORITY, new BaseDataObject().getPriority())) + new AbstractMap.SimpleEntry<>(BIRTH_ORDER, DataObjectFactory.getInstance().getBirthOrder()), + new AbstractMap.SimpleEntry<>(BROKEN, DataObjectFactory.getInstance().isBroken()), + new AbstractMap.SimpleEntry<>(NUM_CHILDREN, DataObjectFactory.getInstance().getNumChildren()), + new AbstractMap.SimpleEntry<>(NUM_SIBLINGS, DataObjectFactory.getInstance().getNumSiblings()), + new AbstractMap.SimpleEntry<>(OUTPUTABLE, DataObjectFactory.getInstance().isOutputable()), + new AbstractMap.SimpleEntry<>(PRIORITY, DataObjectFactory.getInstance().getPriority())) .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey, AbstractMap.SimpleEntry::getValue)))); /** * The XML namespace for "xml". @@ -824,6 +824,6 @@ public static Element protectedElementSha256(final String name, final byte[] byt */ public static Method getIbdoMethod(final String methodName, final Class... parameterTypes) throws NoSuchMethodException { - return IBaseDataObject.class.getDeclaredMethod(methodName, parameterTypes); + return IBaseDataObject.class.getMethod(methodName, parameterTypes); } } diff --git a/src/main/java/emissary/core/IBaseDataObjectXmlHelper.java b/src/main/java/emissary/core/IBaseDataObjectXmlHelper.java index 972c46fe6e..2ecf1be94a 100644 --- a/src/main/java/emissary/core/IBaseDataObjectXmlHelper.java +++ b/src/main/java/emissary/core/IBaseDataObjectXmlHelper.java @@ -64,7 +64,7 @@ private IBaseDataObjectXmlHelper() {} */ public static IBaseDataObject createStandardInitialIbdo(final IBaseDataObject ibdo, final SeekableByteChannelFactory sbcf, final String classification, final String formAndFileType, final KffDataObjectHandler kff) { - final IBaseDataObject tempIbdo = new BaseDataObject(); + final IBaseDataObject tempIbdo = DataObjectFactory.getInstance(); // We want to return the ibdo with the data field equal to null. This can only // be accomplished if the data is never set. Therefore, we have to set the data @@ -96,13 +96,13 @@ public static IBaseDataObject ibdoFromXml(final Document document, final List answerChildren = answersElement.getChildren(); ibdoFromXmlMainElements(answersElement, parentIbdo, decoders); for (final Element answerChild : answerChildren) { - final IBaseDataObject childIbdo = new BaseDataObject(); + final IBaseDataObject childIbdo = DataObjectFactory.getInstance(); final String childName = answerChild.getName(); if (childName.startsWith(EXTRACTED_RECORD_ELEMENT_PREFIX)) { diff --git a/src/main/java/emissary/core/IBaseRecord.java b/src/main/java/emissary/core/IBaseRecord.java new file mode 100755 index 0000000000..699c94d838 --- /dev/null +++ b/src/main/java/emissary/core/IBaseRecord.java @@ -0,0 +1,882 @@ +package emissary.core; + +import emissary.core.channels.SeekableByteChannelFactory; +import emissary.directory.DirectoryEntry; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +public interface IBaseRecord { + + /** + * Define the merge policy values for parameter handling + */ + enum MergePolicy { + DISTINCT, KEEP_EXISTING, KEEP_ALL, DROP_EXISTING + } + + /** + * Default separator when stringing parameter values together + */ + String DEFAULT_PARAM_SEPARATOR = ";"; + + /** + * Checks to see if payload byte arrays visible to external classes have any changes not explicitly saved via a call to + * the {@link IBaseRecord#setData(byte[]) setData(byte[])}, {@link IBaseRecord#setData(byte[], int, int) setData(byte[], + * int, int)}, or {@link IBaseRecord#setChannelFactory(SeekableByteChannelFactory) + * setChannelFactory(SeekableByteChannelFactory)} method. + */ + void checkForUnsafeDataChanges(); + + /** + * Return the data as a byte array. If using a channel to the data, calling this method will only return up to + * Integer.MAX_VALUE bytes of the original data. + * + * @return byte array of the data + * @see #getChannelFactory() as the preferred data accessor for larger data + */ + byte[] data(); + + /** + * Set BaseDataObjects data to byte array passed in. + * + * @param newData byte array to set replacing any existing data + */ + void setData(byte[] newData); + + /** + * Set BaseDataObjects data to the portion of the byte array specified + * + * @param newData array containing desired data + * @param offset the index of the first byte to use + * @param length the number of bytes to use + */ + void setData(final byte[] newData, int offset, int length); + + /** + * Checks if the data is defined with a non-zero length. + * + * @return if data is undefined or zero length. + */ + boolean hasContent() throws IOException; + + /** + * Set the byte channel factory using whichever implementation is providing access to the data. + * + * @param sbcf the new channel factory to set on this object + */ + void setChannelFactory(final SeekableByteChannelFactory sbcf); + + /** + * Returns a new InputStream to the data that this BaseDataObject contains. + *

+ * NOTE 1: Mutating the data elements of this IBaseDataObject while reading from the InputStream will have indeterminate + * results. + *

+ * NOTE 2: The calling code is responsible for closing the returned InputStream. + * + * @return a new stream that reads the data that this object contains, or null if this object has no data. + */ + InputStream newInputStream(); + + /** + * Returns the seekable byte channel factory containing a reference to the data + * + * @return the factory containing the data reference + */ + SeekableByteChannelFactory getChannelFactory(); + + /** + * Get the size of the channel referenced by this object + * + * @return the channel size + * @throws IOException if an error occurs with the underlying channel + */ + long getChannelSize() throws IOException; + + /** + * Return length of the data, up to Integer.MAX_VALUE if the data is in a channel. + * + * Prefer use of {@link #getChannelSize()} going forwards + * + * @return length in bytes of the data + */ + int dataLength(); + + /** + * Set the header byte array + * + * @param arg1 the byte array of header data + */ + void setHeader(byte[] arg1); + + /** + * Return a reference to the header byte array. + * + * @return byte array of header information or null if none + */ + byte[] header(); + + /** + * Set the footer byte array + * + * @param arg1 byte array of footer data + */ + void setFooter(byte[] arg1); + + /** + * Return a reference to the footer byte array. + * + * @return byte array of footer data or null if none + */ + byte[] footer(); + + + /** + * Get the value of headerEncoding. Tells how to interpret the header information. + * + * @return Value of headerEncoding. + */ + String getHeaderEncoding(); + + /** + * Set the value of headerEncoding for proper interpretation and processing later + * + * @param arg1 Value to assign to headerEncoding. + */ + void setHeaderEncoding(String arg1); + + /** + * Get the classification string for the data + * + * @return String classification value + */ + String getClassification(); + + /** + * Set the classification. + * + * @param classification string classification value + */ + void setClassification(String classification); + + + /** + * Sets the number of children that the current agents spawned. + * + * @param num the number value to set + */ + void setNumChildren(int num); + + /** + * Gets the number of children that have this as a parent + * + * @return the number of children that have this parent + */ + int getNumChildren(); + + /** + * Sets the number of siblings for this data object. + * + * @param num the number of siblings to set + */ + void setNumSiblings(int num); + + /** + * Get the number of siblings + * + * @return the number of siblings including this one + */ + int getNumSiblings(); + + /** + * What number is this sibling in the family + * + * @param num the birthorder number value to set + */ + void setBirthOrder(int num); + + /** + * Get this sibling number, count from one. + * + * @return the birth order of this sibling + */ + int getBirthOrder(); + + /** + * Return the header wrapped in a ByteBuffer class. + * + * @return buffer required by the HTML Velocity templates. + */ + @Deprecated(forRemoval = true) + ByteBuffer headerBuffer(); + + /** + * Return the footer wrapped in a ByteBuffer class. + * + * @return buffer required by the HTML Velocity templates. + */ + @Deprecated(forRemoval = true) + ByteBuffer footerBuffer(); + + /** + * Return theData wrapped in a ByteBuffer class. + * + * @return buffer required by the HTML Velocity templates. + */ + @Deprecated(forRemoval = true) + ByteBuffer dataBuffer(); + + /** + * Get the font encoding string + * + * @return string name of font encoding for the data + */ + String getFontEncoding(); + + /** + * Set the font encoding string + * + * @param arg1 string name of font encoding for the data + */ + void setFontEncoding(String arg1); + + /** + * Clear all metadata elements + */ + void clearParameters(); + + /** + * Determine if parameter is present + * + * @param key name of metadata element to check + */ + boolean hasParameter(String key); + + /** + * Replace all of the metadata elements with a new set + * + * @param map the new set + */ + void setParameters(Map map); + + /** + * Set a new parameter value, deleting an old one + * + * @param key the name of the element + * @param val the value of the element + */ + void setParameter(String key, Object val); + + /** + * Put a new metadata element into the map + * + * @param key the name of the element + * @param val the value of the element + */ + void putParameter(String key, Object val); + + /** + * Put a collection of parameters into the metadata map + * + * @param m the map of new parameters + */ + void putParameters(Map m); + + /** + * Put a collection of parameters into the metadata map using the specified merge policy + * + * @param m the map of new parameters + * @param policy the merge policy + */ + void putParameters(Map m, MergePolicy policy); + + /** + * Merge a collection of parameters into the metadata map + * + * @param m the map of new parameters + */ + void mergeParameters(Map m); + + /** + * Put a collection of parameters into the metadata map uniquely + * + * @param m the map of new parameters + */ + void putUniqueParameters(Map m); + + /** + * Retrieve a specified metadata element + * + * @param key name of the metadata element + * @return the value or null if no such element + */ + List getParameter(String key); + + /** + * Append data to the specified metadata element + * + * @param key name of the metadata element + * @param value the value to append + */ + void appendParameter(String key, CharSequence value); + + /** + * Append data values to the specified metadata element + * + * @param key name of the metadata element + * @param values the values to append + */ + void appendParameter(String key, Iterable values); + + /** + * Append data to the specified metadata element if it doesn't exist + * + * @param key name of the metadata element + * @param value the value to append + * @return true if the item is added, false if it already exists + */ + boolean appendUniqueParameter(String key, CharSequence value); + + /** + * Retrieve a specified metadata element as a string value + * + * @param key name of the metadata element + * @return the string value or null if no such element + */ + String getStringParameter(String key); + + /** + * Retrieve a specified metadata element as a string value + * + * @param key name of the metadata element + * @param sep the separator for multivalued fields + * @return the string value or null if no such element + */ + String getStringParameter(String key, String sep); + + /** + * Retrieve all the metadata elements of this object + * + * @return map of metadata elements + */ + Map> getParameters(); + + /** + * Retrieve all the metadata elements of this object in a way that is processed for use external to this instance + * + * @return map of metadata elements + */ + Map getCookedParameters(); + + /** + * Retrieve all of the current metadata keys + * + * @return set of charsequence keys + */ + Set getParameterKeys(); + + /** + * Delete the specified metadata element named + * + * @param key the name of the metadata item to delete + * @return the object deleted of null if none + */ + List deleteParameter(String key); + + /** + * Put the FILETYPE parameter + * + * @param arg1 the value to store + */ + void setFileType(String arg1); + + /** + * Set FILETYPE parameter iff empty. + * + * @param arg1 the value of the filetype to set + * @param arg2 the list of things caller considers equal to being empty + * @return true if it was empty and set + * @deprecated Use {@link #setFileType(String)} instead. + */ + @Deprecated(forRemoval = true) + boolean setFileTypeIfEmpty(String arg1, String[] arg2); + + /** + * Set FILETYPE parameter iff empty using the built-in definition of empty + * + * @param arg1 the value of the filetype to set + * @return true if it was empty and set + */ + boolean setFileTypeIfEmpty(String arg1); + + /** + * Return true if the file type is null or in one of the "don't care" set + * + * @since 3.3.3 + */ + boolean isFileTypeEmpty(); + + /** + * Get the FILETYPE parameter + * + * @return the string value of the FILETYPE parameter + */ + String getFileType(); + + /** + * Disclose how many multipart alternative views of the data exist + * + * @return count of alternate views + */ + int getNumAlternateViews(); + + /** + * Return a specified multipart alternative view of the data + * + * @param arg1 the name of the view to retrieve + * @return byte array of alternate view data + */ + byte[] getAlternateView(String arg1); + + /** + * Return a specified multipart alternative view of the data in a buffer + * + * @param arg1 the name of the view to retrieve + * @return buffer of alternate view data + */ + @Deprecated(forRemoval = true) + ByteBuffer getAlternateViewBuffer(String arg1); + + /** + * Add a multipart alternative view of the data + * + * @param name the name of the new view + * @param data the byte array of data for the view + */ + void addAlternateView(String name, byte[] data); + + /** + * Add a multipart alternative view of the data + * + * @param name the name of the new view + * @param data the byte array conatining data for the view + * @param offset index of the first byte to use + * @param length number of bytes to use + */ + void addAlternateView(String name, byte[] data, int offset, int length); + + /** + * Append the specified data to the alternate view + * + * @param name the name of the new view + * @param data the byte array of data for the view + */ + void appendAlternateView(String name, byte[] data); + + /** + * Append to a multipart alternative view of the data + * + * @param name the name of the view + * @param data the byte array conatining data for the view + * @param offset index of the first byte to use + * @param length number of bytes to use + */ + void appendAlternateView(String name, byte[] data, int offset, int length); + + /** + * Get the set of alt view names for new foreach loops + * + * @return set of alternate view names + */ + Set getAlternateViewNames(); + + /** + * Get the alternate view map. + * + * @return map of alternate views, key = String, value = byte[] + */ + Map getAlternateViews(); + + /** + * Test for broken document + * + * @return true if broken + */ + boolean isBroken(); + + /** + * Set brokenness for document + * + * @param arg1 the message to record + */ + void setBroken(String arg1); + + /** + * Get brokenness indicator message + * + * @return string message of what is broken + */ + String getBroken(); + + /** + * Returns the name of the file without the path with which the file will be written. + * + * @return the short name of the file (no path) + */ + String shortName(); + + /** + * Returns the filename associated with the data. + * + * @return the string name with path + */ + String getFilename(); + + /** + * Returns the internally generated identifier used to track the object + * + * @return a String representing the internal ID + */ + UUID getInternalId(); + + /** + * Set the filename + * + * @param f the new name of the data including path + */ + void setFilename(String f); + + + /** + * Return the current form of the data (top of the stack) + * + * @return string value of current form + */ + String currentForm(); + + /** + * Return the current form at specified position of the list + * + * @param i The specified position + * @return String containing the form or empty string if illegal position + */ + String currentFormAt(int i); + + /** + * Check to see if this value is already on the stack of itinerary items + * + * @param val the string to look for + * @return the position where it was found or -1 + */ + int searchCurrentForm(String val); + + /** + * Check to see one of these values is on the stack of itinerary items + * + * @param values the List of strings to look for + * @return the String that was found out of the list sent in or null + */ + String searchCurrentForm(Collection values); + + /** + * Get the size of the itinerary stack + * + * @return size of form stack + */ + int currentFormSize(); + + /** + * Remove a form from the head of the list + * + * @return the new size of the itinerary stack + */ + String popCurrentForm(); + + /** + * Replace all current forms with specified + * + * @param form the new current form or null if none desired + */ + void replaceCurrentForm(String form); + + /** + * Remove a form where ever it appears in the stack + * + * @param form the value to remove + * @return the number of elements removed from the stack + */ + int deleteCurrentForm(String form); + + /** + * Remove a form at the specified location of the itinerary stack + * + * @param i the position to delete + * @return the new size of the itinerary stack + */ + int deleteCurrentFormAt(int i); + + /** + * Add current form newForm at idx + * + * @param i the position to do the insert + * @param val the value to insert + * @return size of the new stack + */ + int addCurrentFormAt(int i, String val); + + /** + * Add a form to the end of the list (the bottom of the stack) + * + * @param val the new value to add to the tail of the stack + * @return the new size of the itinerary stack + */ + int enqueueCurrentForm(String val); + + /** + * Push a form onto the head of the list + * + * @param val the new value to push on the stack + * @return the new size of the itinerary stack + */ + int pushCurrentForm(String val); + + /** + * Replaces the current form of the data with a new form Does a pop() followed by a push(newForm) to simulate what would + * happen in the old "one form at a time system" + * + * @param val value of the the new form of the data + */ + void setCurrentForm(String val); + + /** + * Replaces the current form of the data with a form passed and potentially clears the entire form stack + * + * @param val value of the the new form of the data + * @param clearAllForms whether or not to clear the entire form stack + */ + void setCurrentForm(String val, boolean clearAllForms); + + /** + * Return a clone the whole current form list Note this is not a reference to our private store + * + * @return ordered list of current forms + */ + List getAllCurrentForms(); + + /** + * Move curForm to the top of the stack pushing everything above it down one slot + * + * @param curForm the form to pull to the top + */ + void pullFormToTop(String curForm); + + /** + * Return BaseDataObjects info as a String. + * + * @return string value of this object + */ + @Override + String toString(); + + + /** + * Record a processing error + * + * @param val the new error message to record + */ + void addProcessingError(String val); + + /** + * Retrieve the processing error(s) + * + * @return string value of processing errors + */ + String getProcessingError(); + + /** + * Replace history with the new history + * + * @param history of new history strings to use + */ + void setHistory(TransformHistory history); + + /** + * Get the transform history + * + * @return history of places visited + */ + TransformHistory getTransformHistory(); + + /** + * List of places the data object was carried to. + * + * @return List of strings making up the history + */ + List transformHistory(); + + /** + * List of places the data object was carried to. + * + * @param includeCoordinated include the places that were coordinated + * @return List of strings making up the history + */ + List transformHistory(boolean includeCoordinated); + + /** + * Clear the transformation history + */ + void clearTransformHistory(); + + /** + * Appends the new key to the transform history. This is called by MobileAgent before moving to the new place. It + * usually adds the four-tuple of a place's key + * + * @see MobileAgent#agentControl + * @param key the new value to append + */ + void appendTransformHistory(String key); + + /** + * Appends the new key to the transform history. This is called by MobileAgent before moving to the new place. It + * usually adds the four-tuple of a place's key. Coordinated history keys are meant for informational purposes and have + * no bearing on the routing algorithm. It is important to list the places visited in coordination, but should not + * report as the last place visited. + * + * @see MobileAgent#agentControl + * @param key the new value to append + * @param coordinated true if history entries are for informational purposes only + */ + void appendTransformHistory(String key, boolean coordinated); + + /** + * Return what machine we are located on + * + * @return string local host name + */ + String whereAmI(); + + /** + * Return an SDE based on the last item in the transform history or null if empty + * + * @return last item in history + */ + DirectoryEntry getLastPlaceVisited(); + + /** + * Return an SDE based on the penultimate item in the transform history or null if empty + * + * @return penultimate item in history + */ + DirectoryEntry getPenultimatePlaceVisited(); + + /** + * Return true if the payload has been to a place matching the key passed in. + * + * @param pattern the key pattern to match + */ + boolean hasVisited(String pattern); + + /** + * True if this payload hasn't had any processing yet Does not count parent processing as being for this payload + * + * @return true if not yet started + */ + boolean beforeStart(); + + /** + * Print the parameters, nicely formatted + */ + String printMeta(); + + /** + * Get data object's priority. + * + * @return int priority (lower the number, higher the priority). + */ + int getPriority(); + + /** + * Set the data object's priority, typically based on input dir/file priority. + * + * @param priority int (lower the number, higher the priority). + */ + void setPriority(int priority); + + /** + * Get the timestamp for when the object was created. This attribute will be used for data provenance. + * + * @return date - the timestamp the object was created + */ + Instant getCreationTimestamp(); + + /** + * Set the timestamp for when the object was created + * + * @param creationTimestamp - the date the object was created + */ + void setCreationTimestamp(Instant creationTimestamp); + + /** + * Test if tree is outputable + * + * @return true if this tree is not able to be output, false otherwise + */ + boolean isOutputable(); + + /** + * Set whether or not the tree is able to be written out + * + * @param outputable true if this tree is not able to be output, false otherwise + */ + void setOutputable(boolean outputable); + + /** + * Get ID + * + * @return the unique identifier of the IBaseDataObject + */ + String getId(); + + /** + * Set the unique identifier of the IBaseDataObject + * + * @param id the unique identifier of the IBaseDataObject + */ + void setId(String id); + + /** + * Get the Work Bundle ID + * + * @return the unique identifier of the {@link emissary.pickup.WorkBundle} + */ + String getWorkBundleId(); + + /** + * Set the unique identifier of the {@link emissary.pickup.WorkBundle} + * + * @param workBundleId the unique identifier of the {@link emissary.pickup.WorkBundle} + */ + void setWorkBundleId(String workBundleId); + + /** + * Get the Transaction ID + * + * @return the unique identifier of the transaction + */ + String getTransactionId(); + + /** + * Set the unique identifier of the transaction + * + * @param transactionId the unique identifier of the transaction + */ + void setTransactionId(String transactionId); +} diff --git a/src/main/java/emissary/core/IExtractedRecord.java b/src/main/java/emissary/core/IExtractedRecord.java new file mode 100755 index 0000000000..b28e719f2a --- /dev/null +++ b/src/main/java/emissary/core/IExtractedRecord.java @@ -0,0 +1,6 @@ +package emissary.core; + + +public interface IExtractedRecord extends IBaseRecord { + +} diff --git a/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java b/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java index 5ceba35feb..0c86d91a2b 100644 --- a/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java +++ b/src/main/java/emissary/core/channels/SeekableByteChannelHelper.java @@ -1,6 +1,6 @@ package emissary.core.channels; -import emissary.core.IBaseDataObject; +import emissary.core.IBaseRecord; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.Validate; @@ -81,7 +81,7 @@ public static SeekableByteChannelFactory inputStream(final long size, final Inpu * @param maxSize to limit the byte array to * @return a byte array of the data from the BDO sized up to maxSize (so could truncate data) */ - public static byte[] getByteArrayFromBdo(final IBaseDataObject ibdo, final int maxSize) { + public static byte[] getByteArrayFromBdo(final IBaseRecord ibdo, final int maxSize) { try (final SeekableByteChannel sbc = ibdo.getChannelFactory().create()) { final long truncatedBy = sbc.size() - maxSize; if (truncatedBy > 0 && logger.isWarnEnabled()) { diff --git a/src/main/java/emissary/kff/KffDataObjectHandler.java b/src/main/java/emissary/kff/KffDataObjectHandler.java index 4e90a11b0a..7baffc9c50 100755 --- a/src/main/java/emissary/kff/KffDataObjectHandler.java +++ b/src/main/java/emissary/kff/KffDataObjectHandler.java @@ -1,7 +1,7 @@ package emissary.kff; import emissary.core.IBaseDataObject; -import emissary.core.IBaseDataObject.MergePolicy; +import emissary.core.IBaseRecord.MergePolicy; import emissary.core.channels.SeekableByteChannelFactory; import org.apache.commons.lang3.StringUtils; diff --git a/src/main/java/emissary/util/PayloadUtil.java b/src/main/java/emissary/util/PayloadUtil.java index e164999a73..ee9c4448c0 100755 --- a/src/main/java/emissary/util/PayloadUtil.java +++ b/src/main/java/emissary/util/PayloadUtil.java @@ -4,6 +4,7 @@ import emissary.config.Configurator; import emissary.core.Family; import emissary.core.IBaseDataObject; +import emissary.core.IBaseRecord; import emissary.core.TransformHistory; import emissary.util.xml.JDOMUtil; @@ -290,7 +291,7 @@ public static String toXmlString(final List list) { */ private static final String SEP = ": "; - public static String printFormattedMetadata(final IBaseDataObject payload) { + public static String printFormattedMetadata(final IBaseRecord payload) { final StringBuilder out = new StringBuilder(); out.append(LS); for (final Map.Entry> entry : payload.getParameters().entrySet()) { diff --git a/src/test/java/emissary/core/BaseDataObjectTest.java b/src/test/java/emissary/core/BaseDataObjectTest.java index f42de51d05..ae69089b7f 100755 --- a/src/test/java/emissary/core/BaseDataObjectTest.java +++ b/src/test/java/emissary/core/BaseDataObjectTest.java @@ -24,7 +24,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; @@ -1168,8 +1167,7 @@ void testBothDataFieldsHaveValue() final BaseDataObject bdo = new BaseDataObject(); final String testData = "This is a test"; bdo.setChannelFactory(SeekableByteChannelHelper.memory(testData.getBytes())); - Field theData = bdo.getClass().getDeclaredField("theData"); - theData.set(bdo, testData.getBytes()); + bdo.theData = testData.getBytes(); final String msg = "Should throw an error when trying to access data on a BDO where we have a byte array and a channel";