From c43513e0dd1d34b90816ac660eb40130fad464b4 Mon Sep 17 00:00:00 2001
From: Stelios Voutsinas <steliosvoutsinas@yahoo.com>
Date: Tue, 7 Jan 2025 18:00:36 -0700
Subject: [PATCH 1/3] Add ResultSetWriter and enable for B2 VOTable
---
build.gradle | 6 +-
...250114_162646_steliosvoutsinas_DM_48132.md | 13 +
.../tap/impl/MaxRecValidatorImpl.java | 1 -
.../opencadc/tap/impl/QServQueryRunner.java | 4 +-
.../opencadc/tap/impl/ResultSetWriter.java | 717 ++++++++++++++++++
.../opencadc/tap/impl/RubinFormatFactory.java | 2 +-
.../opencadc/tap/impl/RubinTableWriter.java | 218 ++++--
7 files changed, 911 insertions(+), 50 deletions(-)
create mode 100644 changelog.d/20250114_162646_steliosvoutsinas_DM_48132.md
create mode 100644 src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
diff --git a/build.gradle b/build.gradle
index da75478..0304cec 100644
--- a/build.gradle
+++ b/build.gradle
@@ -36,7 +36,7 @@ war {
configurations {
intTestCompile.extendsFrom testImplementation
intTestRuntime.extendsFrom testRuntimeOnly
- implementation.exclude group: 'uk.ac.starlink'
+ //implementation.exclude group: 'uk.ac.starlink'
}
dependencies {
@@ -56,6 +56,10 @@ dependencies {
implementation 'org.opencadc:cadc-uws:1.0.5'
implementation 'org.opencadc:cadc-uws-server:1.2.21'
implementation 'org.opencadc:cadc-vosi:1.4.6'
+ implementation 'uk.ac.starlink:stil:[4.0,5.0)'
+
+ implementation 'uk.ac.starlink:jcdf:[1.2.3,2.0)'
+ implementation 'uk.ac.starlink:stil:[4.0,5.0)'
// Switch out this to use any supported database instead of PostgreSQL.
// ## START CUSTOM DATABASE ##
diff --git a/changelog.d/20250114_162646_steliosvoutsinas_DM_48132.md b/changelog.d/20250114_162646_steliosvoutsinas_DM_48132.md
new file mode 100644
index 0000000..d0559b5
--- /dev/null
+++ b/changelog.d/20250114_162646_steliosvoutsinas_DM_48132.md
@@ -0,0 +1,13 @@
+<!-- Delete the sections that don't apply -->
+
+### Added
+
+- ResultSetWriter class, used to write out Binary2 VOTable serialization
+
+### Changed
+
+- Added new Binary2 VOTable serialization, make default. Retain tabledata as an optional format
+
+### Fixed
+
+- Correctly print field metadata
diff --git a/src/main/java/org/opencadc/tap/impl/MaxRecValidatorImpl.java b/src/main/java/org/opencadc/tap/impl/MaxRecValidatorImpl.java
index 6380f91..e4ab9a8 100644
--- a/src/main/java/org/opencadc/tap/impl/MaxRecValidatorImpl.java
+++ b/src/main/java/org/opencadc/tap/impl/MaxRecValidatorImpl.java
@@ -93,7 +93,6 @@ public MaxRecValidatorImpl() {
@Override
public Integer validate() {
- LOGGER.info("");
if (super.sync) {
try {
// no limits on sync
diff --git a/src/main/java/org/opencadc/tap/impl/QServQueryRunner.java b/src/main/java/org/opencadc/tap/impl/QServQueryRunner.java
index 4140d4b..f472508 100644
--- a/src/main/java/org/opencadc/tap/impl/QServQueryRunner.java
+++ b/src/main/java/org/opencadc/tap/impl/QServQueryRunner.java
@@ -371,7 +371,7 @@ private void doIt()
// and restrict to forward only so that client memory usage is minimal since
// we are only interested in reading the ResultSet once
- connection.setAutoCommit(false);
+ connection.setAutoCommit(false);
pstmt = connection.prepareStatement(sql);
pstmt.setFetchSize(1000);
pstmt.setFetchDirection(ResultSet.FETCH_FORWARD);
@@ -397,7 +397,7 @@ private void doIt()
tableWriter.write(resultSet, syncOutput.getOutputStream());
else
tableWriter.write(resultSet, syncOutput.getOutputStream(), maxRows.longValue());
-
+
t2 = System.currentTimeMillis(); dt = t2 - t1; t1 = t2;
diagnostics.add(new Result("diag", URI.create("query:stream:"+dt)));
}
diff --git a/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java b/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
new file mode 100644
index 0000000..b4ab6c5
--- /dev/null
+++ b/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
@@ -0,0 +1,717 @@
+package org.opencadc.tap.impl;
+
+import ca.nrc.cadc.dali.tables.TableWriter;
+import ca.nrc.cadc.dali.tables.votable.GroupElement;
+import ca.nrc.cadc.dali.tables.votable.ParamElement;
+import ca.nrc.cadc.dali.tables.votable.VOTableField;
+import ca.nrc.cadc.dali.tables.votable.VOTableGroup;
+import ca.nrc.cadc.dali.tables.votable.VOTableInfo;
+import ca.nrc.cadc.dali.tables.votable.VOTableParam;
+import ca.nrc.cadc.dali.tables.votable.VOTableResource;
+import ca.nrc.cadc.dali.util.Format;
+import ca.nrc.cadc.dali.util.FormatFactory;
+import ca.nrc.cadc.xml.ContentConverter;
+import ca.nrc.cadc.xml.IterableContent;
+import ca.nrc.cadc.xml.MaxIterations;
+import uk.ac.starlink.table.ColumnInfo;
+import uk.ac.starlink.table.DescribedValue;
+import uk.ac.starlink.table.RowSequence;
+import uk.ac.starlink.table.WrapperRowSequence;
+import uk.ac.starlink.table.jdbc.SequentialResultSetStarTable;
+import uk.ac.starlink.votable.DataFormat;
+import uk.ac.starlink.votable.VOSerializer;
+import uk.ac.starlink.votable.VOTableVersion;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.jdom2.Document;
+import org.jdom2.Element;
+import org.jdom2.Namespace;
+import org.jdom2.output.XMLOutputter;
+
+/**
+ * ResultSet to VOTable writer.
+ *
+ * @author stvoutsin
+ */
+public class ResultSetWriter implements TableWriter<ResultSet> {
+
+ private static final Logger log = Logger.getLogger(ResultSetWriter.class);
+ private static final String BASE_URL = System.getProperty("base_url");
+
+ public static final String CONTENT_TYPE = "application/x-votable+xml";
+ public static final String CONTENT_TYPE_ALT = "text/xml";
+
+ // VOTable Version number.
+ public static final String VOTABLE_VERSION = "1.4";
+
+ // Uri to the XML schema.
+ public static final String XSI_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance";
+
+ // Uri to the VOTable schema.
+ public static final String VOTABLE_11_NS_URI = "http://www.ivoa.net/xml/VOTable/v1.1";
+ public static final String VOTABLE_12_NS_URI = "http://www.ivoa.net/xml/VOTable/v1.2";
+ public static final String VOTABLE_13_NS_URI = "http://www.ivoa.net/xml/VOTable/v1.3";
+ public static final String VOTABLE_14_NS_URI = "http://www.ivoa.net/xml/VOTable/v1.4";
+
+ private FormatFactory formatFactory;
+ private String mimeType;
+ private DataFormat dfmt_;
+ private VOTableVersion version_;
+ private long maxrec_;
+ private List<VOTableInfo> infos;
+ private List<VOTableResource> resources;
+ private ColumnInfo[] columns;
+ private long totalRows = 0;
+
+ /**
+ * Get the total number of rows processed.
+ *
+ * @return the number of rows written
+ */
+ public long getRowCount() {
+ return totalRows;
+ }
+
+ /**
+ * Default constructor.
+ */
+ public ResultSetWriter() {
+ this(null, uk.ac.starlink.votable.DataFormat.BINARY2, VOTableVersion.V14, -1);
+ }
+
+
+ /**
+ * Constructor.
+ *
+ * @param mimeType selects the mimetype string
+ * @param dfmt selects VOTable serialization format
+ * (TABLEDATA, BINARY, BINARY2, FITS)
+ * @param version selects VOTable version
+ * @param maxrec maximum record count before overflow;
+ * negative value means no limit
+ */
+ public ResultSetWriter( String mimeType, DataFormat dfmt, VOTableVersion version, long maxrec ) {
+ this.mimeType = mimeType;
+ this.dfmt_ = dfmt;
+ this.version_ = version;
+ this.maxrec_ = maxrec;
+ this.infos = new ArrayList<>();
+ this.resources = new ArrayList<>();
+ this.columns = null;
+ }
+
+
+ /**
+ * Get the Content-Type for the VOTable.
+ *
+ * @return VOTable Content-Type.
+ */
+ @Override
+ public String getContentType() {
+ if (mimeType == null) {
+ return CONTENT_TYPE;
+ }
+
+ return mimeType;
+ }
+
+ public List<VOTableResource> getResources() {
+ return resources;
+ }
+
+ /**
+ * Get the list of VOTable infos.
+ *
+ * @return List of VOTableInfo objects
+ */
+ public List<VOTableInfo> getInfos() {
+ return infos;
+ }
+
+ public ColumnInfo[] getColumns() {
+ return columns;
+ }
+
+ /**
+ * Set the list of VOTable infos.
+ *
+ * @param infos List of VOTableInfo objects
+ */
+ public void setInfos(List<VOTableInfo> infos) {
+ this.infos = infos;
+ }
+
+ public void setColumns(ColumnInfo[] columns) {
+ this.columns = columns;
+ }
+
+
+ /**
+ * Set the resources of the VOTable.
+ *
+ * @param resources List of VOTableResource objects
+ */
+ public void setResources(List<VOTableResource> resources) {
+ this.resources = resources;
+ }
+
+ /**
+ * Add a single VOTable info.
+ *
+ * @param info VOTableInfo object to add
+ */
+ public void addInfo(VOTableInfo info) {
+ if (info != null) {
+ this.infos.add(info);
+ }
+ }
+
+
+ /**
+ * Add a single VOTableResource object.
+ *
+ * @param resource VOTableResource object to add
+ */
+ public void addResource(VOTableResource resource) {
+ if (resource != null) {
+ this.resources.add(resource);
+ }
+ }
+ /**
+ * Get error content type
+ */
+ public String getErrorContentType() {
+ return getContentType();
+ }
+
+ /**
+ * Get the extension for the VOTable.
+ *
+ * @return VOTable extension.
+ */
+ @Override
+ public String getExtension() {
+ return "xml";
+ }
+
+ @Override
+ public void setFormatFactory(FormatFactory formatFactory) {
+ this.formatFactory = formatFactory;
+ }
+
+ /**
+ * Write the ResultSet to the specified OutputStream.
+ *
+ * @param resultSet ResultSet object to write.
+ * @param ostream OutputStream to write to.
+ * @throws IOException if problem writing to OutputStream.
+ */
+ @Override
+ public void write(ResultSet resultSet, OutputStream ostream)
+ throws IOException {
+ write(resultSet, ostream, Long.MAX_VALUE);
+ }
+
+ /**
+ * Write the ResultSet to the specified OutputStream, only writing maxrec rows.
+ * If the VOTable contains more than maxrec rows, appends an INFO element with
+ * name="QUERY_STATUS" value="OVERFLOW" to the VOTable.
+ *
+ * @param resultSet ResultSet object to write.
+ * @param ostream OutputStream to write to.
+ * @param maxrec maximum number of rows to write.
+ * @throws IOException if problem writing to OutputStream.
+ */
+ @Override
+ public void write(ResultSet resultSet, OutputStream ostream, Long maxrec)
+ throws IOException {
+ Writer writer = new BufferedWriter(new OutputStreamWriter(ostream, "UTF-8"));
+ write(resultSet, writer, maxrec);
+ }
+
+ /**
+ * Write the ResultSet to the specified Writer.
+ *
+ * @param resultSet ResultSet object to write.
+ * @param writer Writer to write to.
+ * @throws IOException if problem writing to the writer.
+ */
+ @Override
+ public void write(ResultSet resultSet, Writer writer)
+ throws IOException {
+ write(resultSet, writer, Long.MAX_VALUE);
+ }
+
+ /**
+ * Write the ResultSet to the specified Writer, only writing maxrec rows.
+ * If the ResultSet contains more than maxrec rows, appends an INFO element with
+ * name="QUERY_STATUS" value="OVERFLOW" to the VOTable.
+ *
+ * @param resultSet ResultSet object to write.
+ * @param writer Writer to write to.
+ * @param maxrec maximum number of rows to write.
+ * @throws IOException if problem writing to the writer.
+ */
+ @Override
+ public void write(ResultSet resultSet, Writer writer, Long maxrec)
+ throws IOException {
+ if (formatFactory == null) {
+ this.formatFactory = new FormatFactory();
+ }
+ writeImpl(resultSet, writer, maxrec);
+ }
+
+ /**
+ * Write the Throwable to a VOTable, creating an INFO element with
+ * name="QUERY_STATUS" value="ERROR" and setting the stacktrace as
+ * the INFO text.
+ *
+ * @param thrown Throwable to write.
+ * @param output OutputStream to write to.
+ * @throws IOException if problem writing to the stream.
+ */
+ public void write(Throwable thrown, OutputStream output)
+ throws IOException {
+ Document document = createDocument();
+ Element root = document.getRootElement();
+ Namespace namespace = root.getNamespace();
+
+ // Create the RESOURCE element and add to the VOTABLE element.
+ Element resource = new Element("RESOURCE", namespace);
+ resource.setAttribute("type", "results");
+ root.addContent(resource);
+
+ // Create the INFO element and add to the RESOURCE element.
+ Element info = new Element("INFO", namespace);
+ info.setAttribute("name", "QUERY_STATUS");
+ info.setAttribute("value", "ERROR");
+ info.setText(getThrownExceptions(thrown));
+ resource.addContent(info);
+
+ // Write out the VOTABLE.
+ XMLOutputter outputter = new XMLOutputter();
+ outputter.setFormat(org.jdom2.output.Format.getPrettyFormat());
+ outputter.output(document, output);
+ }
+
+ /**
+ * Writes a result set to an output stream as a VOTable.
+ *
+ * @param resultSet ResultSet
+ * @param writer Writer
+ * @param maxrec Maximum number of rows
+ */
+ protected void writeImpl(ResultSet resultSet, Writer writer, Long maxrec)
+ throws IOException {
+ log.debug("write, maxrec=" + maxrec);
+
+
+ try (BufferedWriter out = (writer instanceof BufferedWriter)
+ ? (BufferedWriter) writer
+ : new BufferedWriter(writer)) {
+
+ LimitedResultSetStarTable table;
+ try {
+ table = new LimitedResultSetStarTable(this.getColumns(), resultSet, maxrec);
+ } catch (SQLException e) {
+ throw new IOException("Error processing ResultSet: " + e.getMessage(), e);
+ }
+
+ /* Prepares the object that will do the serialization work. */
+ VOSerializer voser =
+ VOSerializer.makeSerializer( dfmt_, version_, table );
+
+ /* Write header. */
+ out.write( "<VOTABLE"
+ + VOSerializer.formatAttribute( "version",
+ version_.getVersionNumber() )
+ + VOSerializer.formatAttribute( "xmlns",
+ version_.getXmlNamespace() )
+ + ">" );
+ out.newLine();
+ out.write( "<RESOURCE>" );
+ out.newLine();
+
+ XMLOutputter outputter = new XMLOutputter();
+ outputter.setFormat(org.jdom2.output.Format.getPrettyFormat());
+
+ // Write all info elements
+ for (VOTableInfo info : infos) {
+ Element infoElement = new Element("INFO");
+ infoElement.setAttribute("name", info.getName());
+ infoElement.setAttribute("value", info.getValue());
+ outputter.output(infoElement, out);
+ out.newLine();
+ }
+
+ /* Write table element. */
+ voser.writeInlineTableElement( out );
+
+ /* Check for overflow and write INFO if required. */
+ if ( table.lastSequenceOverflowed() ) {
+ out.write( "<INFO name='QUERY_STATUS' value='OVERFLOW'/>" );
+ out.newLine();
+ }
+
+ /* Write footer. */
+ out.write( "</RESOURCE>" );
+ out.newLine();
+ for (VOTableResource resource : resources) {
+ Element r = createResource(resource, null);
+ outputter.output(r, out);
+ out.newLine();
+ }
+ out.write( "</VOTABLE>" );
+ out.newLine();
+ out.flush();
+
+ this.totalRows = table.getTotalRows();
+
+ }
+ }
+
+ /**
+ * StarTable implementation which is based on a ResultSet, and which
+ * is limited to a fixed number of rows when its row iterator is used.
+ * Note this implementation is OK for one-pass table output handlers
+ * like VOTable, but won't work for ones which require two passes
+ * such as FITS (which needs row count up front).
+ */
+ private static class LimitedResultSetStarTable
+ extends SequentialResultSetStarTable {
+
+ public static final String TABLE_NAME_INFO = "TABLE_NAME";
+ private final long maxrec_;
+ private boolean overflow_;
+ private long totalRows = 0;
+ private ColumnInfo[] columnInfos;
+
+ /**
+ * Constructor.
+ *
+ * @param rset result set supplying the data
+ * @param maxrec maximum number of rows that will be iterated over;
+ * negative value means no limit
+ */
+ LimitedResultSetStarTable(ColumnInfo[] colInfos, ResultSet rset, long maxrec )
+ throws SQLException {
+ super( rset );
+ maxrec_ = maxrec;
+ columnInfos = colInfos;
+ }
+
+ /**
+ * Get the row count
+ */
+ public long getTotalRows() {
+ return totalRows;
+ }
+
+ /**
+ * Provides column information for a specified index.
+ *
+ * @param colInd index of the column to describe
+ * @return metadata object describing this column
+ */
+ @Override
+ public ColumnInfo getColumnInfo(final int colInd) {
+ return columnInfos[colInd];
+ }
+
+ /**
+ * Indicates whether the last row sequence dispensed by
+ * this table's getRowSequence method was truncated at maxrec rows.
+ *
+ * @return true iff the last row sequence overflowed
+ */
+ public boolean lastSequenceOverflowed() {
+ return overflow_;
+ }
+
+ /**
+ * A RowSequence wrapper that modifies obscore access URLs in result sets.
+ * This class intercepts access_url column values and rewrites them to use a different base URL
+ *
+ * @throws RuntimeException if URL rewriting fails due to malformed URLs
+ */
+ private static class ModifiedLimitRowSequence extends WrapperRowSequence {
+ private final Map<Integer, Boolean> accessUrlColumns;
+
+ ModifiedLimitRowSequence(RowSequence baseSeq, ColumnInfo[] columnInfos) {
+ super(baseSeq);
+ this.accessUrlColumns = findAccessUrlColumns(columnInfos);
+ }
+
+ /**
+ * Find all access_url columns and determine if they belong to ivoa.ObsCore.
+ * Returns a map of column indices to boolean indicating if they should be modified.
+ */
+ private static Map<Integer, Boolean> findAccessUrlColumns(ColumnInfo[] columnInfos) {
+ Map<Integer, Boolean> columns = new HashMap<>();
+ try {
+ for (int i = 0; i < columnInfos.length; i++) {
+ ColumnInfo info = columnInfos[i];
+
+ DescribedValue tableNameValue = info.getAuxDatumByName(TABLE_NAME_INFO);
+ if (tableNameValue != null) {
+ String tableName = tableNameValue.getValue().toString();
+
+ if ("access_url".equals(info.getName())) {
+ boolean isObsCore = "ivoa.ObsCore".equals(tableName);
+ columns.put(i, isObsCore);
+ log.debug("Found access_url column at index " + i +
+ " for table " + tableName +
+ (isObsCore ? " (will modify URLs)" : " (won't modify URLs)"));
+ }
+ }
+ }
+ } catch (NullPointerException ex) {
+ log.debug("Error while trying to find access_url columns");
+ }
+
+ return columns;
+ }
+
+ /**
+ * Returns the value from a single cell, modifying it if it's a URL that needs to be rewritten.
+ *
+ * @param icol the column index
+ * @return the cell contents
+ */
+ @Override
+ public Object getCell(int icol) throws IOException {
+ Object value = super.getCell(icol);
+ if (shouldModifyUrl(icol, value)) {
+ return modifyAccessUrl((String) value);
+ }
+ return value;
+ }
+
+ /**
+ * Returns values from an entire row, modifying any URLs that need to be rewritten.
+ *
+ * @return array containing values for the current row
+ */
+ @Override
+ public Object[] getRow() throws IOException {
+ Object[] row = super.getRow();
+ for (Map.Entry<Integer, Boolean> entry : accessUrlColumns.entrySet()) {
+ int index = entry.getKey();
+ if (index >= 0 && index < row.length &&
+ shouldModifyUrl(index, row[index])) {
+ row[index] = modifyAccessUrl((String) row[index]);
+ }
+ }
+ return row;
+ }
+
+ /**
+ * Determines whether a value at a given index should have its URL modified.
+ *
+ * @param columnIndex index of the column containing the value
+ * @param value the value to check
+ * @return true if the value should have its URL modified
+ */
+ private boolean shouldModifyUrl(int columnIndex, Object value) {
+ return value instanceof String &&
+ accessUrlColumns.containsKey(columnIndex) &&
+ accessUrlColumns.get(columnIndex) &&
+ value != null;
+ }
+
+ /**
+ * Modifies a URL to use the base URL while preserving the path.
+ *
+ * @param url the original URL to modify, or null
+ * @return the modified URL, or null if the input was null
+ * @throws RuntimeException if URL parsing or rewriting fails
+ */
+ private String modifyAccessUrl(String url) {
+ if (url != null) {
+ String s = (String) url;
+ try {
+ URL orig = new URL(s);
+ URL base_url = new URL(BASE_URL);
+ URL rewritten = new URL(orig.getProtocol(), base_url.getHost(), orig.getFile());
+ log.debug( "Rewritten URL: " + rewritten.toExternalForm());
+
+ return rewritten.toExternalForm();
+ } catch (MalformedURLException ex) {
+ throw new RuntimeException("BUG: Failed to rewrite URL: " + s, ex);
+ }
+ }
+ return url;
+ }
+ }
+
+ /**
+ * Returns a row sequence that may be limited to a maximum number of rows.
+ * Creates a sequence that wraps the underlying ResultSet, applies URL modifications
+ * if needed, and enforces any row limit specified by maxrec_.
+ *
+ * @return a row sequence, possibly truncated at maxrec_ rows
+ * @throws IOException if there is an error reading from the base sequence
+ */
+ @Override
+ public RowSequence getRowSequence() throws IOException {
+ overflow_ = false;
+ totalRows = 0;
+ RowSequence baseSeq = super.getRowSequence();
+ baseSeq = new ModifiedLimitRowSequence(baseSeq, columnInfos);
+
+ if (maxrec_ < 0 || maxrec_ == Long.MAX_VALUE) {
+ return new WrapperRowSequence(baseSeq) {
+ @Override
+ public boolean next() throws IOException {
+ boolean hasNext = super.next();
+ if (hasNext) {
+ totalRows++;
+ }
+ return hasNext;
+ }
+ };
+ } else {
+ return new WrapperRowSequence( baseSeq ) {
+ long irow = -1;
+ @Override
+ public boolean next() throws IOException {
+ irow++;
+ if ( irow < maxrec_ ) {
+ boolean hasNext = super.next();
+ if (hasNext) {
+ totalRows++;
+ }
+ return hasNext;
+ }
+ if ( irow == maxrec_ ) {
+ log.debug("Overflow reached while processing rows!");
+ overflow_ = super.next();
+ }
+ return false;
+ }
+ };
+ }
+ }
+ }
+
+ /**
+ * Creates a JDOM Element representing a VOTable RESOURCE from a VOTableResource object.
+ * Converts the VOTableResource into XML format, including all its attributes, description,
+ * INFO elements, parameters, and groups.
+ *
+ * @param votResource the VOTableResource to convert
+ * @param namespace the XML namespace to use, may be null
+ * @return JDOM Element containing the XML representation of the resource
+ */
+ private Element createResource(VOTableResource votResource, Namespace namespace) {
+ // Create the RESOURCE element and add to the VOTABLE element.
+ Element resource = new Element("RESOURCE", namespace);
+
+ resource.setAttribute("type", votResource.getType());
+ log.debug("wrote resource.type: " + votResource.getType());
+
+ if (votResource.id != null) {
+ resource.setAttribute("ID", votResource.id);
+ }
+
+ if (votResource.getName() != null) {
+ resource.setAttribute("name", votResource.getName());
+ }
+
+ if (votResource.utype != null) {
+ resource.setAttribute("utype", votResource.utype);
+ }
+
+ // Create the DESCRIPTION element and add to RESOURCE element.
+ if (votResource.description != null) {
+ Element description = new Element("DESCRIPTION", namespace);
+ description.setText(votResource.description);
+ resource.addContent(description);
+ }
+
+ // Create the INFO element and add to the RESOURCE element.
+ for (VOTableInfo in : votResource.getInfos()) {
+ Element info = new Element("INFO", namespace);
+ info.setAttribute("name", in.getName());
+ info.setAttribute("value", in.getValue());
+ if (in.content != null) {
+ info.setText(in.content);
+ }
+ resource.addContent(info);
+ }
+ log.debug("wrote resource.info: " + votResource.getInfos().size());
+
+ for (VOTableParam param : votResource.getParams()) {
+ resource.addContent(new ParamElement(param, namespace));
+ }
+ log.debug("wrote resource.param: " + votResource.getParams().size());
+
+ for (VOTableGroup vg : votResource.getGroups()) {
+ resource.addContent(new GroupElement(vg, namespace));
+ }
+
+ return resource;
+ }
+
+ /**
+ * Builds a empty VOTable document with the appropriate namespaces and
+ * attributes.
+ *
+ * @return VOTable document.
+ */
+ protected Document createDocument() {
+ // the root VOTABLE element
+ Namespace vot = Namespace.getNamespace(VOTABLE_14_NS_URI);
+ Namespace xsi = Namespace.getNamespace("xsi", XSI_SCHEMA);
+ Element votable = new Element("VOTABLE", vot);
+ votable.setAttribute("version", VOTABLE_VERSION);
+ votable.addNamespaceDeclaration(xsi);
+
+ Document document = new Document();
+ document.addContent(votable);
+
+ return document;
+ }
+
+ /**
+ * Builds a string containing all exception messages from a throwable and its causes.
+ *
+ * @param thrown the throwable to process
+ * @return space-separated string of all exception messages in the chain
+ */
+ private String getThrownExceptions(Throwable thrown) {
+ StringBuilder sb = new StringBuilder();
+ if (thrown.getMessage() == null) {
+ sb.append("");
+ } else {
+ sb.append(thrown.getMessage());
+ }
+ while (thrown.getCause() != null) {
+ thrown = thrown.getCause();
+ sb.append(" ");
+ if (thrown.getMessage() == null) {
+ sb.append("");
+ } else {
+ sb.append(thrown.getMessage());
+ }
+ }
+ return sb.toString();
+ }
+
+
+}
diff --git a/src/main/java/org/opencadc/tap/impl/RubinFormatFactory.java b/src/main/java/org/opencadc/tap/impl/RubinFormatFactory.java
index 630a2ac..29d4038 100644
--- a/src/main/java/org/opencadc/tap/impl/RubinFormatFactory.java
+++ b/src/main/java/org/opencadc/tap/impl/RubinFormatFactory.java
@@ -23,7 +23,7 @@ public Format<Object> getClobFormat(TapSelectItem columnDesc) {
// ivoa.ObsCore
if ("ivoa.ObsCore".equalsIgnoreCase(columnDesc.tableName)) {
if ("access_url".equalsIgnoreCase(columnDesc.getColumnName())) {
- log.info("getClobFormat called for access_url");
+ log.info("getClobFormat called for access_url");
return new RubinURLFormat();
}
}
diff --git a/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java b/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
index ca6faee..e36005c 100644
--- a/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
+++ b/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
@@ -69,7 +69,6 @@
package org.opencadc.tap.impl;
-import ca.nrc.cadc.auth.AuthMethod;
import ca.nrc.cadc.auth.AuthenticationUtil;
import ca.nrc.cadc.dali.tables.ascii.AsciiTableWriter;
import ca.nrc.cadc.dali.tables.votable.VOTableDocument;
@@ -80,6 +79,8 @@
import ca.nrc.cadc.dali.tables.votable.VOTableResource;
import ca.nrc.cadc.dali.tables.votable.VOTableTable;
import ca.nrc.cadc.dali.tables.votable.VOTableWriter;
+
+import org.opencadc.tap.impl.ResultSetWriter;
import ca.nrc.cadc.dali.util.Format;
import ca.nrc.cadc.date.DateUtil;
import ca.nrc.cadc.reg.client.RegistryClient;
@@ -91,6 +92,10 @@
import ca.nrc.cadc.tap.writer.format.FormatFactory;
import ca.nrc.cadc.uws.Job;
import ca.nrc.cadc.uws.ParameterUtil;
+import uk.ac.starlink.table.ColumnInfo;
+import uk.ac.starlink.table.DefaultValueInfo;
+import uk.ac.starlink.table.DescribedValue;
+import uk.ac.starlink.votable.VOStarTable;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
@@ -123,6 +128,7 @@ public class RubinTableWriter implements TableWriter
{
private static final Logger log = Logger.getLogger(RubinTableWriter.class);
+ public static final String TABLE_NAME_INFO = "TABLE_NAME";
private static final String FORMAT = "RESPONSEFORMAT";
private static final String FORMAT_ALT = "FORMAT";
@@ -134,11 +140,14 @@ public class RubinTableWriter implements TableWriter
public static final String TEXT = "text";
public static final String TSV = "tsv";
public static final String VOTABLE = "votable";
+ public static final String VOTABLE_TD = "votable-td";
public static final String RSS = "rss";
// content-type
// private static final String APPLICATION_FITS = "application/fits";
private static final String APPLICATION_VOTABLE_XML = "application/x-votable+xml";
+ private static final String APPLICATION_VOTABLE_B2 = "application/x-votable+xml;serialization=binary2";
+ private static final String APPLICATION_VOTABLE_TD_XML = "application/x-votable+xml;serialization=tabledata";
private static final String APPLICATION_RSS = "application/rss+xml";
private static final String TEXT_XML_VOTABLE = "text/xml;content=x-votable"; // the SIAv1 mimetype
private static final String TEXT_CSV = "text/csv";
@@ -155,6 +164,8 @@ public class RubinTableWriter implements TableWriter
static
{
knownFormats.put(APPLICATION_VOTABLE_XML, VOTABLE);
+ knownFormats.put(APPLICATION_VOTABLE_B2, VOTABLE);
+ knownFormats.put(APPLICATION_VOTABLE_TD_XML, VOTABLE_TD);
knownFormats.put(TEXT_XML, VOTABLE);
knownFormats.put(TEXT_XML_VOTABLE, VOTABLE);
knownFormats.put(TEXT_CSV, CSV);
@@ -172,8 +183,9 @@ public class RubinTableWriter implements TableWriter
private String extension;
// RssTableWriter not yet ported to cadcDALI
- private ca.nrc.cadc.dali.tables.TableWriter<VOTableDocument> tableWriter;
private RssTableWriter rssTableWriter;
+ private ca.nrc.cadc.dali.tables.TableWriter<ResultSet> resultSetWriter;
+ private ca.nrc.cadc.dali.tables.TableWriter<VOTableDocument> voDocumentWriter;
private FormatFactory formatFactory;
private boolean errorWriter = false;
@@ -191,7 +203,7 @@ public RubinTableWriter() {
public RubinTableWriter(boolean errorWriter) {
this.errorWriter = errorWriter;
}
-
+
@Override
public void setJob(Job job)
{
@@ -216,17 +228,26 @@ public void setQueryInfo(String queryInfo)
@Override
public String getContentType()
{
- return tableWriter.getContentType();
+ String contentType = null;
+
+ if (resultSetWriter != null) {
+ contentType = resultSetWriter.getContentType();
+ } else if (resultSetWriter != null) {
+ contentType = resultSetWriter.getContentType();
+ }
+
+ return contentType;
}
@Override
public String getErrorContentType()
{
- return tableWriter.getErrorContentType();
+ return resultSetWriter.getErrorContentType();
}
/**
* Get the number of rows the output table
+ *
* @return number of result rows written in output table
*/
@Override
@@ -264,26 +285,44 @@ private void initFormat()
// Note: This needs to be done before the write method is called so the contentType
// can be determined from the table writer.
- if (type.equals(RSS))
- {
+ switch (type) {
+ case RSS:
rssTableWriter = new RssTableWriter();
rssTableWriter.setJob(job);
- // for error handling
- tableWriter = new AsciiTableWriter(AsciiTableWriter.ContentType.TSV);
- }
- else if (type.equals(VOTABLE)) {
- tableWriter = new VOTableWriter(format);
- } else if (type.equals(CSV)) {
- tableWriter = new AsciiTableWriter(AsciiTableWriter.ContentType.CSV);
- } else if (type.equals(TSV)) {
- tableWriter = new AsciiTableWriter(AsciiTableWriter.ContentType.TSV);
- }
-
- if (tableWriter == null) {
+ // For error handling
+ voDocumentWriter = new AsciiTableWriter(AsciiTableWriter.ContentType.TSV);
+ break;
+
+ case VOTABLE:
+ resultSetWriter = new ResultSetWriter();
+ this.contentType = resultSetWriter.getContentType();
+ this.extension = resultSetWriter.getExtension();
+ break;
+
+ case VOTABLE_TD:
+ voDocumentWriter = new VOTableWriter();
+ this.contentType = voDocumentWriter.getContentType();
+ this.extension = voDocumentWriter.getExtension();
+ break;
+
+ case CSV:
+ voDocumentWriter = new AsciiTableWriter(AsciiTableWriter.ContentType.CSV);
+ break;
+
+ case TSV:
+ voDocumentWriter = new AsciiTableWriter(AsciiTableWriter.ContentType.TSV);
+ break;
+
+ default:
throw new UnsupportedOperationException("unsupported format: " + type);
- }
- this.contentType = tableWriter.getContentType();
- this.extension = tableWriter.getExtension();
+ }
+
+ if (voDocumentWriter != null) {
+ this.contentType = voDocumentWriter.getContentType();
+ this.extension = voDocumentWriter.getExtension();
+ }
+
+
}
public void setFormatFactory(FormatFactory formatFactory)
@@ -300,22 +339,29 @@ public void setFormatFactory(ca.nrc.cadc.dali.util.FormatFactory formatFactory)
public void write(Throwable t, OutputStream out)
throws IOException
{
- tableWriter.write(t, out);
+ if (this.resultSetWriter != null) {
+ this.resultSetWriter.write(t, out);
+ } else if (this.voDocumentWriter != null) {
+ this.voDocumentWriter.write(t, out);
+ }
}
@Override
public void write(ResultSet rs, OutputStream out) throws IOException
{
- this.write(rs, out, null);
+ try (Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"))) {
+ this.write(rs, writer, null);
+ }
}
@Override
public void write(ResultSet rs, OutputStream out, Long maxrec)
throws IOException
{
- Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"));
- this.write(rs, writer, maxrec);
+ try (Writer writer = new BufferedWriter(new OutputStreamWriter(out, "UTF-8"))) {
+ write(rs, writer, maxrec);
+ }
}
@Override
@@ -354,6 +400,8 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
List<String> columnNames = new ArrayList<String>();
int listIndex = 0;
+
+ List<ColumnInfo> columnInfoList = new ArrayList<>();
// Add the metadata elements.
for (TapSelectItem resultCol : selectList)
@@ -367,7 +415,20 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
String fullColumnName = resultCol.tableName + "_" + resultCol.getColumnName();
columnNames.add(fullColumnName.replace(".", "_"));
listIndex++;
+
+ // Generate a ColumnInfo list, to be used by ResultSetWriter for generating the field metadata
+ ColumnInfo colInfo = new ColumnInfo(resultCol.getColumnName(), getDatatypeClass(resultCol.getDatatype(), newField.getArraysize()), newField.description);
+ colInfo.setUCD(resultCol.ucd);
+ colInfo.setUtype(resultCol.utype);
+ colInfo.setUnitString(resultCol.unit);
+ colInfo.setXtype(newField.xtype);
+ colInfo.setAuxDatum(new DescribedValue(VOStarTable.ID_INFO, newField.id));
+ colInfo.setAuxDatum(new DescribedValue(new DefaultValueInfo(TABLE_NAME_INFO, String.class),
+ resultCol.tableName));
+ columnInfoList.add(colInfo);
+
}
+ ColumnInfo[] columnInfoArray = columnInfoList.toArray(new ColumnInfo[0]);
List<String> serviceIDs = determineDatalinks(columnNames);
@@ -376,7 +437,9 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
// Add the "meta" resources to describe services for each columnID in
// list columnIDs that we recognize
- addMetaResources(votableDocument, serviceIDs, columnNames);
+
+ List<VOTableResource> metaResources = generateMetaResources(serviceIDs, columnNames);
+ votableDocument.getResources().addAll(metaResources);
VOTableInfo info = new VOTableInfo("QUERY_STATUS", "OK");
resultsResource.getInfos().add(info);
@@ -392,31 +455,95 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
info = new VOTableInfo("QUERY", queryInfo);
resultsResource.getInfos().add(info);
}
+
+ if (resultSetWriter != null) {
+ ((ResultSetWriter) resultSetWriter).setInfos(resultsResource.getInfos());
+ ((ResultSetWriter) resultSetWriter).setResources(metaResources);
+ ((ResultSetWriter) resultSetWriter).setColumns(columnInfoArray);
+ if (maxrec != null) {
+ resultSetWriter.write(rs, out, maxrec);
+ } else {
+ resultSetWriter.write(rs, out);
+ }
+ this.rowcount = ((ResultSetWriter) resultSetWriter).getRowCount();
+
+ } else if (voDocumentWriter != null) {
+ ResultSetTableData tableData = new ResultSetTableData(rs, formats);
+ resultsTable.setTableData(tableData);
+ if (maxrec != null) {
+ voDocumentWriter.write(votableDocument, out, maxrec);
+ } else {
+ voDocumentWriter.write(votableDocument, out);
+ }
+ this.rowcount = tableData.getRowCount();
+ }
+
+ log.debug("Final row count after processing: " + this.rowcount);
- ResultSetTableData tableData = new ResultSetTableData(rs, formats);
- resultsTable.setTableData(tableData);
-
- if (maxrec != null)
- tableWriter.write(votableDocument, out, maxrec);
- else
- tableWriter.write(votableDocument, out);
-
- this.rowcount = tableData.getRowCount();
}
-
- private void addMetaResources(VOTableDocument votableDocument, List<String> serviceIDs, List<String> columns)
- throws IOException
- {
- for (String serviceID : serviceIDs)
- {
+
+ /**
+ * Determines the appropriate Java class type for a given TAP data type and array size specification.
+ *
+ * @param datatype The TAP data type specification object containing the type information.
+ * @param arraysize The size specification for array types.
+ *
+ * @return The corresponding Java Class object that represents the specified data type.
+ *
+ * @throws NullPointerException if datatype is null
+ */
+ protected static final Class<?> getDatatypeClass(final TapDataType datatype, final String arraysize) {
+ boolean isScalar = arraysize == null || (arraysize.length() == 1 && arraysize.equals("1"));
+ switch(datatype.getDatatype().toUpperCase()) {
+ case "BLOB":
+ return boolean[].class;
+ case "BOOLEAN":
+ return isScalar ? Boolean.class : boolean[].class;
+ case "DOUBLE":
+ return isScalar ? Double.class : double[].class;
+ case "DOUBLECOMPLEX":
+ return double[].class;
+ case "FLOAT":
+ return isScalar ? Float.class : float[].class;
+ case "FLOATCOMPLEX":
+ return float[].class;
+ case "INT":
+ return isScalar ? Integer.class : int[].class;
+ case "LONG":
+ return isScalar ? Long.class : long[].class;
+ case "SHORT":
+ return isScalar ? Short.class : short[].class;
+ case "UNSIGNEDBYTE":
+ return isScalar ? Short.class : short[].class;
+ case "CHAR":
+ case "UNICODECHAR":
+ default: /* If the type is not know (theoretically, never happens), return char[*] by default. */
+ return isScalar ? Character.class : String.class;
+ }
+ }
+
+ /**
+ * Generates a list of VOTable meta resources.
+ *
+ * @param serviceIDs A list of service identifiers used to locate corresponding XML templates.
+ *
+ * @param columns A list of column names that will be mapped to template variables.
+ *
+ * @return A list of VOTableResource objects containing the processed meta information
+ *
+ * @throws IOException
+ * @throws IllegalStateException
+ */
+ private List<VOTableResource> generateMetaResources(List<String> serviceIDs, List<String> columns) throws IOException {
+ List<VOTableResource> metaResources = new ArrayList<>();
+ for (String serviceID : serviceIDs) {
Path snippetPath = Path.of(datalinkConfig + serviceID + ".xml");
String content = Files.readString(snippetPath, StandardCharsets.US_ASCII);
ST datalinkTemplate = new ST(content, '$', '$');
datalinkTemplate.add("baseUrl", baseUrl);
int columnIndex = 0;
- for (String col : columns)
- {
+ for (String col : columns) {
datalinkTemplate.add(col, "col_" + columnIndex);
columnIndex++;
}
@@ -424,8 +551,9 @@ private void addMetaResources(VOTableDocument votableDocument, List<String> serv
VOTableReader reader = new VOTableReader();
VOTableDocument serviceDocument = reader.read(datalinkTemplate.render());
VOTableResource metaResource = serviceDocument.getResourceByType("meta");
- votableDocument.getResources().add(metaResource);
+ metaResources.add(metaResource);
}
+ return metaResources;
}
private List<String> determineDatalinks(List<String> columns)
From e3dafcc90136d582bc5aaeeb44f076b8b35e643f Mon Sep 17 00:00:00 2001
From: Stelios Voutsinas <steliosvoutsinas@yahoo.com>
Date: Tue, 14 Jan 2025 15:35:56 -0700
Subject: [PATCH 2/3] Bugfix for column aliases
---
.../java/org/opencadc/tap/impl/ResultSetWriter.java | 5 ++++-
.../java/org/opencadc/tap/impl/RubinTableWriter.java | 10 +++++++---
2 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java b/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
index b4ab6c5..4c65d29 100644
--- a/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
+++ b/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
@@ -396,6 +396,7 @@ private static class LimitedResultSetStarTable
extends SequentialResultSetStarTable {
public static final String TABLE_NAME_INFO = "TABLE_NAME";
+ public static final String ACTUAL_COLUMN_NAME_INFO = "COLUMN_NAME";
private final long maxrec_;
private boolean overflow_;
private long totalRows = 0;
@@ -468,10 +469,12 @@ private static Map<Integer, Boolean> findAccessUrlColumns(ColumnInfo[] columnInf
ColumnInfo info = columnInfos[i];
DescribedValue tableNameValue = info.getAuxDatumByName(TABLE_NAME_INFO);
+ DescribedValue actualColumnValue = info.getAuxDatumByName(ACTUAL_COLUMN_NAME_INFO);
if (tableNameValue != null) {
String tableName = tableNameValue.getValue().toString();
+ String actualColName = actualColumnValue.getValue().toString();
- if ("access_url".equals(info.getName())) {
+ if ("access_url".equals(actualColName)) {
boolean isObsCore = "ivoa.ObsCore".equals(tableName);
columns.put(i, isObsCore);
log.debug("Found access_url column at index " + i +
diff --git a/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java b/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
index e36005c..b62c27a 100644
--- a/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
+++ b/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
@@ -129,6 +129,7 @@ public class RubinTableWriter implements TableWriter
private static final Logger log = Logger.getLogger(RubinTableWriter.class);
public static final String TABLE_NAME_INFO = "TABLE_NAME";
+ public static final String ACTUAL_COLUMN_NAME_INFO = "COLUMN_NAME";
private static final String FORMAT = "RESPONSEFORMAT";
private static final String FORMAT_ALT = "FORMAT";
@@ -415,9 +416,8 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
String fullColumnName = resultCol.tableName + "_" + resultCol.getColumnName();
columnNames.add(fullColumnName.replace(".", "_"));
listIndex++;
-
// Generate a ColumnInfo list, to be used by ResultSetWriter for generating the field metadata
- ColumnInfo colInfo = new ColumnInfo(resultCol.getColumnName(), getDatatypeClass(resultCol.getDatatype(), newField.getArraysize()), newField.description);
+ ColumnInfo colInfo = new ColumnInfo(resultCol.getName(), getDatatypeClass(resultCol.getDatatype(), newField.getArraysize()), newField.description);
colInfo.setUCD(resultCol.ucd);
colInfo.setUtype(resultCol.utype);
colInfo.setUnitString(resultCol.unit);
@@ -425,9 +425,13 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
colInfo.setAuxDatum(new DescribedValue(VOStarTable.ID_INFO, newField.id));
colInfo.setAuxDatum(new DescribedValue(new DefaultValueInfo(TABLE_NAME_INFO, String.class),
resultCol.tableName));
- columnInfoList.add(colInfo);
+ colInfo.setAuxDatum(new DescribedValue(new DefaultValueInfo(ACTUAL_COLUMN_NAME_INFO, String.class),
+ resultCol.getColumnName()));
+
+ columnInfoList.add(colInfo);
}
+
ColumnInfo[] columnInfoArray = columnInfoList.toArray(new ColumnInfo[0]);
List<String> serviceIDs = determineDatalinks(columnNames);
From 73f6996aa8f242eef1ee3178e8651af1ba7a6bd7 Mon Sep 17 00:00:00 2001
From: Stelios Voutsinas <steliosvoutsinas@yahoo.com>
Date: Wed, 15 Jan 2025 11:32:29 -0700
Subject: [PATCH 3/3] Workaround for maxrec=0 issue & char fields
---
.../opencadc/tap/impl/ResultSetWriter.java | 243 +++++++++++++++---
.../opencadc/tap/impl/RubinTableWriter.java | 34 ++-
2 files changed, 243 insertions(+), 34 deletions(-)
diff --git a/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java b/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
index 4c65d29..852991d 100644
--- a/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
+++ b/src/main/java/org/opencadc/tap/impl/ResultSetWriter.java
@@ -57,7 +57,7 @@ public class ResultSetWriter implements TableWriter<ResultSet> {
public static final String CONTENT_TYPE_ALT = "text/xml";
// VOTable Version number.
- public static final String VOTABLE_VERSION = "1.4";
+ public static final String VOTABLE_VERSION = "1.3";
// Uri to the XML schema.
public static final String XSI_SCHEMA = "http://www.w3.org/2001/XMLSchema-instance";
@@ -91,7 +91,7 @@ public long getRowCount() {
* Default constructor.
*/
public ResultSetWriter() {
- this(null, uk.ac.starlink.votable.DataFormat.BINARY2, VOTableVersion.V14, -1);
+ this(null, uk.ac.starlink.votable.DataFormat.BINARY2, VOTableVersion.V13, -1);
}
@@ -325,6 +325,12 @@ protected void writeImpl(ResultSet resultSet, Writer writer, Long maxrec)
? (BufferedWriter) writer
: new BufferedWriter(writer)) {
+
+ if (maxrec != null && maxrec == 0 || resultSet == null) {
+ writeEmptyResult(out);
+ return;
+ }
+
LimitedResultSetStarTable table;
try {
table = new LimitedResultSetStarTable(this.getColumns(), resultSet, maxrec);
@@ -344,7 +350,7 @@ protected void writeImpl(ResultSet resultSet, Writer writer, Long maxrec)
version_.getXmlNamespace() )
+ ">" );
out.newLine();
- out.write( "<RESOURCE>" );
+ out.write( "<RESOURCE type='results'>" );
out.newLine();
XMLOutputter outputter = new XMLOutputter();
@@ -384,6 +390,136 @@ protected void writeImpl(ResultSet resultSet, Writer writer, Long maxrec)
}
}
+
+ /**
+ * Write out an empty result (Used when maxrec=0)
+ *
+ * @param out
+ */
+ void writeEmptyResult(BufferedWriter out) {
+ try {
+ /* Write header. */
+ out.write("<VOTABLE"
+ + VOSerializer.formatAttribute("version", version_.getVersionNumber())
+ + VOSerializer.formatAttribute("xmlns", version_.getXmlNamespace())
+ + ">");
+ out.newLine();
+ out.write("<RESOURCE type='results'>");
+ out.newLine();
+
+ XMLOutputter outputter = new XMLOutputter();
+ outputter.setFormat(org.jdom2.output.Format.getPrettyFormat());
+
+ // Write all info elements
+ for (VOTableInfo info : infos) {
+ Element infoElement = new Element("INFO");
+ infoElement.setAttribute("name", info.getName());
+ infoElement.setAttribute("value", info.getValue());
+ outputter.output(infoElement, out);
+ out.newLine();
+ }
+
+ // Add TABLE element and column metadata
+ out.write("<TABLE>\n");
+
+ // Write all column metadata
+ for (ColumnInfo colInfo : columns) {
+ Element field = new Element("FIELD");
+ field.setAttribute("name", colInfo.getName());
+
+ // Set datatype
+ if (colInfo.getContentClass() != null) {
+ String datatype = getVOTableDatatype(colInfo.getContentClass());
+ if (datatype != null) {
+ field.setAttribute("datatype", datatype);
+
+ // Handle array types
+ if (colInfo.getContentClass().isArray()) {
+ if (colInfo.getContentClass().getComponentType() == String.class) {
+ field.setAttribute("arraysize", "*");
+ } else if (colInfo.getShape() != null && colInfo.getShape().length > 0) {
+ field.setAttribute("arraysize", String.valueOf(colInfo.getShape()[0]));
+ }
+ }
+ }
+ }
+
+ // Add standard metadata
+ if (colInfo.getUnitString() != null) {
+ field.setAttribute("unit", colInfo.getUnitString());
+ }
+ if (colInfo.getUCD() != null) {
+ field.setAttribute("ucd", colInfo.getUCD());
+ }
+ if (colInfo.getUtype() != null) {
+ field.setAttribute("utype", colInfo.getUtype());
+ }
+
+ // Add ID if present
+ DescribedValue idValue = colInfo.getAuxDatumByName("ID_INFO");
+ if (idValue != null && idValue.getValue() != null) {
+ field.setAttribute("ID", idValue.getValue().toString());
+ }
+
+ // Add description if present
+ String description = colInfo.getDescription();
+ if (description != null && !description.isEmpty()) {
+ Element desc = new Element("DESCRIPTION");
+ desc.setText(description);
+ field.addContent(desc);
+ }
+
+ outputter.output(field, out);
+ out.newLine();
+ }
+
+ // Write the empty data section
+ out.write("<DATA>\n");
+ out.write("<BINARY2>\n");
+ out.write("<STREAM encoding='base64'>\n");
+ out.write("</STREAM>\n");
+ out.write("</BINARY2>\n");
+ out.write("</DATA>\n");
+
+ out.write("</TABLE>\n"); // Close the TABLE element
+
+ /* Write footer. */
+ out.write("</RESOURCE>");
+ out.newLine();
+ for (VOTableResource resource : resources) {
+ Element r = createResource(resource, null);
+ outputter.output(r, out);
+ out.newLine();
+ }
+ out.write("</VOTABLE>");
+ out.newLine();
+ out.flush();
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ this.totalRows = 0;
+ }
+
+ /**
+ * Convert Java class to VOTable datatype
+ *
+ * @param clazz
+ * @return
+ */
+ private String getVOTableDatatype(Class<?> clazz) {
+ if (clazz == Boolean.class || clazz == boolean.class) return "boolean";
+ if (clazz == Byte.class || clazz == byte.class) return "unsignedByte";
+ if (clazz == Short.class || clazz == short.class) return "short";
+ if (clazz == Integer.class || clazz == int.class) return "int";
+ if (clazz == Long.class || clazz == long.class) return "long";
+ if (clazz == Float.class || clazz == float.class) return "float";
+ if (clazz == Double.class || clazz == double.class) return "double";
+ if (clazz == Character.class || clazz == char.class) return "char";
+ if (clazz == String.class) return "char";
+ return null;
+ }
/**
* StarTable implementation which is based on a ResultSet, and which
@@ -452,10 +588,12 @@ public boolean lastSequenceOverflowed() {
*/
private static class ModifiedLimitRowSequence extends WrapperRowSequence {
private final Map<Integer, Boolean> accessUrlColumns;
-
+ private ColumnInfo[] columnInfos;
+
ModifiedLimitRowSequence(RowSequence baseSeq, ColumnInfo[] columnInfos) {
super(baseSeq);
this.accessUrlColumns = findAccessUrlColumns(columnInfos);
+ this.columnInfos = columnInfos;
}
/**
@@ -477,9 +615,6 @@ private static Map<Integer, Boolean> findAccessUrlColumns(ColumnInfo[] columnInf
if ("access_url".equals(actualColName)) {
boolean isObsCore = "ivoa.ObsCore".equals(tableName);
columns.put(i, isObsCore);
- log.debug("Found access_url column at index " + i +
- " for table " + tableName +
- (isObsCore ? " (will modify URLs)" : " (won't modify URLs)"));
}
}
}
@@ -491,38 +626,83 @@ private static Map<Integer, Boolean> findAccessUrlColumns(ColumnInfo[] columnInf
}
/**
- * Returns the value from a single cell, modifying it if it's a URL that needs to be rewritten.
+ * Return values from a row, applying necessary type conversions and URL modifications.
*
- * @param icol the column index
- * @return the cell contents
+ * This method handles two types of transformations:
+ * 1. String to Character conversion for VOTable char datatype fields
+ * 2. URL rewriting for access_url columns in ObsCore tables
+ *
+ * For char datatype columns:
+ * - If the input is a string and the column expects char, take the first character
+ * - Empty strings are converted to null
+ * - Non-string values are passed through unchanged
+ *
+ * @return array containing values for the current row, with appropriate type conversions
+ * @throws IOException if there is an error reading from the underlying sequence
*/
@Override
- public Object getCell(int icol) throws IOException {
- Object value = super.getCell(icol);
- if (shouldModifyUrl(icol, value)) {
- return modifyAccessUrl((String) value);
+ public Object[] getRow() throws IOException {
+ Object[] row = super.getRow();
+ if (row == null) {
+ return null;
}
- return value;
+
+ for (int i = 0; i < row.length; i++) {
+ Object value = row[i];
+ ColumnInfo info = columnInfos[i];
+
+ if (value != null) {
+
+ // Handle conversion from String to Character for char columns
+ if (value instanceof String && info.getContentClass() == Character.class) {
+ String strVal = (String)value;
+ if (strVal.length() > 0) {
+ row[i] = strVal.charAt(0);
+ } else {
+ row[i] = null;
+ }
+ }
+ }
+
+ if (shouldModifyUrl(i, row[i])) {
+ row[i] = modifyAccessUrl((String)row[i]);
+ }
+ }
+
+ return row;
}
/**
- * Returns values from an entire row, modifying any URLs that need to be rewritten.
+ * Returns the value from a single cell, applying necessary type conversions and URL modifications.
*
- * @return array containing values for the current row
+ * @param icol the column index
+ * @return the cell contents after any necessary conversions
+ * @throws IOException if there is an error reading from the underlying sequence
*/
@Override
- public Object[] getRow() throws IOException {
- Object[] row = super.getRow();
- for (Map.Entry<Integer, Boolean> entry : accessUrlColumns.entrySet()) {
- int index = entry.getKey();
- if (index >= 0 && index < row.length &&
- shouldModifyUrl(index, row[index])) {
- row[index] = modifyAccessUrl((String) row[index]);
+ public Object getCell(int icol) throws IOException {
+ Object value = super.getCell(icol);
+ ColumnInfo info = columnInfos[icol];
+
+ if (value != null) {
+ // Handle conversion from String to Character for char columns
+ if (value instanceof String && info.getContentClass() == Character.class) {
+ String strVal = (String)value;
+ if (strVal.length() > 0) {
+ value = strVal.charAt(0);
+ } else {
+ value = null;
+ }
}
}
- return row;
+
+ if (shouldModifyUrl(icol, value)) {
+ value = modifyAccessUrl((String)value);
+ }
+
+ return value;
}
-
+
/**
* Determines whether a value at a given index should have its URL modified.
*
@@ -552,12 +732,11 @@ private String modifyAccessUrl(String url) {
URL base_url = new URL(BASE_URL);
URL rewritten = new URL(orig.getProtocol(), base_url.getHost(), orig.getFile());
log.debug( "Rewritten URL: " + rewritten.toExternalForm());
-
return rewritten.toExternalForm();
} catch (MalformedURLException ex) {
- throw new RuntimeException("BUG: Failed to rewrite URL: " + s, ex);
- }
+ throw new RuntimeException("BUG: Failed to rewrite URL: " + s, ex);
}
+ }
return url;
}
}
@@ -679,7 +858,7 @@ private Element createResource(VOTableResource votResource, Namespace namespace)
*/
protected Document createDocument() {
// the root VOTABLE element
- Namespace vot = Namespace.getNamespace(VOTABLE_14_NS_URI);
+ Namespace vot = Namespace.getNamespace(VOTABLE_13_NS_URI);
Namespace xsi = Namespace.getNamespace("xsi", XSI_SCHEMA);
Element votable = new Element("VOTABLE", vot);
votable.setAttribute("version", VOTABLE_VERSION);
@@ -713,7 +892,9 @@ private String getThrownExceptions(Throwable thrown) {
sb.append(thrown.getMessage());
}
}
- return sb.toString();
+ String result = sb.toString().trim();
+ return result.isEmpty() ? "An unknown error occurred" : result;
+
}
diff --git a/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java b/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
index b62c27a..92b25a8 100644
--- a/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
+++ b/src/main/java/org/opencadc/tap/impl/RubinTableWriter.java
@@ -417,19 +417,26 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
columnNames.add(fullColumnName.replace(".", "_"));
listIndex++;
// Generate a ColumnInfo list, to be used by ResultSetWriter for generating the field metadata
- ColumnInfo colInfo = new ColumnInfo(resultCol.getName(), getDatatypeClass(resultCol.getDatatype(), newField.getArraysize()), newField.description);
+ String colArraySize = newField.getArraysize();
+ if (colArraySize == null && "CHAR".equals(newField.getDatatype().toUpperCase())) {
+ colArraySize = "1";
+ }
+
+ ColumnInfo colInfo = new ColumnInfo(resultCol.getName(), getDatatypeClass(resultCol.getDatatype(), colArraySize), newField.description);
colInfo.setUCD(resultCol.ucd);
colInfo.setUtype(resultCol.utype);
colInfo.setUnitString(resultCol.unit);
colInfo.setXtype(newField.xtype);
+ colInfo.setShape(getShape(colArraySize));
+
+
colInfo.setAuxDatum(new DescribedValue(VOStarTable.ID_INFO, newField.id));
colInfo.setAuxDatum(new DescribedValue(new DefaultValueInfo(TABLE_NAME_INFO, String.class),
resultCol.tableName));
colInfo.setAuxDatum(new DescribedValue(new DefaultValueInfo(ACTUAL_COLUMN_NAME_INFO, String.class),
resultCol.getColumnName()));
-
+
columnInfoList.add(colInfo);
-
}
ColumnInfo[] columnInfoArray = columnInfoList.toArray(new ColumnInfo[0]);
@@ -526,6 +533,27 @@ protected static final Class<?> getDatatypeClass(final TapDataType datatype, fin
}
}
+ /**
+ * Convert the given VOTable arraysize into a {@link ColumnInfo} shape.
+ *
+ * @param arraysize Value of the VOTable attribute "arraysize".
+ *
+ * @return The corresponding {@link ColumnInfo} shape.
+ */
+ protected static final int[] getShape(final String arraysize) {
+ if (arraysize == null)
+ return new int[0];
+ else if (arraysize.charAt(arraysize.length() - 1) == '*')
+ return new int[]{ -1 };
+ else {
+ try {
+ return new int[]{ Integer.parseInt(arraysize) };
+ } catch(NumberFormatException nfe) {
+ return new int[0];
+ }
+ }
+ }
+
/**
* Generates a list of VOTable meta resources.
*