diff --git a/solarnet/datum/build.gradle b/solarnet/datum/build.gradle index 968e758ab..a78396950 100644 --- a/solarnet/datum/build.gradle +++ b/solarnet/datum/build.gradle @@ -14,7 +14,7 @@ dependencyManagement { } description = 'SolarNet: Datum' -version = '1.14.0' +version = '1.15.0' base { archivesName = 'solarnet-datum' diff --git a/solarnet/datum/src/main/java/net/solarnetwork/central/datum/domain/DatumFilterCommand.java b/solarnet/datum/src/main/java/net/solarnetwork/central/datum/domain/DatumFilterCommand.java index 5b9f3214f..69ada4e28 100644 --- a/solarnet/datum/src/main/java/net/solarnetwork/central/datum/domain/DatumFilterCommand.java +++ b/solarnet/datum/src/main/java/net/solarnetwork/central/datum/domain/DatumFilterCommand.java @@ -249,6 +249,7 @@ public String toString() { if ( sourceIds != null && sourceIds.length > 0 ) { builder.append("sourceIds="); builder.append(Arrays.toString(sourceIds)); + builder.append(", "); } builder.append("mostRecent="); builder.append(mostRecent); diff --git a/solarnet/datum/src/main/java/net/solarnetwork/central/datum/export/domain/Configuration.java b/solarnet/datum/src/main/java/net/solarnetwork/central/datum/export/domain/Configuration.java index 33fca5c5c..5a12bd253 100644 --- a/solarnet/datum/src/main/java/net/solarnetwork/central/datum/export/domain/Configuration.java +++ b/solarnet/datum/src/main/java/net/solarnetwork/central/datum/export/domain/Configuration.java @@ -33,13 +33,14 @@ import java.time.temporal.IsoFields; import java.util.LinkedHashMap; import java.util.Map; +import java.util.regex.Pattern; import net.solarnetwork.central.datum.export.biz.DatumExportOutputFormatService; /** * A complete configuration for a scheduled export job. * * @author matt - * @version 1.0 + * @version 1.1 * @since 1.23 */ public interface Configuration { @@ -109,6 +110,29 @@ public interface Configuration { /** A runtime property for a filename extension. */ String PROP_FILENAME_EXTENSION = "ext"; + /** + * A runtime property for the configuration name. + * + * @since 1.1 + */ + String PROP_JOB_NAME = "jobName"; + + /** + * A regular expression to remove unfriendly characters from file names + * (like the {@link #PROP_JOB_NAME} parameter). + * + * @since 1.1 + */ + Pattern PROP_NAME_SANITIZER = Pattern.compile("(?U)[^\\w\\._-]+"); + + /** + * A runtime property for the job export process time, as a millisecond Unix + * epoch integer. + * + * @since 1.1 + */ + String PROP_CURRENT_TIME = "now"; + // @formatter:off /** A formatter for week, in {@literal YYYY'W'WWD} form. */ DateTimeFormatter WEEK_DATE = new DateTimeFormatterBuilder() @@ -164,6 +188,12 @@ default Map createRuntimeProperties(Instant exportTime, result.put(PROP_FILENAME_EXTENSION, ext); } + if ( getName() != null ) { + result.put(PROP_JOB_NAME, PROP_NAME_SANITIZER.matcher(getName()).replaceAll("_")); + } + + result.put(PROP_CURRENT_TIME, System.currentTimeMillis()); + return result; } diff --git a/solarnet/datum/src/main/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/SelectDatum.java b/solarnet/datum/src/main/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/SelectDatum.java index 4a630aa26..3aab57230 100644 --- a/solarnet/datum/src/main/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/SelectDatum.java +++ b/solarnet/datum/src/main/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/SelectDatum.java @@ -161,10 +161,18 @@ private void sqlSelect(StringBuilder buf) { buf.append("SELECT "); if ( combine != null ) { buf.append("s.vstream_id AS stream_id,\n"); - buf.append(" s.obj_rank,\n"); - buf.append(" s.source_rank,\n"); - buf.append(" s.names_i,\n"); - buf.append(" s.names_a,\n"); + if ( aggregation == Aggregation.Week ) { + buf.append(" solarcommon.first(s.obj_rank) AS obj_rank,\n"); + buf.append(" solarcommon.first(s.source_rank) AS source_rank,\n"); + buf.append(" solarcommon.first(s.names_i) AS names_i,\n"); + buf.append(" solarcommon.first(s.names_a) AS names_a,\n"); + + } else { + buf.append(" s.obj_rank,\n"); + buf.append(" s.source_rank,\n"); + buf.append(" s.names_i,\n"); + buf.append(" s.names_a,\n"); + } } else { buf.append(" datum.stream_id,\n"); } @@ -361,8 +369,14 @@ private void sqlCore(StringBuilder buf, boolean ordered) { sqlFrom(buf); sqlWhere(buf); if ( aggregation == Aggregation.Week ) { + buf.append("GROUP BY "); + if ( combine != null ) { + buf.append("s.vstream_id"); + } else { + buf.append("datum.stream_id"); + } buf.append( - "GROUP BY datum.stream_id, date_trunc('week', datum.ts_start AT TIME ZONE s.time_zone) AT TIME ZONE s.time_zone\n"); + ", date_trunc('week', datum.ts_start AT TIME ZONE s.time_zone) AT TIME ZONE s.time_zone\n"); } if ( combine != null ) { if ( isMinuteAggregation() ) { diff --git a/solarnet/datum/src/main/resources/net/solarnetwork/central/datum/export/dest/s3/S3DestinationProperties.properties b/solarnet/datum/src/main/resources/net/solarnetwork/central/datum/export/dest/s3/S3DestinationProperties.properties index d96a2a15a..150572473 100644 --- a/solarnet/datum/src/main/resources/net/solarnetwork/central/datum/export/dest/s3/S3DestinationProperties.properties +++ b/solarnet/datum/src/main/resources/net/solarnetwork/central/datum/export/dest/s3/S3DestinationProperties.properties @@ -20,5 +20,7 @@ filenameTemplate.desc = A template filename to use. \ This template is allowed to contain parameters in the form \ {key}, which are replaced at runtime by the value of a \ parameter key, or an empty string if no such parameter \ - exists. The supported parameters are {date} and \ - {ext} (file extension). \ No newline at end of file + exists. The supported parameters are: {date}, \ + {ext} (file extension), and {jobName} \ + (the export job configuration name), and {now} \ + (the export process time as a millisecond Unix epoch). \ No newline at end of file diff --git a/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/dest/s3/test/S3DatumExportDestinationServiceTests.java b/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/dest/s3/test/S3DatumExportDestinationServiceTests.java index 205f2e164..7a5886031 100644 --- a/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/dest/s3/test/S3DatumExportDestinationServiceTests.java +++ b/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/dest/s3/test/S3DatumExportDestinationServiceTests.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import org.junit.BeforeClass; import org.junit.Test; @@ -181,7 +182,7 @@ private String getObjectKeyPrefix() { @Test public void export() throws IOException { - // given + // GIVEN AmazonS3 client = getS3Client(); cleanS3Folder(client); @@ -191,6 +192,7 @@ public void export() throws IOException { .toInstant(); BasicConfiguration config = new BasicConfiguration(); + config.setName(UUID.randomUUID().toString()); BasicDestinationConfiguration destConfig = new BasicDestinationConfiguration(); destConfig.setServiceIdentifier(service.getId()); @@ -204,7 +206,7 @@ public void export() throws IOException { DatumExportResource rsrc = getTestResource(); - // when + // WHEN List progress = new ArrayList<>(8); service.export(config, Collections.singleton(rsrc), runtimeProps, new ProgressListener() { @@ -216,7 +218,7 @@ public void progressChanged(DatumExportService context, double amountComplete) { } }); - // then + // THEN assertThat("Progress was made", progress, not(hasSize(0))); assertThat("Progress complete", progress.get(progress.size() - 1), equalTo((Double) 1.0)); diff --git a/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/domain/test/ConfigurationTests.java b/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/domain/test/ConfigurationTests.java index f238c5278..9b22c51cf 100644 --- a/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/domain/test/ConfigurationTests.java +++ b/solarnet/datum/src/test/java/net/solarnetwork/central/datum/export/domain/test/ConfigurationTests.java @@ -22,16 +22,15 @@ package net.solarnetwork.central.datum.export.domain.test; +import static org.assertj.core.api.BDDAssertions.then; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.notNullValue; import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; import java.util.UUID; +import org.assertj.core.api.InstanceOfAssertFactories; import org.hamcrest.Matchers; import org.junit.Test; import net.solarnetwork.central.datum.export.biz.DatumExportOutputFormatService; @@ -97,13 +96,27 @@ public void dateFormatterMonthly() { @Test public void runtimePropsHourlyNoOutputService() { + // GIVEN + final long now = System.currentTimeMillis(); BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Hourly); + + // WHEN Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, null); - assertThat("Props created", props, notNullValue()); - assertThat("Props size", props.keySet(), hasSize(2)); - assertThat("Timestamp", props, - hasEntry("ts", TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ)))); - assertThat("Date", props, hasEntry("date", "2018-04-23T19:05")); + + // THEN + // @formatter:off + then(props) + .as("Expected props created") + .hasSize(3) + .as("Date time provided") + .containsEntry(Configuration.PROP_DATE_TIME, TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ))) + .as("Formatted date") + .containsEntry(Configuration.PROP_DATE, "2018-04-23T19:05") + .as("Current time epoch") + .extractingByKey(Configuration.PROP_CURRENT_TIME, InstanceOfAssertFactories.LONG) + .isGreaterThanOrEqualTo(now); + ; + // @formatter:on } private DatumExportOutputFormatService createOutputService(String ext) { @@ -133,65 +146,175 @@ public ExportContext createExportContext(OutputConfiguration config) { @Test public void runtimePropsWithOutputService() { + // GIVEN + final long now = System.currentTimeMillis(); BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Hourly); DatumExportOutputFormatService outputService = createOutputService("json"); + + // WHEN Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, outputService); - assertThat("Props created", props, notNullValue()); - assertThat("Props size", props.keySet(), hasSize(3)); - assertThat("Timestamp", props, - hasEntry("ts", TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ)))); - assertThat("Date", props, hasEntry("date", "2018-04-23T19:05")); - assertThat("Extension", props, hasEntry("ext", "json")); + + // THEN + // @formatter:off + then(props) + .as("Expected props created") + .hasSize(4) + .as("File extension provided") + .containsEntry(Configuration.PROP_FILENAME_EXTENSION, "json") + .as("Date time provided") + .containsEntry(Configuration.PROP_DATE_TIME, TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ))) + .as("Formatted date") + .containsEntry(Configuration.PROP_DATE, "2018-04-23T19:05") + .as("Current time epoch") + .extractingByKey(Configuration.PROP_CURRENT_TIME, InstanceOfAssertFactories.LONG) + .isGreaterThanOrEqualTo(now); + ; + // @formatter:on } @Test public void runtimePropsWithOutputServiceAndCompression() { + // GIVEN + final long now = System.currentTimeMillis(); BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Hourly); BasicOutputConfiguration outputConf = new BasicOutputConfiguration(); outputConf.setCompressionType(OutputCompressionType.GZIP); conf.setOutputConfiguration(outputConf); DatumExportOutputFormatService outputService = createOutputService("json"); + + // WHEN Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, outputService); - assertThat("Props created", props, notNullValue()); - assertThat("Props size", props.keySet(), hasSize(3)); - assertThat("Timestamp", props, - hasEntry("ts", TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ)))); - assertThat("Date", props, hasEntry("date", "2018-04-23T19:05")); - assertThat("Extension", props, hasEntry("ext", "json.gz")); + + // THEN + // @formatter:off + then(props) + .as("Expected props created") + .hasSize(4) + .as("File extension provided") + .containsEntry(Configuration.PROP_FILENAME_EXTENSION, "json.gz") + .as("Date time provided") + .containsEntry(Configuration.PROP_DATE_TIME, TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ))) + .as("Formatted date") + .containsEntry(Configuration.PROP_DATE, "2018-04-23T19:05") + .as("Current time epoch") + .extractingByKey(Configuration.PROP_CURRENT_TIME, InstanceOfAssertFactories.LONG) + .isGreaterThanOrEqualTo(now); + ; + // @formatter:on } @Test public void runtimePropsDailyNoOutputService() { + // GIVEN + final long now = System.currentTimeMillis(); BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Daily); + + // WHEN Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, null); - assertThat("Props created", props, notNullValue()); - assertThat("Props size", props.keySet(), hasSize(2)); - assertThat("Timestamp", props, - hasEntry("ts", TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ)))); - assertThat("Date", props, hasEntry("date", "2018-04-23")); + + // THEN + // @formatter:off + then(props) + .as("Expected props created") + .hasSize(3) + .as("Date time provided") + .containsEntry(Configuration.PROP_DATE_TIME, TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ))) + .as("Formatted date") + .containsEntry(Configuration.PROP_DATE, "2018-04-23") + .as("Current time epoch") + .extractingByKey(Configuration.PROP_CURRENT_TIME, InstanceOfAssertFactories.LONG) + .isGreaterThanOrEqualTo(now); + ; + // @formatter:on } @Test public void runtimePropsWeeklyNoOutputService() { + // GIVEN + final long now = System.currentTimeMillis(); BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Weekly); + + // WHEN Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, null); - assertThat("Props created", props, notNullValue()); - assertThat("Props size", props.keySet(), hasSize(2)); - assertThat("Timestamp", props, - hasEntry("ts", TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ)))); - assertThat("Date", props, hasEntry("date", "2018W171")); + + // THEN + // @formatter:off + then(props) + .as("Expected props created") + .hasSize(3) + .as("Date time provided") + .containsEntry(Configuration.PROP_DATE_TIME, TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ))) + .as("Formatted date") + .containsEntry(Configuration.PROP_DATE, "2018W171") + .as("Current time epoch") + .extractingByKey(Configuration.PROP_CURRENT_TIME, InstanceOfAssertFactories.LONG) + .isGreaterThanOrEqualTo(now); + ; + // @formatter:on } @Test public void runtimePropsMonthlyNoOutputService() { + // GIVEN + final long now = System.currentTimeMillis(); + BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Monthly); + + // WHEN + Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, null); + + // THEN + // @formatter:off + then(props) + .as("Expected props created") + .hasSize(3) + .as("Date time provided") + .containsEntry(Configuration.PROP_DATE_TIME, TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ))) + .as("Formatted date") + .containsEntry(Configuration.PROP_DATE, "2018-04") + .as("Current time epoch") + .extractingByKey(Configuration.PROP_CURRENT_TIME, InstanceOfAssertFactories.LONG) + .isGreaterThanOrEqualTo(now); + ; + // @formatter:on + } + + @Test + public void runtimePropsJobName() { + // GIVEN + BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Monthly); + conf.setName(UUID.randomUUID().toString()); + + // WHEN + Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, null); + + // THEN + // @formatter:off + then(props) + .as("Props created with name") + .containsEntry(Configuration.PROP_JOB_NAME, conf.getName()) + ; + // @formatter:on + + } + + @Test + public void runtimePropsJobName_sanitize() { + // GIVEN BasicConfiguration conf = createConfiguration(TEST_TZ, ScheduleType.Monthly); + conf.setName("All the fun characters:/ ⲁあアピⰄ⠷☃😀 / oh yeah!"); + + // WHEN Map props = conf.createRuntimeProperties(TEST_DATE.toInstant(), null, null); - assertThat("Props created", props, notNullValue()); - assertThat("Props size", props.keySet(), hasSize(2)); - assertThat("Timestamp", props, - hasEntry("ts", TEST_DATE.withZoneSameInstant(ZoneId.of(TEST_TZ)))); - assertThat("Date", props, hasEntry("date", "2018-04")); + + // THEN + // @formatter:off + then(props) + .as("Props created with sanitized name, preserving as many language characters as possible") + .containsEntry(Configuration.PROP_JOB_NAME, "All_the_fun_characters_ⲁあアピⰄ_oh_yeah_") + ; + // @formatter:on + } } diff --git a/solarnet/datum/src/test/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/SelectDatumTests.java b/solarnet/datum/src/test/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/SelectDatumTests.java index bcf920a5f..292be9307 100644 --- a/solarnet/datum/src/test/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/SelectDatumTests.java +++ b/solarnet/datum/src/test/java/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/SelectDatumTests.java @@ -833,6 +833,29 @@ public void sql_find_daily_vids_nodeAndSources_absoluteDates_sortTimeNodeSource_ TestSqlResources.class, SQL_COMMENT)); } + @Test + public void sql_find_weekly_vids_nodeAndSources_absoluteDates() { + // GIVEN + BasicDatumCriteria filter = new BasicDatumCriteria(); + filter.setAggregation(Aggregation.Week); + filter.setNodeIds(new Long[] { 1L, 2L, 3L }); + filter.setSourceIds(new String[] { "a", "b", "c" }); + filter.setStartDate(Instant.now().truncatedTo(ChronoUnit.DAYS)); + filter.setEndDate(filter.getStartDate().plusSeconds(TimeUnit.DAYS.toSeconds(7))); + filter.setCombiningType(CombiningType.Sum); + filter.setObjectIdMaps(new String[] { "100:1,2,3" }); + filter.setSourceIdMaps(new String[] { "V:a,b,c" }); + + // WHEN + String sql = new SelectDatum(filter).getSql(); + + // THEN + log.debug("Generated SQL:\n{}", sql); + assertThat("SQL matches", sql, + equalToTextResource("select-datum-weekly-virtual-nodesAndSources-dates.sql", + TestSqlResources.class, SQL_COMMENT)); + } + @Test public void sql_find_seasonal_hod_nodesAndSources_absoluteDates() { // GIVEN diff --git a/solarnet/datum/src/test/resources/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/select-datum-weekly-virtual-nodesAndSources-dates.sql b/solarnet/datum/src/test/resources/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/select-datum-weekly-virtual-nodesAndSources-dates.sql new file mode 100644 index 000000000..f6d82013f --- /dev/null +++ b/solarnet/datum/src/test/resources/net/solarnetwork/central/datum/v2/dao/jdbc/sql/test/select-datum-weekly-virtual-nodesAndSources-dates.sql @@ -0,0 +1,148 @@ +WITH rs AS ( + SELECT s.stream_id + , CASE + WHEN array_position(?, s.node_id) IS NOT NULL THEN ? + ELSE s.node_id + END AS node_id + , COALESCE(array_position(?, s.node_id), 0) AS obj_rank + , CASE + WHEN array_position(?, s.source_id::TEXT) IS NOT NULL THEN ? + ELSE s.source_id + END AS source_id + , COALESCE(array_position(?, s.source_id::TEXT), 0) AS source_rank + , s.names_i + , s.names_a, COALESCE(l.time_zone, 'UTC') AS time_zone + FROM solardatm.da_datm_meta s + LEFT OUTER JOIN solarnet.sn_node n ON n.node_id = s.node_id + LEFT OUTER JOIN solarnet.sn_loc l ON l.id = n.loc_id + WHERE s.node_id = ANY(?) + AND s.source_id ~ ANY(ARRAY(SELECT solarcommon.ant_pattern_to_regexp(unnest(?)))) +) +, s AS ( + SELECT solardatm.virutal_stream_id(node_id, source_id) AS vstream_id + , * + FROM rs +) +, vs AS ( + SELECT DISTINCT ON (vstream_id) vstream_id, node_id, source_id + FROM s +) +, d AS ( + SELECT s.vstream_id AS stream_id, + solarcommon.first(s.obj_rank) AS obj_rank, + solarcommon.first(s.source_rank) AS source_rank, + solarcommon.first(s.names_i) AS names_i, + solarcommon.first(s.names_a) AS names_a, + date_trunc('week', datum.ts_start AT TIME ZONE s.time_zone) AT TIME ZONE s.time_zone AS ts, + (solardatm.rollup_agg_data( + (datum.data_i, datum.data_a, datum.data_s, datum.data_t, datum.stat_i, datum.read_a)::solardatm.agg_data + ORDER BY datum.ts_start)).* + FROM s + INNER JOIN solardatm.agg_datm_daily datum ON datum.stream_id = s.stream_id + WHERE datum.ts_start >= ? + AND datum.ts_start < ? + GROUP BY s.vstream_id, date_trunc('week', datum.ts_start AT TIME ZONE s.time_zone) AT TIME ZONE s.time_zone +) +-- calculate instantaneous values per date + property NAME (to support joining different streams with different index orders) +-- ordered by object/source ranking defined by query metadata; assume names are unique per stream +, wi AS ( + SELECT + d.stream_id + , d.ts + , p.val + , rank() OVER slot as prank + , d.names_i[p.idx] AS pname + , d.stat_i[p.idx][1] AS cnt + , SUM(d.stat_i[p.idx][1]) OVER slot AS tot_cnt + FROM d + INNER JOIN unnest(d.data_i) WITH ORDINALITY AS p(val, idx) ON TRUE + WHERE p.val IS NOT NULL + WINDOW slot AS (PARTITION BY d.stream_id, d.ts, d.names_i[p.idx] ORDER BY d.obj_rank, d.source_rank RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) + ORDER BY d.stream_id, d.ts, d.names_i[p.idx], d.obj_rank, d.source_rank +) +-- calculate instantaneous statistics +, di AS ( + SELECT + stream_id + , ts + , pname + , to_char(SUM(val), 'FM999999999999999999990.999999999')::NUMERIC AS val + , SUM(cnt) AS cnt + FROM wi + GROUP BY stream_id, ts, pname + ORDER BY stream_id, ts, pname +) +-- join property data back into arrays; no stat_i for virtual stream +, di_ary AS ( + SELECT + stream_id + , ts + , array_agg(val ORDER BY pname) AS data_i + , array_agg(pname ORDER BY pname) AS names_i + FROM di + GROUP BY stream_id, ts + ORDER BY stream_id, ts +) +-- calculate accumulating values per date + property NAME (to support joining different streams with different index orders) +-- ordered by object/source ranking defined by query metadata; assume names are unique per stream +, wa AS ( + SELECT + d.stream_id + , d.ts + , p.val + , rank() OVER slot as prank + , d.names_a[p.idx] AS pname + , d.read_a[p.idx][1] AS rdiff + , first_value(d.read_a[p.idx][2]) OVER slot AS rstart + , last_value(d.read_a[p.idx][3]) OVER slot AS rend + FROM d + INNER JOIN unnest(d.data_a) WITH ORDINALITY AS p(val, idx) ON TRUE + WHERE p.val IS NOT NULL + WINDOW slot AS (PARTITION BY d.stream_id, d.ts, d.names_a[p.idx] ORDER BY d.obj_rank, d.source_rank RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) + ORDER BY d.stream_id, d.ts, d.names_a[p.idx], d.obj_rank, d.source_rank +) +-- calculate accumulating statistics +, da AS ( + SELECT + stream_id + , ts + , pname + , to_char(SUM(val), 'FM999999999999999999990.999999999')::NUMERIC AS val + , to_char(SUM(rdiff), 'FM999999999999999999990.999999999')::NUMERIC AS rdiff + FROM wa + GROUP BY stream_id, ts, pname + ORDER BY stream_id, ts, pname +) +-- join property data back into arrays; only read_a.diff for virtual stream +, da_ary AS ( + SELECT + stream_id + , ts + , array_agg(val ORDER BY pname) AS data_a + , array_agg( + ARRAY[rdiff, NULL, NULL] ORDER BY pname + ) AS read_a + , array_agg(pname ORDER BY pname) AS names_a + FROM da + GROUP BY stream_id, ts + ORDER BY stream_id, ts +) +, datum AS ( + SELECT + COALESCE(di_ary.stream_id, da_ary.stream_id) AS stream_id + , COALESCE(di_ary.ts, da_ary.ts) AS ts + , di_ary.data_i + , da_ary.data_a + , NULL::BIGINT[] AS data_s + , NULL::TEXT[] AS data_t + , NULL::BIGINT[][] AS stat_i + , da_ary.read_a + , di_ary.names_i + , da_ary.names_a + FROM di_ary + FULL OUTER JOIN da_ary ON da_ary.stream_id = di_ary.stream_id AND da_ary.ts = di_ary.ts +) +SELECT datum.*, vs.node_id, vs.source_id +FROM datum +INNER JOIN vs ON vs.vstream_id = datum.stream_id +ORDER BY datum.stream_id, ts \ No newline at end of file diff --git a/solarnet/solarjobs/build.gradle b/solarnet/solarjobs/build.gradle index 552b35659..62967c309 100644 --- a/solarnet/solarjobs/build.gradle +++ b/solarnet/solarjobs/build.gradle @@ -9,7 +9,7 @@ apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' description = 'SolarJobs' -version = '1.8.0' +version = '1.9.0' base { archivesName = 'solarjobs' diff --git a/solarnet/solarquery/build.gradle b/solarnet/solarquery/build.gradle index 0715fe865..c3aa2b6cc 100644 --- a/solarnet/solarquery/build.gradle +++ b/solarnet/solarquery/build.gradle @@ -9,7 +9,7 @@ apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' description = 'SolarQuery' -version = '1.9.0' +version = '1.10.0' base { archivesName = 'solarquery' diff --git a/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/AopServices.java b/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/AopServices.java new file mode 100644 index 000000000..1a3ca8141 --- /dev/null +++ b/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/AopServices.java @@ -0,0 +1,39 @@ +/* ================================================================== + * AopServices.java - 18/10/2023 10:13:03 am + * + * Copyright 2023 SolarNetwork.net Dev Team + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + * 02111-1307 USA + * ================================================================== + */ + +package net.solarnetwork.central.query.aop; + +/** + * Marker interface for AOP service package. + * + * @author matt + * @version 1.0 + */ +public interface AopServices { + + /** A profile name for excluding AOP security services. */ + String WITHOUT_AOP_SECURITY = "without-aop-security"; + + /** A profile name for including AOP security services. */ + String WITH_AOP_SECURITY = "!" + WITHOUT_AOP_SECURITY; + +} diff --git a/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/QuerySecurityAspect.java b/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/QuerySecurityAspect.java index ff9b3f373..393559d8e 100644 --- a/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/QuerySecurityAspect.java +++ b/solarnet/solarquery/src/main/java/net/solarnetwork/central/query/aop/QuerySecurityAspect.java @@ -36,6 +36,7 @@ import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Before; import org.aspectj.lang.annotation.Pointcut; +import org.springframework.context.annotation.Profile; import org.springframework.security.core.Authentication; import org.springframework.stereotype.Component; import org.springframework.util.AntPathMatcher; @@ -68,6 +69,7 @@ */ @Aspect @Component +@Profile(AopServices.WITH_AOP_SECURITY) public class QuerySecurityAspect extends AuthorizationSupport { public static final String FILTER_KEY_NODE_ID = "nodeId"; diff --git a/solarnet/solaruser/build.gradle b/solarnet/solaruser/build.gradle index f190c4f04..c9eb63164 100644 --- a/solarnet/solaruser/build.gradle +++ b/solarnet/solaruser/build.gradle @@ -11,7 +11,7 @@ apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' description = 'SolarUser' -version = '1.20.0' +version = '1.21.0' base { archivesName = 'solaruser' diff --git a/solarnet/solaruser/src/main/resources/static/js/export.js b/solarnet/solaruser/src/main/resources/static/js/export.js index 93c4ddbc7..2ea000534 100644 --- a/solarnet/solaruser/src/main/resources/static/js/export.js +++ b/solarnet/solaruser/src/main/resources/static/js/export.js @@ -351,6 +351,8 @@ $(document).ready(function() { } } return data; + }, { + urlSerializer: (url) => url }); return false; }) @@ -377,6 +379,8 @@ $(document).ready(function() { SolarReg.Settings.handlePostEditServiceForm(event, function(req, res) { populateDestinationConfigs([res], true); SolarReg.storeServiceConfiguration(res, exportConfigs.destinationConfigs); + }, undefined, { + urlSerializer: (url) => url }); return false; }) @@ -405,6 +409,8 @@ $(document).ready(function() { SolarReg.Settings.handlePostEditServiceForm(event, function(req, res) { populateOutputConfigs([res], true); SolarReg.storeServiceConfiguration(res, exportConfigs.outputConfigs); + }, undefined, { + urlSerializer: (url) => url }); return false; }) diff --git a/solarnet/solaruser/src/main/resources/templates/sec/expire/edit-data-modal.html b/solarnet/solaruser/src/main/resources/templates/sec/expire/edit-data-modal.html index 34cb31b8a..8c6df5eef 100644 --- a/solarnet/solaruser/src/main/resources/templates/sec/expire/edit-data-modal.html +++ b/solarnet/solaruser/src/main/resources/templates/sec/expire/edit-data-modal.html @@ -104,7 +104,7 @@

Edit Poli The .delete-confirm class will be shown/hidden via the .hidden class to confirm the delete action. */--> diff --git a/solarnet/solaruser/src/main/resources/templates/sec/export/edit-adhoc-export-job-modal.html b/solarnet/solaruser/src/main/resources/templates/sec/export/edit-adhoc-export-job-modal.html index 93c311807..7eb09052e 100644 --- a/solarnet/solaruser/src/main/resources/templates/sec/export/edit-adhoc-export-job-modal.html +++ b/solarnet/solaruser/src/main/resources/templates/sec/export/edit-adhoc-export-job-modal.html @@ -115,7 +115,7 @@

-

+

Are you sure you want to delete this configuration?

diff --git a/solarnet/solaruser/src/main/resources/templates/sec/export/edit-output-modal.html b/solarnet/solaruser/src/main/resources/templates/sec/export/edit-output-modal.html index 7ac7771f0..308195197 100644 --- a/solarnet/solaruser/src/main/resources/templates/sec/export/edit-output-modal.html +++ b/solarnet/solaruser/src/main/resources/templates/sec/export/edit-output-modal.html @@ -67,7 +67,7 @@

Edit ou The .delete-confirm class will be shown/hidden via the .hidden class to confirm the delete action. */-->