Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NMS-17066: Fix Measurement TrendLine #7597

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
+428 −20
Open

NMS-17066: Fix Measurement TrendLine #7597

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4d16296
Fix Measurement TrendLine
Jan 17, 2025
62cf192
Update features/measurements/impl/src/main/java/org/opennms/netmgt/me…
rssntn67 Feb 19, 2025
53ec4a8
Update features/measurements/impl/src/main/java/org/opennms/netmgt/me…
rssntn67 Feb 19, 2025
90061a6
Fix copy right according to Chandra Request
Feb 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 17 additions & 20 deletions ...asurements/impl/src/main/java/org/opennms/netmgt/measurements/filters/impl/TrendLine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,35 +21,20 @@
*/
package org.opennms.netmgt.measurements.filters.impl;

import java.util.Date;
import java.util.Map;

import com.google.common.base.Preconditions;
import com.google.common.collect.RowSortedTable;
import org.apache.commons.math3.analysis.polynomials.PolynomialFunction;
import org.apache.commons.math3.fitting.PolynomialCurveFitter;
import org.apache.commons.math3.fitting.WeightedObservedPoints;
import org.apache.commons.math3.fitting.leastsquares.EvaluationRmsChecker;
import org.apache.commons.math3.fitting.leastsquares.GaussNewtonOptimizer;
import org.apache.commons.math3.fitting.leastsquares.LeastSquaresBuilder;
import org.apache.commons.math3.fitting.leastsquares.LeastSquaresOptimizer;
import org.apache.commons.math3.fitting.leastsquares.LeastSquaresProblem;
import org.apache.commons.math3.fitting.leastsquares.LevenbergMarquardtOptimizer;
import org.apache.commons.math3.linear.ArrayRealVector;
import org.apache.commons.math3.linear.RealVector;
import org.opennms.netmgt.integrations.R.RScriptException;
import org.opennms.netmgt.integrations.R.RScriptExecutor;
import org.opennms.netmgt.integrations.R.RScriptInput;
import org.opennms.netmgt.integrations.R.RScriptOutput;
import org.opennms.netmgt.measurements.api.Filter;
import org.opennms.netmgt.measurements.api.FilterInfo;
import org.opennms.netmgt.measurements.api.FilterParam;
import org.opennms.netmgt.measurements.filters.impl.Utils.TableLimits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableTable;
import com.google.common.collect.Maps;
import com.google.common.collect.RowSortedTable;
import java.util.Date;

/**
* Fits a trend line to the samples in a column.
Expand Down Expand Up @@ -83,11 +68,13 @@ public TrendLine(String outputColumn, String inputColumn, long secondsAhead, int

@Override
public void filter(RowSortedTable<Long, String, Double> table) throws RScriptException {
LOG.debug("filter: values\n{}", table.rowMap().values());
Preconditions.checkArgument(table.containsColumn(TIMESTAMP_COLUMN_NAME), String.format("Data source must have a '%s' column.", Filter.TIMESTAMP_COLUMN_NAME));

// Determine the index of the first and last non-NaN values
// Assume the values between these are contiguous
TableLimits limits = Utils.getRowsWithValues(table, m_inputColumn);
LOG.info("filter: limits: {}, {}", limits.firstRowWithValues, limits.lastRowWithValues);

// Make sure we have some samples
long numSampleRows = limits.lastRowWithValues - limits.firstRowWithValues;
Expand All @@ -98,14 +85,20 @@ public void filter(RowSortedTable<Long, String, Double> table) throws RScriptExc

// Gather the values [(x,y),...]
final WeightedObservedPoints obs = new WeightedObservedPoints();
for (long i = limits.firstRowWithValues; i < limits.lastRowWithValues; i++) {
obs.add(table.get(i, TIMESTAMP_COLUMN_NAME), table.get(i, m_inputColumn));
for (long i = limits.firstRowWithValues; i <= limits.lastRowWithValues; i++) {
Double value = table.get(i, m_inputColumn);
Double ts = table.get(i, TIMESTAMP_COLUMN_NAME);
if (ts != null && !Double.isNaN(ts) && value != null && !Double.isNaN(value)) {
obs.add(ts,value);
}
}
LOG.info("filter: WeightedObservedPoints: {}", obs.toList().size());

// Fit the polynomial
final PolynomialCurveFitter fitter = PolynomialCurveFitter.create(m_polynomialOrder);
final double[] coeff = fitter.fit(obs.toList());
PolynomialFunction polynomialFunction = new PolynomialFunction(coeff);
LOG.info("filter: polynomialFunction: {}", polynomialFunction);

// Determine the step size
Date lastTimestamp = new Date(table.get(limits.lastRowWithValues, TIMESTAMP_COLUMN_NAME).longValue());
Expand All @@ -116,6 +109,9 @@ public void filter(RowSortedTable<Long, String, Double> table) throws RScriptExc
numStepsAhead = Math.max(1, numStepsAhead);

// Calculate the value of the polynomial for all the samples and the requested number of steps ahead
LOG.info("filter: limits.firstRowWithValues:{}", limits.firstRowWithValues);
LOG.info("filter: limits.lastRowWithValues:{}", limits.lastRowWithValues);
LOG.info("filter: numStepsAhead:{}", numStepsAhead);
for (long i = limits.firstRowWithValues; i <= (limits.lastRowWithValues + numStepsAhead); i++) {
if (i >= limits.lastRowWithValues) {
table.put(i, TIMESTAMP_COLUMN_NAME, (double)new Date(lastTimestamp.getTime() + stepInMs * (i-limits.lastRowWithValues)).getTime());
Expand All @@ -124,5 +120,6 @@ public void filter(RowSortedTable<Long, String, Double> table) throws RScriptExc
// Compute the value
table.put(i, m_outputColumn, polynomialFunction.value(timestamp));
}
LOG.debug("filter: values:\n{}", table.rowMap().values());
}
}
89 changes: 89 additions & 0 deletions ...easurements/impl/src/test/java/org/opennms/netmgt/measurements/filters/impl/LoadData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Licensed to The OpenNMS Group, Inc (TOG) under one or more
* contributor license agreements. See the LICENSE.md file
* distributed with this work for additional information
* regarding copyright ownership.
*
* TOG licenses this file to You under the GNU Affero General
* Public License Version 3 (the "License") or (at your option)
* any later version. You may not use this file except in
* compliance with the License. You may obtain a copy of the
* License at:
*
* https://www.gnu.org/licenses/agpl-3.0.txt
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific
* language governing permissions and limitations under the
* License.
*/

package org.opennms.netmgt.measurements.filters.impl;

import org.apache.commons.io.IOUtils;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

public class LoadData {

protected final static ObjectMapper mapper = new ObjectMapper();

static {
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES,true);
mapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS,true);
}

private static final String firstArrayOfValue=
"[{Load: NaN, timestamp: 1.732623E12}, {Load: 2.3, timestamp: 1.7326233E12}, {Load: 1.8334154666666667, timestamp: 1.7326236E12}, {Load: 1.6969653333333334, timestamp: 1.7326239E12}, {Load: 1.5994372000000001, timestamp: 1.7326242E12}, {Load: 1.4341295333333333, timestamp: 1.7326245E12}, {Load: 1.4953480666666668, timestamp: 1.7326248E12}, {Load: 1.4459626666666665, timestamp: 1.7326251E12}, {Load: 1.4572672333333334, timestamp: 1.7326254E12}, " +
"{Load: 1.6113477666666667, timestamp: 1.7326257E12}, {Load: 1.504247, timestamp: 1.732626E12}, {Load: 1.3507988333333334, timestamp: 1.7326263E12}, {Load: 1.2941548, timestamp: 1.7326266E12}, {Load: 1.477585, timestamp: 1.7326269E12}, {Load: 1.7080315000000001, timestamp: 1.7326272E12}, {Load: 2.36085, timestamp: 1.7326275E12}, {Load: 2.9174625, timestamp: 1.7326278E12}, {Load: 2.732814266666667, timestamp: 1.7326281E12}, " +
"{Load: 2.2755045000000003, timestamp: 1.7326284E12}, {Load: 1.6708403, timestamp: 1.7326287E12}, {Load: 1.4709091666666667, timestamp: 1.732629E12}, {Load: 1.5814612333333335, timestamp: 1.7326293E12}, {Load: 1.345355, timestamp: 1.7326296E12}, {Load: 1.5043948, timestamp: 1.7326299E12}, {Load: 2.0734238333333335, timestamp: 1.7326302E12}, {Load: 1.8433515000000003, timestamp: 1.7326305E12}, {Load: 1.4480430666666666, timestamp: 1.7326308E12}, " +
"{Load: 1.5140407, timestamp: 1.7326311E12}, {Load: 1.6470922666666667, timestamp: 1.7326314E12}, {Load: 1.5975026666666667, timestamp: 1.7326317E12}, {Load: 1.5934968999999999, timestamp: 1.732632E12}, {Load: 1.4603045666666667, timestamp: 1.7326323E12}, {Load: 1.7643385999999999, timestamp: 1.7326326E12}, {Load: 1.657308533333333, timestamp: 1.7326329E12}, {Load: 1.6436730000000002, timestamp: 1.7326332E12}, {Load: 1.8063498666666669, timestamp: 1.7326335E12}, " +
"{Load: 1.5891790000000001, timestamp: 1.7326338E12}, {Load: 1.3490829, timestamp: 1.7326341E12}, {Load: 1.4693056, timestamp: 1.7326344E12}, {Load: 1.5882341333333334, timestamp: 1.7326347E12}, {Load: 1.8092199, timestamp: 1.732635E12}, {Load: 1.7346240000000002, timestamp: 1.7326353E12}, {Load: 1.5345828000000001, timestamp: 1.7326356E12}, {Load: 1.6975562666666666, timestamp: 1.7326359E12}, {Load: 1.9960614666666667, timestamp: 1.7326362E12}, " +
"{Load: 1.8785454000000001, timestamp: 1.7326365E12}, {Load: 1.690535, timestamp: 1.7326368E12}, {Load: 2.0739177333333334, timestamp: 1.7326371E12}, {Load: 2.249542466666667, timestamp: 1.7326374E12}, {Load: 1.9728408333333336, timestamp: 1.7326377E12}, {Load: 1.7000496666666667, timestamp: 1.732638E12}, {Load: 1.7387256666666668, timestamp: 1.7326383E12}, {Load: 1.8121746, timestamp: 1.7326386E12}, {Load: 1.5660853333333333, timestamp: 1.7326389E12}, " +
"{Load: 1.4647701999999998, timestamp: 1.7326392E12}, {Load: 1.8838901333333333, timestamp: 1.7326395E12}, {Load: 1.6907516, timestamp: 1.7326398E12}, {Load: 1.4694273333333334, timestamp: 1.7326401E12}, {Load: 1.4158627, timestamp: 1.7326404E12}, {Load: 1.2887408333333332, timestamp: 1.7326407E12}, {Load: 1.4300443999999999, timestamp: 1.732641E12}, {Load: 1.4444888000000002, timestamp: 1.7326413E12}, {Load: 1.4054076, timestamp: 1.7326416E12}, " +
"{Load: 1.7406400666666668, timestamp: 1.7326419E12}, {Load: 1.9189550333333334, timestamp: 1.7326422E12}, {Load: 1.7359551999999998, timestamp: 1.7326425E12}, {Load: 1.7357926, timestamp: 1.7326428E12}, {Load: 2.0664267333333335, timestamp: 1.7326431E12}, {Load: 1.7626766666666667, timestamp: 1.7326434E12}, {Load: 1.64, timestamp: 1.7326437E12}, {Load: 1.6250301, timestamp: 1.732644E12}, {Load: 1.7646332666666666, timestamp: 1.7326443E12}, " +
"{Load: 1.4316430000000002, timestamp: 1.7326446E12}, {Load: 1.37, timestamp: 1.7326449E12}]";

private final static String secondArrayOfValue=
"[{Load: 1.5100000000000002, timestamp: 1.7326236E12}, {Load: 1.6969653333333334, timestamp: 1.7326239E12}, {Load: 1.5994372000000001, timestamp: 1.7326242E12}, {Load: 1.4341295333333333, timestamp: 1.7326245E12}, {Load: 1.4953480666666668, timestamp: 1.7326248E12}, {Load: 1.4459626666666665, timestamp: 1.7326251E12}, {Load: 1.4572672333333334, timestamp: 1.7326254E12}, {Load: 1.6113477666666667, timestamp: 1.7326257E12}, {Load: 1.504247, timestamp: 1.732626E12}, " +
"{Load: 1.3507988333333334, timestamp: 1.7326263E12}, {Load: 1.2941548, timestamp: 1.7326266E12}, {Load: 1.477585, timestamp: 1.7326269E12}, {Load: 1.7080315000000001, timestamp: 1.7326272E12}, {Load: 2.36085, timestamp: 1.7326275E12}, {Load: 2.9174625, timestamp: 1.7326278E12}, {Load: 2.732814266666667, timestamp: 1.7326281E12}, {Load: 2.2755045000000003, timestamp: 1.7326284E12}, {Load: 1.6708403, timestamp: 1.7326287E12}, {Load: 1.4709091666666667, timestamp: 1.732629E12}, " +
"{Load: 1.5814612333333335, timestamp: 1.7326293E12}, {Load: 1.345355, timestamp: 1.7326296E12}, {Load: 1.5043948, timestamp: 1.7326299E12}, {Load: 2.0734238333333335, timestamp: 1.7326302E12}, {Load: 1.8433515000000003, timestamp: 1.7326305E12}, {Load: 1.4480430666666666, timestamp: 1.7326308E12}, {Load: 1.5140407, timestamp: 1.7326311E12}, {Load: 1.6470922666666667, timestamp: 1.7326314E12}, {Load: 1.5975026666666667, timestamp: 1.7326317E12}, {Load: 1.5934968999999999, timestamp: 1.732632E12}, " +
"{Load: 1.4603045666666667, timestamp: 1.7326323E12}, {Load: 1.7643385999999999, timestamp: 1.7326326E12}, {Load: 1.657308533333333, timestamp: 1.7326329E12}, {Load: 1.6436730000000002, timestamp: 1.7326332E12}, {Load: 1.8063498666666669, timestamp: 1.7326335E12}, {Load: 1.5891790000000001, timestamp: 1.7326338E12}, {Load: 1.3490829, timestamp: 1.7326341E12}, {Load: 1.4693056, timestamp: 1.7326344E12}, {Load: 1.5882341333333334, timestamp: 1.7326347E12}, {Load: 1.8092199, timestamp: 1.732635E12}, " +
"{Load: 1.7346240000000002, timestamp: 1.7326353E12}, {Load: 1.5345828000000001, timestamp: 1.7326356E12}, {Load: 1.6975562666666666, timestamp: 1.7326359E12}, {Load: 1.9960614666666667, timestamp: 1.7326362E12}, {Load: 1.8785454000000001, timestamp: 1.7326365E12}, {Load: 1.690535, timestamp: 1.7326368E12}, {Load: 2.0739177333333334, timestamp: 1.7326371E12}, {Load: 2.249542466666667, timestamp: 1.7326374E12}, {Load: 1.9728408333333336, timestamp: 1.7326377E12}, {Load: 1.7000496666666667, timestamp: 1.732638E12}, " +
"{Load: 1.7387256666666668, timestamp: 1.7326383E12}, {Load: 1.8121746, timestamp: 1.7326386E12}, {Load: 1.5660853333333333, timestamp: 1.7326389E12}, {Load: 1.4647701999999998, timestamp: 1.7326392E12}, {Load: 1.8838901333333333, timestamp: 1.7326395E12}, {Load: 1.6907516, timestamp: 1.7326398E12}, {Load: 1.4694273333333334, timestamp: 1.7326401E12}, {Load: 1.4158627, timestamp: 1.7326404E12}, {Load: 1.2887408333333332, timestamp: 1.7326407E12}, {Load: 1.4300443999999999, timestamp: 1.732641E12}, " +
"{Load: 1.4444888000000002, timestamp: 1.7326413E12}, {Load: 1.4054076, timestamp: 1.7326416E12}, {Load: 1.7406400666666668, timestamp: 1.7326419E12}, {Load: 1.9189550333333334, timestamp: 1.7326422E12}, {Load: 1.7359551999999998, timestamp: 1.7326425E12}, {Load: 1.7357926, timestamp: 1.7326428E12}, {Load: 2.0664267333333335, timestamp: 1.7326431E12}, {Load: 1.7626766666666667, timestamp: 1.7326434E12}, {Load: 1.64, timestamp: 1.7326437E12}, {Load: 1.6250301, timestamp: 1.732644E12}, {Load: 1.7646332666666666, timestamp: 1.7326443E12}, " +
"{Load: 1.4316430000000002, timestamp: 1.7326446E12}, {Load: 1.5742506666666667, timestamp: 1.7326449E12}, {Load: 1.72, timestamp: 1.7326452E12}, {Load: NaN, timestamp: 1.7326455E12}]";

protected static Entry[] getEntries(String json) throws IOException {
return mapper.readValue(json, Entry[].class);
}


protected static Entry[] getFirst() throws IOException {
return getEntries(firstArrayOfValue);
}

protected static Entry[] getSecond() throws IOException {
return getEntries(secondArrayOfValue);
}

protected static Entry[] getThird() throws IOException {
FileInputStream fis = new FileInputStream("src/test/resources/trendline.txt");
return getEntries(IOUtils.toString(fis, StandardCharsets.UTF_8));
}

protected static class Entry {
@JsonProperty("Load")
public Double load;
@JsonProperty("timestamp")
public Double timestamp;
}

}
Loading