Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

NMS-17066: Fix Measurement TrendLine #7597

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

NMS-17066: Fix Measurement TrendLine #7597

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 18 additions & 20 deletions ...asurements/impl/src/main/java/org/opennms/netmgt/measurements/filters/impl/TrendLine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,35 +21,20 @@
*/
package org.opennms.netmgt.measurements.filters.impl;

import java.util.Date;
import java.util.Map;

import com.google.common.base.Preconditions;
import com.google.common.collect.RowSortedTable;
import org.apache.commons.math3.analysis.polynomials.PolynomialFunction;
import org.apache.commons.math3.fitting.PolynomialCurveFitter;
import org.apache.commons.math3.fitting.WeightedObservedPoints;
import org.apache.commons.math3.fitting.leastsquares.EvaluationRmsChecker;
import org.apache.commons.math3.fitting.leastsquares.GaussNewtonOptimizer;
import org.apache.commons.math3.fitting.leastsquares.LeastSquaresBuilder;
import org.apache.commons.math3.fitting.leastsquares.LeastSquaresOptimizer;
import org.apache.commons.math3.fitting.leastsquares.LeastSquaresProblem;
import org.apache.commons.math3.fitting.leastsquares.LevenbergMarquardtOptimizer;
import org.apache.commons.math3.linear.ArrayRealVector;
import org.apache.commons.math3.linear.RealVector;
import org.opennms.netmgt.integrations.R.RScriptException;
import org.opennms.netmgt.integrations.R.RScriptExecutor;
import org.opennms.netmgt.integrations.R.RScriptInput;
import org.opennms.netmgt.integrations.R.RScriptOutput;
import org.opennms.netmgt.measurements.api.Filter;
import org.opennms.netmgt.measurements.api.FilterInfo;
import org.opennms.netmgt.measurements.api.FilterParam;
import org.opennms.netmgt.measurements.filters.impl.Utils.TableLimits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableTable;
import com.google.common.collect.Maps;
import com.google.common.collect.RowSortedTable;
import java.util.Date;

/**
* Fits a trend line to the samples in a column.
Expand Down Expand Up @@ -83,11 +68,13 @@ public TrendLine(String outputColumn, String inputColumn, long secondsAhead, int

@Override
public void filter(RowSortedTable<Long, String, Double> table) throws RScriptException {
LOG.debug("filter: values\n{}", table.rowMap().values());
Preconditions.checkArgument(table.containsColumn(TIMESTAMP_COLUMN_NAME), String.format("Data source must have a '%s' column.", Filter.TIMESTAMP_COLUMN_NAME));

// Determine the index of the first and last non-NaN values
// Assume the values between these are contiguous
TableLimits limits = Utils.getRowsWithValues(table, m_inputColumn);
LOG.info("filter: limits: {}, {}", limits.firstRowWithValues, limits.lastRowWithValues);

// Make sure we have some samples
long numSampleRows = limits.lastRowWithValues - limits.firstRowWithValues;
Expand All @@ -98,14 +85,20 @@ public void filter(RowSortedTable<Long, String, Double> table) throws RScriptExc

// Gather the values [(x,y),...]
final WeightedObservedPoints obs = new WeightedObservedPoints();
for (long i = limits.firstRowWithValues; i < limits.lastRowWithValues; i++) {
obs.add(table.get(i, TIMESTAMP_COLUMN_NAME), table.get(i, m_inputColumn));
for (long i = limits.firstRowWithValues; i <= limits.lastRowWithValues; i++) {
Double value = table.get(i,m_inputColumn);
Copy link
Contributor

@christianpape christianpape Jan 23, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
Double value = table.get(i,m_inputColumn);
Double value = table.get(i, m_inputColumn);

Double ts = table.get(i, TIMESTAMP_COLUMN_NAME);
if (ts != null && !Double.isNaN(ts) && value != null && !Double.isNaN(value)) {
obs.add(ts,value);
}
}
LOG.info("filter: WeightedObservedPoints: {}", obs.toList().size());

// Fit the polynomial
final PolynomialCurveFitter fitter = PolynomialCurveFitter.create(m_polynomialOrder);
final double[] coeff = fitter.fit(obs.toList());
PolynomialFunction polynomialFunction = new PolynomialFunction(coeff);
LOG.info("filter: polynomialFunction: {}", polynomialFunction);

// Determine the step size
Date lastTimestamp = new Date(table.get(limits.lastRowWithValues, TIMESTAMP_COLUMN_NAME).longValue());
Expand All @@ -116,6 +109,9 @@ public void filter(RowSortedTable<Long, String, Double> table) throws RScriptExc
numStepsAhead = Math.max(1, numStepsAhead);

// Calculate the value of the polynomial for all the samples and the requested number of steps ahead
LOG.info("filter: limits.firstRowWithValues:{}", limits.firstRowWithValues);
LOG.info("filter: limits.lastRowWithValues:{}", limits.lastRowWithValues);
LOG.info("filter: numStepsAhead:{}", numStepsAhead);
for (long i = limits.firstRowWithValues; i <= (limits.lastRowWithValues + numStepsAhead); i++) {
if (i >= limits.lastRowWithValues) {
table.put(i, TIMESTAMP_COLUMN_NAME, (double)new Date(lastTimestamp.getTime() + stepInMs * (i-limits.lastRowWithValues)).getTime());
Expand All @@ -124,5 +120,7 @@ public void filter(RowSortedTable<Long, String, Double> table) throws RScriptExc
// Compute the value
table.put(i, m_outputColumn, polynomialFunction.value(timestamp));
}
LOG.debug("filter: values:\n{}", table.rowMap().values());

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

}
}
95 changes: 95 additions & 0 deletions ...easurements/impl/src/test/java/org/opennms/netmgt/measurements/filters/impl/LoadData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*******************************************************************************
* This file is part of OpenNMS(R).
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

copyright headers can be updated to new header that is independent of year.
see copyright header on TrendLine.java

*
* Copyright (C) 2016 The OpenNMS Group, Inc.
* OpenNMS(R) is Copyright (C) 1999-2016 The OpenNMS Group, Inc.
*
* OpenNMS(R) is a registered trademark of The OpenNMS Group, Inc.
*
* OpenNMS(R) is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License,
* or (at your option) any later version.
*
* OpenNMS(R) is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with OpenNMS(R). If not, see:
* http://www.gnu.org/licenses/
*
* For more information contact:
* OpenNMS(R) Licensing <[email protected]>
* http://www.opennms.org/
* http://www.opennms.com/
*******************************************************************************/

package org.opennms.netmgt.measurements.filters.impl;

import org.apache.commons.io.IOUtils;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.FileInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

public class LoadData {

protected final static ObjectMapper mapper = new ObjectMapper();

static {
mapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES,true);
mapper.configure(JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS,true);
}

private static final String firstArrayOfValue=
"[{Load: NaN, timestamp: 1.732623E12}, {Load: 2.3, timestamp: 1.7326233E12}, {Load: 1.8334154666666667, timestamp: 1.7326236E12}, {Load: 1.6969653333333334, timestamp: 1.7326239E12}, {Load: 1.5994372000000001, timestamp: 1.7326242E12}, {Load: 1.4341295333333333, timestamp: 1.7326245E12}, {Load: 1.4953480666666668, timestamp: 1.7326248E12}, {Load: 1.4459626666666665, timestamp: 1.7326251E12}, {Load: 1.4572672333333334, timestamp: 1.7326254E12}, " +
"{Load: 1.6113477666666667, timestamp: 1.7326257E12}, {Load: 1.504247, timestamp: 1.732626E12}, {Load: 1.3507988333333334, timestamp: 1.7326263E12}, {Load: 1.2941548, timestamp: 1.7326266E12}, {Load: 1.477585, timestamp: 1.7326269E12}, {Load: 1.7080315000000001, timestamp: 1.7326272E12}, {Load: 2.36085, timestamp: 1.7326275E12}, {Load: 2.9174625, timestamp: 1.7326278E12}, {Load: 2.732814266666667, timestamp: 1.7326281E12}, " +
"{Load: 2.2755045000000003, timestamp: 1.7326284E12}, {Load: 1.6708403, timestamp: 1.7326287E12}, {Load: 1.4709091666666667, timestamp: 1.732629E12}, {Load: 1.5814612333333335, timestamp: 1.7326293E12}, {Load: 1.345355, timestamp: 1.7326296E12}, {Load: 1.5043948, timestamp: 1.7326299E12}, {Load: 2.0734238333333335, timestamp: 1.7326302E12}, {Load: 1.8433515000000003, timestamp: 1.7326305E12}, {Load: 1.4480430666666666, timestamp: 1.7326308E12}, " +
"{Load: 1.5140407, timestamp: 1.7326311E12}, {Load: 1.6470922666666667, timestamp: 1.7326314E12}, {Load: 1.5975026666666667, timestamp: 1.7326317E12}, {Load: 1.5934968999999999, timestamp: 1.732632E12}, {Load: 1.4603045666666667, timestamp: 1.7326323E12}, {Load: 1.7643385999999999, timestamp: 1.7326326E12}, {Load: 1.657308533333333, timestamp: 1.7326329E12}, {Load: 1.6436730000000002, timestamp: 1.7326332E12}, {Load: 1.8063498666666669, timestamp: 1.7326335E12}, " +
"{Load: 1.5891790000000001, timestamp: 1.7326338E12}, {Load: 1.3490829, timestamp: 1.7326341E12}, {Load: 1.4693056, timestamp: 1.7326344E12}, {Load: 1.5882341333333334, timestamp: 1.7326347E12}, {Load: 1.8092199, timestamp: 1.732635E12}, {Load: 1.7346240000000002, timestamp: 1.7326353E12}, {Load: 1.5345828000000001, timestamp: 1.7326356E12}, {Load: 1.6975562666666666, timestamp: 1.7326359E12}, {Load: 1.9960614666666667, timestamp: 1.7326362E12}, " +
"{Load: 1.8785454000000001, timestamp: 1.7326365E12}, {Load: 1.690535, timestamp: 1.7326368E12}, {Load: 2.0739177333333334, timestamp: 1.7326371E12}, {Load: 2.249542466666667, timestamp: 1.7326374E12}, {Load: 1.9728408333333336, timestamp: 1.7326377E12}, {Load: 1.7000496666666667, timestamp: 1.732638E12}, {Load: 1.7387256666666668, timestamp: 1.7326383E12}, {Load: 1.8121746, timestamp: 1.7326386E12}, {Load: 1.5660853333333333, timestamp: 1.7326389E12}, " +
"{Load: 1.4647701999999998, timestamp: 1.7326392E12}, {Load: 1.8838901333333333, timestamp: 1.7326395E12}, {Load: 1.6907516, timestamp: 1.7326398E12}, {Load: 1.4694273333333334, timestamp: 1.7326401E12}, {Load: 1.4158627, timestamp: 1.7326404E12}, {Load: 1.2887408333333332, timestamp: 1.7326407E12}, {Load: 1.4300443999999999, timestamp: 1.732641E12}, {Load: 1.4444888000000002, timestamp: 1.7326413E12}, {Load: 1.4054076, timestamp: 1.7326416E12}, " +
"{Load: 1.7406400666666668, timestamp: 1.7326419E12}, {Load: 1.9189550333333334, timestamp: 1.7326422E12}, {Load: 1.7359551999999998, timestamp: 1.7326425E12}, {Load: 1.7357926, timestamp: 1.7326428E12}, {Load: 2.0664267333333335, timestamp: 1.7326431E12}, {Load: 1.7626766666666667, timestamp: 1.7326434E12}, {Load: 1.64, timestamp: 1.7326437E12}, {Load: 1.6250301, timestamp: 1.732644E12}, {Load: 1.7646332666666666, timestamp: 1.7326443E12}, " +
"{Load: 1.4316430000000002, timestamp: 1.7326446E12}, {Load: 1.37, timestamp: 1.7326449E12}]";

private final static String secondArrayOfValue=
"[{Load: 1.5100000000000002, timestamp: 1.7326236E12}, {Load: 1.6969653333333334, timestamp: 1.7326239E12}, {Load: 1.5994372000000001, timestamp: 1.7326242E12}, {Load: 1.4341295333333333, timestamp: 1.7326245E12}, {Load: 1.4953480666666668, timestamp: 1.7326248E12}, {Load: 1.4459626666666665, timestamp: 1.7326251E12}, {Load: 1.4572672333333334, timestamp: 1.7326254E12}, {Load: 1.6113477666666667, timestamp: 1.7326257E12}, {Load: 1.504247, timestamp: 1.732626E12}, " +
"{Load: 1.3507988333333334, timestamp: 1.7326263E12}, {Load: 1.2941548, timestamp: 1.7326266E12}, {Load: 1.477585, timestamp: 1.7326269E12}, {Load: 1.7080315000000001, timestamp: 1.7326272E12}, {Load: 2.36085, timestamp: 1.7326275E12}, {Load: 2.9174625, timestamp: 1.7326278E12}, {Load: 2.732814266666667, timestamp: 1.7326281E12}, {Load: 2.2755045000000003, timestamp: 1.7326284E12}, {Load: 1.6708403, timestamp: 1.7326287E12}, {Load: 1.4709091666666667, timestamp: 1.732629E12}, " +
"{Load: 1.5814612333333335, timestamp: 1.7326293E12}, {Load: 1.345355, timestamp: 1.7326296E12}, {Load: 1.5043948, timestamp: 1.7326299E12}, {Load: 2.0734238333333335, timestamp: 1.7326302E12}, {Load: 1.8433515000000003, timestamp: 1.7326305E12}, {Load: 1.4480430666666666, timestamp: 1.7326308E12}, {Load: 1.5140407, timestamp: 1.7326311E12}, {Load: 1.6470922666666667, timestamp: 1.7326314E12}, {Load: 1.5975026666666667, timestamp: 1.7326317E12}, {Load: 1.5934968999999999, timestamp: 1.732632E12}, " +
"{Load: 1.4603045666666667, timestamp: 1.7326323E12}, {Load: 1.7643385999999999, timestamp: 1.7326326E12}, {Load: 1.657308533333333, timestamp: 1.7326329E12}, {Load: 1.6436730000000002, timestamp: 1.7326332E12}, {Load: 1.8063498666666669, timestamp: 1.7326335E12}, {Load: 1.5891790000000001, timestamp: 1.7326338E12}, {Load: 1.3490829, timestamp: 1.7326341E12}, {Load: 1.4693056, timestamp: 1.7326344E12}, {Load: 1.5882341333333334, timestamp: 1.7326347E12}, {Load: 1.8092199, timestamp: 1.732635E12}, " +
"{Load: 1.7346240000000002, timestamp: 1.7326353E12}, {Load: 1.5345828000000001, timestamp: 1.7326356E12}, {Load: 1.6975562666666666, timestamp: 1.7326359E12}, {Load: 1.9960614666666667, timestamp: 1.7326362E12}, {Load: 1.8785454000000001, timestamp: 1.7326365E12}, {Load: 1.690535, timestamp: 1.7326368E12}, {Load: 2.0739177333333334, timestamp: 1.7326371E12}, {Load: 2.249542466666667, timestamp: 1.7326374E12}, {Load: 1.9728408333333336, timestamp: 1.7326377E12}, {Load: 1.7000496666666667, timestamp: 1.732638E12}, " +
"{Load: 1.7387256666666668, timestamp: 1.7326383E12}, {Load: 1.8121746, timestamp: 1.7326386E12}, {Load: 1.5660853333333333, timestamp: 1.7326389E12}, {Load: 1.4647701999999998, timestamp: 1.7326392E12}, {Load: 1.8838901333333333, timestamp: 1.7326395E12}, {Load: 1.6907516, timestamp: 1.7326398E12}, {Load: 1.4694273333333334, timestamp: 1.7326401E12}, {Load: 1.4158627, timestamp: 1.7326404E12}, {Load: 1.2887408333333332, timestamp: 1.7326407E12}, {Load: 1.4300443999999999, timestamp: 1.732641E12}, " +
"{Load: 1.4444888000000002, timestamp: 1.7326413E12}, {Load: 1.4054076, timestamp: 1.7326416E12}, {Load: 1.7406400666666668, timestamp: 1.7326419E12}, {Load: 1.9189550333333334, timestamp: 1.7326422E12}, {Load: 1.7359551999999998, timestamp: 1.7326425E12}, {Load: 1.7357926, timestamp: 1.7326428E12}, {Load: 2.0664267333333335, timestamp: 1.7326431E12}, {Load: 1.7626766666666667, timestamp: 1.7326434E12}, {Load: 1.64, timestamp: 1.7326437E12}, {Load: 1.6250301, timestamp: 1.732644E12}, {Load: 1.7646332666666666, timestamp: 1.7326443E12}, " +
"{Load: 1.4316430000000002, timestamp: 1.7326446E12}, {Load: 1.5742506666666667, timestamp: 1.7326449E12}, {Load: 1.72, timestamp: 1.7326452E12}, {Load: NaN, timestamp: 1.7326455E12}]";

protected static Entry[] getEntries(String json) throws IOException {
return mapper.readValue(json, Entry[].class);
}


protected static Entry[] getFirst() throws IOException {
return getEntries(firstArrayOfValue);
}

protected static Entry[] getSecond() throws IOException {
return getEntries(secondArrayOfValue);
}

protected static Entry[] getThird() throws IOException {
FileInputStream fis = new FileInputStream("src/test/resources/trendline.txt");
return getEntries(IOUtils.toString(fis, StandardCharsets.UTF_8));
}

protected static class Entry {
@JsonProperty("Load")
public Double load;
@JsonProperty("timestamp")
public Double timestamp;
}

}
Loading
Loading