Skip to content

Commit

Permalink
Merge pull request #38 from rpauly18/pecos_update
Browse files Browse the repository at this point in the history 
Pecos update
  • Loading branch information
@rpauly18
rpauly18 authored Nov 3, 2020
2 parents 8acbf14 + 4924a87 commit 14f4f6b
Show file tree
Hide file tree
Showing 10 changed files with 642 additions and 676 deletions.
1,033 changes: 515 additions & 518 deletions examples/qc_example.html
View file

Large diffs are not rendered by default.

Binary file modified examples/qc_example.mlx
View file
Binary file not shown.
Binary file modified mhkit.mltbx
View file
Binary file not shown.
28 changes: 13 additions & 15 deletions mhkit/qc/check_corrupt.m
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
function results = check_corrupt(data,vals,varargin)
function results = check_corrupt(data,vals,options)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Check for data that is outside expected range
Expand Down Expand Up @@ -27,13 +27,13 @@
% key: string (optional)
% Data column name or translation dictionary key. If not specified
% or set to py.None, all columns are used for test.
% to call: check_corrupt(data,vals,"key",key)
%
% min_failures: int (optional)
% Minimum number of consecutive failures required for reporting
% default = 1
% to call: check_corrupt(data,vals,"min_failures",min_failures)
%
% Must set previous arguments to use later optional arguments
% (i.e. must set key to use min_failures).
%
% Returns
% ---------
Expand All @@ -51,26 +51,24 @@
% Same as input times
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

arguments
data
vals
options.key = py.None;
options.min_failures = 1;
end

py.importlib.import_module('pecos');

% check to see if a pandas dataframe or not
if (isa(data,'py.pandas.core.frame.DataFrame')~=1)
data=qc_data_to_dataframe(data);
end
vals = py.list(vals);
% Must use optional qc arguments in order, for now.
if nargin == 2

r = struct(py.pecos.monitoring.check_corrupt(data,vals));
elseif nargin == 3
r = struct(py.pecos.monitoring.check_corrupt(data,vals,varargin{1}));
elseif nargin == 4
r = struct(py.pecos.monitoring.check_corrupt(data,vals,varargin{1},varargin{2}));
else
ME = MException('MATLAB:qc_corrupt','incorrect number of arguments (2 to 4)');
throw(ME);
end

r = struct(py.pecos.monitoring.check_corrupt(data,vals,pyargs("key",options.key,...
"min_failures",int32(options.min_failures))));
% Convert to qcdata structure
results.values=double(r.cleaned_data.values);
results.mask=int64(r.mask.values);
Expand Down
53 changes: 18 additions & 35 deletions mhkit/qc/check_delta.m
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
function results = check_delta(data,bound,varargin)
function results = check_delta(data,bound,options)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Check for stagnant data and/or abrupt changes in the data using
Expand Down Expand Up @@ -26,25 +26,21 @@
% key: string (optional)
% Data column name or translation dictionary key. If not specified
% or set to py.None, all columns are used for test.
%
% window: int (optional)
% Size of rolling window (in seconds) used to compute delta
% default = 3600
% to call: check_delta(data,bound,"key",key)
%
% direction: string (optional)
% Options: 'positive', 'negative', or py.None (default)
% If direction is positive, then only identify positive deltas
% (the min occurs before the max)
% If direction is negative, then only identify negative deltas (the max occurs before the min)
% If direction is py.None, then identify both positive and negative deltas
% to call: check_delta(data,bound,"direction",direction)
%
% min_failures: int (optional)
%
% Minimum number of consecutive failures required for reporting,
% default = 1
% to call: check_delta(data,bound,"min_failures",min_failures)
%
% Must set previous arguments to use later optional arguments
% (i.e. must set key to use min_failures).
%
% Returns
% ---------
Expand All @@ -63,40 +59,27 @@
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

arguments
data
bound
options.key = py.None;
options.direction = py.None;
options.min_failures = 1;
end

py.importlib.import_module('pecos');


% check to see if a pandas dataframe or not
if (isa(data,'py.pandas.core.frame.DataFrame')~=1)
data=qc_data_to_dataframe(data);
end
bound = py.list(bound);
if nargin == 2

r = struct(py.pecos.monitoring.check_delta(data,bound));
elseif nargin == 3

r = struct(py.pecos.monitoring.check_delta(data,bound,...
varargin{1}));
elseif nargin == 4

r = struct(py.pecos.monitoring.check_delta(data,bound,...
varargin{1},varargin{2}));
elseif nargin == 5

r = struct(py.pecos.monitoring.check_delta(data,bound,...
varargin{1},varargin{2},varargin{3}));
elseif nargin == 6

r = struct(py.pecos.monitoring.check_delta(data,bound,...
varargin{1},varargin{2},varargin{3},varargin{4}));
elseif nargin == 7

r = struct(py.pecos.monitoring.check_delta(data,bound,...
varargin{1},varargin{2},varargin{3},varargin{4},varargin{5}));
else
ME = MException('MATLAB:qc_delta','incorrect number of arguments (2 to 7)');
throw(ME);
end


r = struct(py.pecos.monitoring.check_delta(data,bound,...
pyargs("key",options.key,"direction",options.direction,...
"min_failures",int32(options.min_failures))));

% Convert to qcdata structure
results.values=double(r.cleaned_data.values);
Expand Down
42 changes: 19 additions & 23 deletions mhkit/qc/check_increment.m
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
function results = check_increment(data,bound,varargin)
function results = check_increment(data,bound,options)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Check data increments using the difference between values
Expand All @@ -25,19 +25,21 @@
% key: string (optional)
% Data column name or translation dictionary key. If not specified
% or set to py.None, all columns are used for test.
% to call: check_increment(data,bound,"key",key)
%
% increment: int (optional)
% Time step shift used to compute difference, default = 1
% to call: check_increment(data,bound,"increment",increment)
%
% absolute_value: logical (optional)
% Use the absolute value of increment data, default = 1 (True)
% Use the absolute value of increment data, default = py.True
% to call: check_increment(data,bound,"absolute_value",absolute_value)
%
% min_failures: int (optional)
% Minimum number of consecutive failures required for reporting
% default = 1
% to call: check_increment(data,bound,"min_failures",min_failures)
%
% Must set previous arguments to use later optional arguments
% (i.e. must set key to use min_failures).
%
% Returns
% ---------
Expand All @@ -55,33 +57,27 @@
% Same as input times
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

arguments
data
bound
options.key = py.None;
options.increment = 1;
options.absolute_value = py.True;
options.min_failures = 1;
end

py.importlib.import_module('pecos');

% check to see if a pandas dataframe or not
if (isa(data,'py.pandas.core.frame.DataFrame')~=1)
data=qc_data_to_dataframe(data);
end
bound=py.list(bound);
if nargin == 2

r = struct(py.pecos.monitoring.check_increment(data,bound));
elseif nargin == 3
r = struct(py.pecos.monitoring.check_increment(data,bound,...
varargin{1}));
elseif nargin == 4
r = struct(py.pecos.monitoring.check_increment(data,bound,...
varargin{1},varargin{2}));
elseif nargin == 5
r = struct(py.pecos.monitoring.check_increment(data,bound,...
varargin{1},varargin{2},varargin{3}));
elseif nargin == 6
r = struct(py.pecos.monitoring.check_increment(data,bound,...
varargin{1},varargin{2},varargin{3},varargin{4}));
else
ME = MException('MATLAB:qc_increment','incorrect number of arguments (2 to 6)');
throw(ME);
end

r = struct(py.pecos.monitoring.check_increment(data,bound,...
pyargs("key",options.key,"increment",int32(options.increment),...
"absolute_value",options.absolute_value,"min_failures",int32(options.min_failures))));
% Convert to qcdata structure
results.values=double(r.cleaned_data.values);
results.mask=int64(r.mask.values);
Expand Down
26 changes: 11 additions & 15 deletions mhkit/qc/check_missing.m
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
function results = check_missing(data)
function results = check_missing(data,options)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Check for missing data
%
Expand All @@ -20,13 +20,13 @@
% key: string (optional)
% Data column name or translation dictionary key. If not specified
% or set to py.None, all columns are used for test.
% to call: check_missing(data,"key",key)
%
% min_failures: int (optional)
% Minimum number of consecutive failures required for reporting,
% default = 1
% to call: check_missing(data,"min_failures",min_failures)
%
% Must set previous arguments to use later optional arguments
% (i.e. must set key to use min_failures).
%
% Returns
% ---------
Expand All @@ -45,25 +45,21 @@
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

arguments
data
options.key = py.None;
options.min_failures = 1;
end

py.importlib.import_module('pecos');

% check to see if a pandas dataframe or not
if (isa(data,'py.pandas.core.frame.DataFrame')~=1)
data=qc_data_to_dataframe(data);
end

if nargin == 1
r = struct(py.pecos.monitoring.check_missing(data));
elseif nargin == 2
r = struct(py.pecos.monitoring.check_missing(data,...
varargin{1}));
elseif nargin == 3
r = struct(py.pecos.monitoring.check_missing(data,...
varargin{1},varargin{2}));
else
ME = MException('MATLAB:qc_missing','incorrect number of arguments (1 to 3)');
throw(ME);
end
r = struct(py.pecos.monitoring.check_missing(data,...
pyargs("key",options.key,"min_failures",int32(options.min_failures))));

% Convert to qcdata structure
results.values=double(r.cleaned_data.values);
Expand Down
45 changes: 22 additions & 23 deletions mhkit/qc/check_outlier.m
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
function results = check_outlier(data,bound,varargin)
function results = check_outlier(data,bound,options)

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Check or outliers using normalized data within a rolling window
Expand Down Expand Up @@ -27,22 +27,26 @@
% key: string (optional)
% Data column name or translation dictionary key. If not specified
% or set to py.None, all columns are used for test.
% to call: check_outlier(data,bound,"key",key)
%
% window: int (optional)
% Size of rolling window (in seconds) used to normalize data
% default = 3600. If window is set to py.None, data is normalized
% using mean and stddev of entire data set (column by column)
% to call: check_outlier(data,bound,"window",window)
%
% absolute_value: logical (optional)
% Use the absolute value of the normalized data, default = 1 (True)
% Use the absolute value of the normalized data, default = py.True
% to call: check_outlier(data,bound,"absolute_value",absolute_value)
%
% min_failures: int (optional)
%
% Minimum number of consecutive failures required for reporting,
% default = 1
% to call: check_outlier(data,bound,"min_failures",min_failures)
%
% Must set previous arguments to use later optional arguments
% (i.e. must set key to use min_failures).
% streaming: logical (optional)
% Indicates if streaming analysis should be used, default = py.False
% to call: check_outlier(data,bound,"streaming",streaming)
%
% Returns
% ---------
Expand All @@ -60,6 +64,15 @@
% Same as input times
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
arguments
data
bound
options.key = py.None;
options.window = 3600;
options.absolute_value = py.True;
options.min_failures = 1;
options.streaming = py.False;
end

py.importlib.import_module('pecos');

Expand All @@ -68,25 +81,11 @@
data=qc_data_to_dataframe(data);
end
bound = py.list(bound);
if nargin == 2
r = struct(py.pecos.monitoring.check_outlier(data,bound));
elseif nargin == 3
r = struct(py.pecos.monitoring.check_outlier(data,bound,...
varargin{1}));
elseif nargin == 4
r = struct(py.pecos.monitoring.check_outlier(data,bound,...
varargin{1},varargin{2}));
elseif nargin == 5
r = struct(py.pecos.monitoring.check_outlier(data,bound,...
varargin{1},varargin{2},varargin{3}));
elseif nargin == 6
r = struct(py.pecos.monitoring.check_outlier(data,bound,...
varargin{1},varargin{2},varargin{3},varargin{4}));
else
ME = MException('MATLAB:qc_outlier','incorrect number of arguments (2 to 7)');
throw(ME);
end

r = struct(py.pecos.monitoring.check_outlier(data,bound,...
pyargs("key",options.key,"window",options.window,...
"absolute_value",options.absolute_value,"min_failures",int32(options.min_failures),...
"streaming",options.streaming)));
% Convert to qcdata structure
results.values=double(r.cleaned_data.values);
results.mask=int64(r.mask.values);
Expand Down
Loading

0 comments on commit 14f4f6b

Please sign in to comment.