From 8c5f4a21cca186a8854e171ec0074439b7e1115a Mon Sep 17 00:00:00 2001 From: Fabian Prasser Date: Mon, 13 Jun 2022 16:08:26 +0200 Subject: [PATCH 1/2] Support statistics with or without special values through the API --- .../arx/aggregates/StatisticsBuilder.java | 259 +++++++++++++++++- .../StatisticsBuilderInterruptible.java | 110 +++++++- 2 files changed, 348 insertions(+), 21 deletions(-) diff --git a/src/main/org/deidentifier/arx/aggregates/StatisticsBuilder.java b/src/main/org/deidentifier/arx/aggregates/StatisticsBuilder.java index b4f9b6093..9630f5c27 100644 --- a/src/main/org/deidentifier/arx/aggregates/StatisticsBuilder.java +++ b/src/main/org/deidentifier/arx/aggregates/StatisticsBuilder.java @@ -130,7 +130,7 @@ public StatisticsClassification getClassificationPerformance(String[] features, interrupt, progress); } - + /** * Returns a contingency table for the given columns. * @@ -146,9 +146,33 @@ public StatisticsClassification getClassificationPerformance(String[] features, */ public StatisticsContingencyTable getContingencyTable(int column1, boolean orderFromDefinition1, int column2, boolean orderFromDefinition2) { + return getContingencyTable(column1, orderFromDefinition1, + column2, orderFromDefinition2, + true, true); + } + /** + * Returns a contingency table for the given columns. + * + * @param column1 The first column + * @param orderFromDefinition1 Indicates whether the order that should be assumed for string data items + * can (and should) be derived from the hierarchy provided in the data + * definition (if any) + * @param column2 The second column + * @param orderFromDefinition2 Indicates whether the order that should be assumed for string data items + * can (and should) be derived from the hierarchy provided in the data + * definition (if any) + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsContingencyTable getContingencyTable(int column1, boolean orderFromDefinition1, + int column2, boolean orderFromDefinition2, + boolean includeSuppressedValues, + boolean includeNullValues) { return getContingencyTable(column1, getHierarchy(column1, orderFromDefinition1), - column2, getHierarchy(column2, orderFromDefinition2)); + column2, getHierarchy(column2, orderFromDefinition2), + includeSuppressedValues, includeNullValues); } /** @@ -163,6 +187,23 @@ public StatisticsContingencyTable getContingencyTable(int column1, boolean order public StatisticsContingencyTable getContingencyTable(int column1, int column2) { return getContingencyTable(column1, true, column2, true); } + + /** + * Returns a contingency table for the given columns. This method assumes that the + * order of string data items will be derived from the hierarchies provided + * in the data definition (if any) + * + * @param column1 The first column + * @param column2 The second column + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsContingencyTable getContingencyTable(int column1, int column2, + boolean includeSuppressedValues, + boolean includeNullValues) { + return getContingencyTable(column1, true, column2, true, includeSuppressedValues, includeNullValues); + } /** * Returns a contingency table for the given columns. @@ -189,6 +230,37 @@ public StatisticsContingencyTable getContingencyTable(int column1, return getContingencyTable(column1, size1, getHierarchy(column1, orderFromDefinition1), column2, size2, getHierarchy(column2, orderFromDefinition2)); } + + /** + * Returns a contingency table for the given columns. + * + * @param column1 The first column + * @param size1 The maximal size in this dimension + * @param orderFromDefinition1 Indicates whether the order that should be assumed for string data items + * can (and should) be derived from the hierarchy provided in the data + * definition (if any) + * @param column2 The second column + * @param size2 The maximal size in this dimension + * @param orderFromDefinition2 Indicates whether the order that should be assumed for string data items + * can (and should) be derived from the hierarchy provided in the data + * definition (if any) + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsContingencyTable getContingencyTable(int column1, + int size1, + boolean orderFromDefinition1, + int column2, + int size2, + boolean orderFromDefinition2, + boolean includeSuppressedValues, + boolean includeNullValues) { + + return getContingencyTable(column1, size1, getHierarchy(column1, orderFromDefinition1), + column2, size2, getHierarchy(column2, orderFromDefinition2), + includeSuppressedValues, includeNullValues); + } /** * Returns a contingency table for the given columns. This method assumes that the @@ -207,7 +279,29 @@ public StatisticsContingencyTable getContingencyTable(int column1, int size2) { return getContingencyTable(column1, size1, true, column2, size2, true); } - + + /** + * Returns a contingency table for the given columns. This method assumes that the + * order of string data items can (and should) be derived from the hierarchies provided + * in the data definition (if any) + * + * @param column1 The first column + * @param size1 The maximal size in this dimension + * @param column2 The second column + * @param size2 The maximal size in this dimension + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsContingencyTable getContingencyTable(int column1, + int size1, + int column2, + int size2, + boolean includeSuppressedValues, + boolean includeNullValues) { + return getContingencyTable(column1, size1, true, column2, size2, true, includeSuppressedValues, includeNullValues); + } + /** * Returns a contingency table for the given columns. The order for string data items is derived * from the provided hierarchies @@ -226,6 +320,33 @@ public StatisticsContingencyTable getContingencyTable(int column1, int column2, int size2, String[][] hierarchy2) { + return getContingencyTable(column1, size1, hierarchy1, + column2, size2, hierarchy2, + true, true); + } + + /** + * Returns a contingency table for the given columns. The order for string data items is derived + * from the provided hierarchies + * + * @param column1 The first column + * @param size1 The maximal size in this dimension + * @param hierarchy1 The hierarchy for the first column, may be null + * @param column2 The second column + * @param size2 The maximal size in this dimension + * @param hierarchy2 The hierarchy for the second column, may be null + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsContingencyTable getContingencyTable(int column1, + int size1, + String[][] hierarchy1, + int column2, + int size2, + String[][] hierarchy2, + boolean includeSuppressedValues, + boolean includeNullValues) { // Reset stop flag interrupt.value = false; @@ -239,7 +360,9 @@ public StatisticsContingencyTable getContingencyTable(int column1, StatisticsContingencyTable table = getContingencyTable(column1, hierarchy1, column2, - hierarchy2); + hierarchy2, + includeSuppressedValues, + includeNullValues); // Check if suitable if (table.values1.length <= size1 && @@ -325,7 +448,7 @@ public void remove() { // Result result return new StatisticsContingencyTable(values1, values2, table.count, max, iterator); } - + /** * Returns a contingency table for the given columns. The order for string data items is derived * from the provided hierarchies @@ -340,13 +463,34 @@ public StatisticsContingencyTable getContingencyTable(int column1, String[][] hierarchy1, int column2, String[][] hierarchy2) { + return getContingencyTable(column1, hierarchy1, column2, hierarchy2, true, true); + } + + /** + * Returns a contingency table for the given columns. The order for string data items is derived + * from the provided hierarchies + * + * @param column1 The first column + * @param hierarchy1 The hierarchy for the first column, may be null + * @param column2 The second column + * @param hierarchy2 The hierarchy for the second column, may be null + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsContingencyTable getContingencyTable(int column1, + String[][] hierarchy1, + int column2, + String[][] hierarchy2, + boolean includeSuppressedValues, + boolean includeNullValues) { // Reset stop flag interrupt.value = false; // Init - String[] values1 = getDistinctValuesOrdered(column1, hierarchy1); - String[] values2 = getDistinctValuesOrdered(column2, hierarchy2); + String[] values1 = getDistinctValuesOrdered(column1, hierarchy1, includeSuppressedValues, includeNullValues); + String[] values2 = getDistinctValuesOrdered(column2, hierarchy2, includeSuppressedValues, includeNullValues); // Create maps of indexes Map indexes1 = new HashMap(); @@ -365,8 +509,11 @@ public StatisticsContingencyTable getContingencyTable(int column1, final Map entries = new HashMap(); for (int row = 0; row < handle.getNumRows(); row++) { checkInterrupt(); - int index1 = indexes1.get(handle.getValue(row, column1)); - int index2 = indexes2.get(handle.getValue(row, column2)); + Integer index1 = indexes1.get(handle.getValue(row, column1)); + Integer index2 = indexes2.get(handle.getValue(row, column2)); + if (index1 == null || index2 == null) { + continue; + } Entry entry = new Entry(index1, index2); Integer previous = entries.get(entry); int value = previous != null ? previous + 1 : 1; @@ -421,12 +568,43 @@ public void remove() { * @return */ public String[] getDistinctValues(int column) { - return this.handle.getDistinctValues(column, new InterruptHandler() { + return this.getDistinctValues(column, true, true); + } + + /** + * Returns the distinct set of data items from the given column. + * + * @param column The column + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public String[] getDistinctValues(int column, + boolean includeSuppressedValues, + boolean includeNullValues) { + + // Get from handle + String[] list = this.handle.getDistinctValues(column, new InterruptHandler() { @Override public void checkInterrupt() { StatisticsBuilder.this.checkInterrupt(); } }); + + // Remove unwanted values + if (!includeNullValues || !includeSuppressedValues) { + List result = new ArrayList<>(); + for (String s : list) { + if ((includeSuppressedValues || !DataType.isAny(s)) && + (includeNullValues || !DataType.isNull(s))) { + result.add(s); + } + } + list = result.toArray(new String[result.size()]); + } + + // Done + return list; } /** @@ -453,7 +631,7 @@ public String[] getDistinctValuesOrdered(int column) { public String[] getDistinctValuesOrdered(int column, boolean orderFromDefinition) { return getDistinctValuesOrdered(column, getHierarchy(column, orderFromDefinition)); } - + /** * Returns an ordered list of the distinct set of data items from the given column. This method assumes * that the order of string data items can (and should) be derived from the provided hierarchy @@ -463,6 +641,23 @@ public String[] getDistinctValuesOrdered(int column, boolean orderFromDefinition * @return */ public String[] getDistinctValuesOrdered(int column, String[][] hierarchy) { + return this.getDistinctValuesOrdered(column, hierarchy, true, true); + } + + /** + * Returns an ordered list of the distinct set of data items from the given column. This method assumes + * that the order of string data items can (and should) be derived from the provided hierarchy + * + * @param column The column + * @param hierarchy The hierarchy, may be null + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public String[] getDistinctValuesOrdered(int column, + String[][] hierarchy, + boolean includeSuppressedValues, + boolean includeNullValues) { // Reset stop flag interrupt.value = false; @@ -557,6 +752,18 @@ public String[] getDistinctValuesOrdered(int column, String[][] hierarchy) { progress.value = 40; + // Remove unwanted values + if (!includeNullValues || !includeSuppressedValues) { + List result = new ArrayList<>(); + for (String s : list) { + if ((includeSuppressedValues || !DataType.isAny(s)) && + (includeNullValues || !DataType.isNull(s))) { + result.add(s); + } + } + list = result.toArray(new String[result.size()]); + } + // Done return list; } @@ -667,13 +874,34 @@ public StatisticsFrequencyDistribution getFrequencyDistribution(int column, bool * @param hierarchy The hierarchy, may be null * @return */ - public StatisticsFrequencyDistribution getFrequencyDistribution(int column, String[][] hierarchy) { + public StatisticsFrequencyDistribution getFrequencyDistribution(int column, + String[][] hierarchy) { + return getFrequencyDistribution(column, hierarchy, true, true); + } + + /** + * Returns a frequency distribution for the values in the given column. The order for string data items + * is derived from the provided hierarchy + * + * @param column The column + * @param hierarchy The hierarchy, may be null + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsFrequencyDistribution getFrequencyDistribution(int column, + String[][] hierarchy, + boolean includeSuppressedValues, + boolean includeNullValues) { // Reset stop flag interrupt.value = false; // Init - String[] values = getDistinctValuesOrdered(column, hierarchy); + String[] values = getDistinctValuesOrdered(column, + hierarchy, + includeSuppressedValues, + includeNullValues); double[] frequencies = new double[values.length]; // Create map of indexes @@ -689,7 +917,10 @@ public StatisticsFrequencyDistribution getFrequencyDistribution(int column, Stri for (int row = 0; row < handle.getNumRows(); row++) { checkInterrupt(); String value = handle.getValue(row, column); - frequencies[indexes.get(value)]++; + Integer index = indexes.get(value); + if (index != null) { + frequencies[index]++; + } } progress.value = 80; diff --git a/src/main/org/deidentifier/arx/aggregates/StatisticsBuilderInterruptible.java b/src/main/org/deidentifier/arx/aggregates/StatisticsBuilderInterruptible.java index 14da9f6ac..7d07fc3e6 100644 --- a/src/main/org/deidentifier/arx/aggregates/StatisticsBuilderInterruptible.java +++ b/src/main/org/deidentifier/arx/aggregates/StatisticsBuilderInterruptible.java @@ -195,10 +195,28 @@ public StatisticsClassification getClassificationPerformance(String[] features, * @return * @throws InterruptedException */ - public StatisticsContingencyTable - getContingencyTable(int column1, int column2) throws InterruptedException { + public StatisticsContingencyTable getContingencyTable(int column1, int column2) throws InterruptedException { + return this.getContingencyTable(column1, column2, true, true); + } + + /** + * Returns a contingency table for the given columns. This method assumes + * that the order of string data items will be derived from the + * hierarchies provided in the data definition (if any) + * + * @param column1 The first column + * @param column2 The second column + * @param includeSuppressedValues + * @param includeNullValues + * @return + * @throws InterruptedException + */ + public StatisticsContingencyTable getContingencyTable( int column1, + int column2, + boolean includeSuppressedValues, + boolean includeNullValues) throws InterruptedException { try { - return builder.getContingencyTable(column1, column2); + return builder.getContingencyTable(column1, column2, includeSuppressedValues, includeNullValues); } catch (Exception e) { if (e instanceof ComputationInterruptedException) { throw new InterruptedException("Interrupted"); @@ -328,6 +346,27 @@ public String[] getDistinctValues(int column) throws InterruptedException { } } + /** + * Returns the distinct set of data items from the given column. + * @param column + * @param includeSuppressedValues + * @param includeNullValues + * @return + * @throws InterruptedException + */ + public String[] getDistinctValues(int column, boolean includeSuppressedValues, + boolean includeNullValues) throws InterruptedException { + try { + return builder.getDistinctValues(column, includeSuppressedValues, includeNullValues); + } catch (Exception e) { + if (e instanceof ComputationInterruptedException) { + throw new InterruptedException("Interrupted"); + } else { + throw new UnexpectedErrorException(e); + } + } + } + /** * Returns an ordered list of the distinct set of data items from the given * column. This method assumes that the order of string data items can (and @@ -385,8 +424,7 @@ public String[] getDistinctValues(int column) throws InterruptedException { * @return * @throws InterruptedException */ - public String[] - getDistinctValuesOrdered(int column, Hierarchy hierarchy) throws InterruptedException { + public String[] getDistinctValuesOrdered(int column, Hierarchy hierarchy) throws InterruptedException { try { return builder.getDistinctValuesOrdered(column, hierarchy == null ? null : hierarchy.getHierarchy()); @@ -399,14 +437,44 @@ public String[] getDistinctValues(int column) throws InterruptedException { } } + /** + * Returns an ordered list of the distinct set of data items from the given + * column. This method assumes that the order of string data items can (and + * should) be derived from the provided hierarchy + * + * @param column The column + * @param hierarchy The hierarchy, may be null + * @param includeSuppressedValues + * @param includeNullValues + * @return + * @throws InterruptedException + */ + public String[] getDistinctValuesOrdered(int column, + Hierarchy hierarchy, + boolean includeSuppressedValues, + boolean includeNullValues) throws InterruptedException { + + try { + return builder.getDistinctValuesOrdered(column, + hierarchy == null ? null : hierarchy.getHierarchy(), + includeSuppressedValues, + includeNullValues); + } catch (Exception e) { + if (e instanceof ComputationInterruptedException) { + throw new InterruptedException("Interrupted"); + } else { + throw new UnexpectedErrorException(e); + } + } + } + /** * Returns statistics about the equivalence classes. * * @return * @throws InterruptedException */ - public StatisticsEquivalenceClasses - getEquivalenceClassStatistics() throws InterruptedException { + public StatisticsEquivalenceClasses getEquivalenceClassStatistics() throws InterruptedException { try { return builder.getEquivalenceClassStatistics(); } catch (Exception e) { @@ -485,6 +553,34 @@ public StatisticsFrequencyDistribution getFrequencyDistribution(int column, Hier } } + /** + * Returns a frequency distribution for the values in the given column. The order for string data items + * is derived from the provided hierarchy + * + * @param column The column + * @param hierarchy The hierarchy, may be null + * @param includeSuppressedValues + * @param includeNullValues + * @return + */ + public StatisticsFrequencyDistribution getFrequencyDistribution(int column, + Hierarchy hierarchy, + boolean includeSuppressedValues, + boolean includeNullValues) throws InterruptedException { + try { + return builder.getFrequencyDistribution(column, + hierarchy == null ? null : hierarchy.getHierarchy(), + includeSuppressedValues, + includeNullValues); + } catch (Exception e) { + if (e instanceof ComputationInterruptedException) { + throw new InterruptedException("Interrupted"); + } else { + throw new UnexpectedErrorException(e); + } + } + } + /** * If supported by the according builder, this method will report a progress * value in [0,100]. Otherwise, it will always return 0 From 67fbb0f515dc48879c8526a25a0b6248eb501b46 Mon Sep 17 00:00:00 2001 From: Fabian Prasser Date: Mon, 13 Jun 2022 16:08:59 +0200 Subject: [PATCH 2/2] Supporting showing / hiding special values in selected views of the UI --- .../org/deidentifier/arx/gui/model/Model.java | 158 ++++++++++++------ .../arx/gui/model/ModelEvent.java | 5 +- .../arx/gui/resources/messages.properties | 2 + .../arx/gui/resources/null_value.png | Bin 0 -> 751 bytes .../arx/gui/resources/suppressed_value.png | Bin 0 -> 694 bytes .../gui/view/impl/utility/DensityData.java | 34 ++-- .../impl/utility/LayoutUtilityStatistics.java | 76 ++++++++- .../gui/view/impl/utility/ViewStatistics.java | 17 +- .../utility/ViewStatisticsClassification.java | 2 +- .../ViewStatisticsContingencyHeatmap.java | 25 +-- .../ViewStatisticsContingencyTable.java | 13 +- .../ViewStatisticsDistributionHistogram.java | 7 +- .../ViewStatisticsDistributionTable.java | 13 +- .../ViewStatisticsEquivalenceClassTable.java | 2 +- .../impl/utility/ViewStatisticsQuality.java | 2 +- .../utility/ViewStatisticsSummaryTable.java | 2 +- 16 files changed, 256 insertions(+), 102 deletions(-) create mode 100644 src/gui/org/deidentifier/arx/gui/resources/null_value.png create mode 100644 src/gui/org/deidentifier/arx/gui/resources/suppressed_value.png diff --git a/src/gui/org/deidentifier/arx/gui/model/Model.java b/src/gui/org/deidentifier/arx/gui/model/Model.java index d0dce8f6d..d7dee0035 100644 --- a/src/gui/org/deidentifier/arx/gui/model/Model.java +++ b/src/gui/org/deidentifier/arx/gui/model/Model.java @@ -271,30 +271,34 @@ public static enum Perspective { * RISK ANALYSIS ******************************************/ /** Selected quasi identifiers */ - private Set selectedQuasiIdentifiers = null; + private Set selectedQuasiIdentifiers = null; - /* ***************************************** + /* ***************************************** * LOCAL RECODING ******************************************/ + /** The local recoding model */ private ModelLocalRecoding localRecodingModel = new ModelLocalRecoding(); /** Heuristic search threshold */ - private Integer heuristicSearchThreshold; + private Integer heuristicSearchThreshold = null; /** Heuristic search threshold */ - private Integer heuristicSearchTimeLimit; + private Integer heuristicSearchTimeLimit = null; /** Heuristic search threshold */ - private Integer heuristicSearchStepLimit; + private Integer heuristicSearchStepLimit = null; - /** General anonymization configuration. Proxy for some fields for backwards compatibility */ - private ModelAnonymizationConfiguration anonymizationConfiguration; + /** + * General anonymization configuration. Proxy for some fields for backwards + * compatibility + */ + private ModelAnonymizationConfiguration anonymizationConfiguration = null; - /* ***************************************** + /* ***************************************** * Data Mining - * ***************************************** - */ + * ******************************************/ + /** Selected attributes */ private Set selectedFeatures = null; @@ -313,7 +317,17 @@ public static enum Perspective { /* ****************************************** * Help dialog * ******************************************/ - private Boolean helpDialogModal; + /** Whether the help dialog should be modal */ + private Boolean helpDialogModal = null; + + + /* ****************************************** + * Settings regarding values included in basic statistics + * ******************************************/ + /** Include suppressed values in statistics */ + private Boolean showSuppressedValues = null; + /** Include null values in statistics */ + private Boolean showNullValues = null; /** * Creates a new instance. @@ -530,7 +544,7 @@ public void createConfig() { // Heuristic search limit config.setHeuristicSearchThreshold(this.getHeuristicSearchThreshold()); } - + /** * Returns the current anonymization configuration * @return @@ -541,7 +555,7 @@ public ModelAnonymizationConfiguration getAnonymizationConfiguration() { } return anonymizationConfiguration; } - + /** * Returns the current anonymizer. * @@ -571,7 +585,7 @@ public List getAuditTrail() { } return auditTrail; } - + /** * Returns the b-Likeness privacy model. * @@ -588,7 +602,7 @@ public Map getBLikenessModel() { } return bLikenessModel; } - + /** * Return charset. Returns null for projects with unknown charset, "UTF-8" else. * @return @@ -631,7 +645,7 @@ public CSVSyntax getCSVSyntax() { } return csvSyntax; } - + /** * Returns the d-disclosure privacy model. * @@ -669,7 +683,7 @@ public ModelDifferentialPrivacyCriterion getDifferentialPrivacyModel() { } return differentialPrivacyModel; } - + /** * Returns the d-presence model. * @@ -678,7 +692,7 @@ public ModelDifferentialPrivacyCriterion getDifferentialPrivacyModel() { public ModelDPresenceCriterion getDPresenceModel() { return dPresenceModel; } - + /** * Returns a list of indices of all equivalence classes. * @@ -708,7 +722,7 @@ public Integer getHeuristicSearchThreshold() { } return heuristicSearchThreshold; } - + /** * @return the heuristicSearchTimeLimit */ @@ -718,7 +732,7 @@ public Integer getHeuristicSearchTimeLimit() { } return heuristicSearchTimeLimit; } - + /** * Returns the according parameter. * @@ -755,7 +769,7 @@ public long getInputBytes() { public ModelConfiguration getInputConfig() { return inputConfig; } - + /** * Returns the input definition. * @@ -774,7 +788,7 @@ public DataDefinition getInputDefinition(){ public ARXPopulationModel getInputPopulationModel() { return getRiskModel().getPopulationModel(); } - + /** * Returns the k-anonymity model. * @@ -795,7 +809,7 @@ public ModelKMapCriterion getKMapModel() { } return kMapModel; } - + /** * Returns the l-diversity model. * @@ -820,7 +834,7 @@ public Locale getLocale() { return locale; } } - + /** * Returns the model for local recoding * @return @@ -841,7 +855,7 @@ public ModelLocalRecoding getLocalRecodingModel() { public int getMaximalSizeForComplexOperations(){ return this.maximalSizeForComplexOperations; } - + /** * Returns the maximal size of a sub-lattice that will be displayed * by the viewer. @@ -884,7 +898,7 @@ public MetricDescription getMetricDescription() { } return this.metricDescription; } - + /** * Returns the name of this project. * @@ -893,7 +907,7 @@ public MetricDescription getMetricDescription() { public String getName() { return name; } - + /** * Returns the current filter. * @@ -918,7 +932,7 @@ public String getOptimalNodeAsString() { public DataHandle getOutput() { return output; } - + /** * Returns the output config. * @@ -927,7 +941,7 @@ public DataHandle getOutput() { public ModelConfiguration getOutputConfig() { return outputConfig; } - + /** * Returns the output definition. * @@ -968,7 +982,7 @@ public ARXPopulationModel getOutputPopulationModel() { } return null; } - + /** * Returns the currently applied transformation. * @@ -1006,7 +1020,7 @@ public ARXProcessStatistics getProcessStatistics() { } return optimizationStatistics; } - + /** * Returns the current query. * @@ -1059,7 +1073,7 @@ public ModelRisk getRiskModel() { public String getSelectedAttribute() { return selectedAttribute; } - + /** * Returns the selected classes * @return @@ -1070,7 +1084,7 @@ public Set getSelectedClasses() { } return this.selectedClasses; } - + /** * Returns the selected classes, ordered by occurrence in the dataset * @return @@ -1098,7 +1112,7 @@ public Set getSelectedFeatures() { } return this.selectedFeatures; } - + /** * Returns the selected features, ordered by occurrence in the dataset * @return @@ -1157,7 +1171,7 @@ public Set getSelectedQuasiIdentifiers() { } return this.selectedQuasiIdentifiers; } - + /** * Returns the separator. * @@ -1167,7 +1181,6 @@ public char getSeparator() { return separator; } - /** * Returns the according parameter. * @@ -1176,7 +1189,7 @@ public char getSeparator() { public double getSnapshotSizeDataset() { return snapshotSizeDataset; } - + /** * Returns the according parameter. * @@ -1214,7 +1227,7 @@ public double getSolutionSpaceSize() { // Return return size; } - + /** * Returns the configuration object for the stackelberg privacy model * @return @@ -1226,6 +1239,7 @@ public ModelProfitabilityCriterion getStackelbergModel() { return stackelbergPrivacyModel; } + /** * Returns the origin of the subset. * @@ -1343,6 +1357,28 @@ public boolean isSensitiveAttributeSelected() { return (getInputDefinition().getAttributeType(getSelectedAttribute()) == AttributeType.SENSITIVE_ATTRIBUTE); } + /** + * Whether to show certain values in basic statistics + * @return the showNullValues + */ + public boolean isShowNullValues() { + if (showNullValues == null) { + showNullValues = true; + } + return showNullValues; + } + + /** + * Whether to show certain values in basic statistics + * @return the showSuppressedValues + */ + public boolean isShowSuppressedValues() { + if (showSuppressedValues == null) { + showSuppressedValues = true; + } + return showSuppressedValues; + } + /** * Returns whether visualization is enabled. * @@ -1382,7 +1418,7 @@ public void reset() { this.selectedClassValue = null; this.selectedAttribute = null; } - + /** * Returns the last two selected attributes. */ @@ -1433,7 +1469,7 @@ public void setAnonymizer(final ARXAnonymizer anonymizer) { setModified(); this.anonymizer = anonymizer; } - + /** * Sets the charset * @param charset @@ -1451,7 +1487,7 @@ public void setDebugEnabled(boolean value){ this.debugEnabled = value; this.setModified(); } - + /** * Sets the project description. * @@ -1461,7 +1497,7 @@ public void setDescription(final String description) { this.description = description; setModified(); } - + /** * Updates features and classes to reflect the definition provided * @param definition @@ -1518,7 +1554,7 @@ public void setHeuristicSearchThreshold(Integer heuristicSearchThreshold) { public void setHeuristicSearchTimeLimit(Integer heuristicSearchTimeLimit) { this.heuristicSearchTimeLimit = heuristicSearchTimeLimit; } - + /** * Sets the according parameter. * @@ -1528,7 +1564,7 @@ public void setHistorySize(final int historySize) { this.historySize = historySize; setModified(); } - + /** * Sets the according parameter. * @@ -1548,7 +1584,7 @@ public void setInputBytes(final long inputBytes) { setModified(); this.inputBytes = inputBytes; } - + /** * Sets the input config. * @@ -1557,7 +1593,7 @@ public void setInputBytes(final long inputBytes) { public void setInputConfig(final ModelConfiguration config) { this.inputConfig = config; } - + /** * Sets the project locale. * @@ -1603,7 +1639,7 @@ public void setMetricDescription(MetricDescription description) { public void setModified() { modified = true; } - + /** * Sets the project name. * @@ -1613,7 +1649,7 @@ public void setName(final String name) { this.name = name; setModified(); } - + /** * Sets a filter. * @@ -1640,7 +1676,7 @@ public void setOutput(final DataHandle output, final ARXNode node) { } setModified(); } - + /** * Sets the current output, deserialized from a project * @@ -1682,7 +1718,7 @@ public void setOutputConfig(final ModelConfiguration config) { public void setPath(final String path) { this.path = path; } - + /** * @param perspective the perspective to set */ @@ -1763,7 +1799,7 @@ public void setSelectedClasses(Set set) { public void setSelectedClassValue(final String classValue) { selectedClassValue = classValue; } - + /** * Sets a set of selected attributes * @param set @@ -1772,7 +1808,7 @@ public void setSelectedFeatures(Set set) { this.selectedFeatures = set; this.setModified(); } - + /** * Sets the selected node. * @@ -1782,7 +1818,7 @@ public void setSelectedNode(final ARXNode node) { selectedNode = node; setModified(); } - + /** * Sets a set of quasi identifiers selected for risk analysis * @param set @@ -1791,6 +1827,22 @@ public void setSelectedQuasiIdentifiers(Set set) { this.selectedQuasiIdentifiers = set; this.setModified(); } + + /** + * Whether to show certain values in basic statistics + * @param showNullValues the showNullValues to set + */ + public void setShowNullValues(boolean showNullValues) { + this.showNullValues = showNullValues; + } + + /** + * Whether to show certain values in basic statistics + * @param showSuppressedValues the showSuppressedValues to set + */ + public void setShowSuppressedValues(boolean showSuppressedValues) { + this.showSuppressedValues = showSuppressedValues; + } /** * diff --git a/src/gui/org/deidentifier/arx/gui/model/ModelEvent.java b/src/gui/org/deidentifier/arx/gui/model/ModelEvent.java index c95f2b67c..8a0f46cbe 100644 --- a/src/gui/org/deidentifier/arx/gui/model/ModelEvent.java +++ b/src/gui/org/deidentifier/arx/gui/model/ModelEvent.java @@ -128,7 +128,10 @@ public static enum ModelPart { RESPONSE_VARIABLES, /** ATTRIBUTE_TYPE */ - ATTRIBUTE_TYPE_BULK_UPDATE + ATTRIBUTE_TYPE_BULK_UPDATE, + + /** SHOW NULL OR SUPPRESSED IN ANALYSIS */ + SHOW_SPECIAL_VALUES } /** The part of the model that has changed. */ diff --git a/src/gui/org/deidentifier/arx/gui/resources/messages.properties b/src/gui/org/deidentifier/arx/gui/resources/messages.properties index 5a47d636b..7aeb35d9f 100644 --- a/src/gui/org/deidentifier/arx/gui/resources/messages.properties +++ b/src/gui/org/deidentifier/arx/gui/resources/messages.properties @@ -665,6 +665,8 @@ StatisticsView.9=Classification models StatisticsView.10=Classification performance StatisticsView.12=ROC curves StatisticsView.11=Quality models +StatisticsView.13=Include suppressed values +StatisticsView.14=Include null values WorkerAnonymize.0=Anonymizing WorkerAnonymize.1=Task interrupted\! WorkerAnonymize.2=Calculating score diff --git a/src/gui/org/deidentifier/arx/gui/resources/null_value.png b/src/gui/org/deidentifier/arx/gui/resources/null_value.png new file mode 100644 index 0000000000000000000000000000000000000000..d7351a7c6b19ae551694d7405a44ff70575a2e35 GIT binary patch literal 751 zcmVEX>4Tx04R}tkv&MmKp2MKrfNl69PA+CkfG{gK~%(1s#pXIrLEAagUO|T(4-+r zad8w}3l9D)RvlcNb#-tR1i>E=M<*vm7b)?(q|hS9JC1vJ?|WbFz5|3tg{fxO1fXh` zk&4H}Y;IKyyrKsonll)an5ieyiy3&1uY36Tei!9g-gSSDkdikU;1h}Gm~L3a8^kl4 zmd<&fILu0tLVQjPFL_LKB_ zTZ_565M4$uNJd7XSbN24YJ`L;&&t@c{NKZ}yA;000SaNLh0L z01ejw01ejxLMWSf00007bV*G`2j&J14L2dBRa|=j008SrL_t(I%bn9pN&`U<#_?aw zO~jqspkNm6^H?fI5MoZ^BA4)S1+S3D&Y0Z5tm48;h`OmrGqlr#1qDM-{ky-fs+<4E zLEVob9EZ?LDtv?uW*FlnavW=GT;c=gIrtsdHL~yUiBV4WrU0MC$y2;0 zb_mb22Jl|?0{yh$DSS48VG@n~jU_q_VbR!mI?J!Br+}Yv6$hlz!gspC7RL=?QTvuI z4EB<0WLGWss*=_*=MsE_RmQT5xFpj8yu;yN-5$jynd26xk&~GHDhIdYVRREaecT7M hi0@ubcJXPJ;sQ07$=Lt^002ovPDHLkV1nekQRM&t literal 0 HcmV?d00001 diff --git a/src/gui/org/deidentifier/arx/gui/resources/suppressed_value.png b/src/gui/org/deidentifier/arx/gui/resources/suppressed_value.png new file mode 100644 index 0000000000000000000000000000000000000000..50ef3c00e4c0d17e80c5e624da2dcefbacff71cd GIT binary patch literal 694 zcmV;n0!jUeP)EX>4Tx04R}tkv&MmKp2MKrfNl69PA(>lA$_T5EXHhDi*;)X)CnqU~=gnG-*gu zTpR`0f`dPcRRbq^ok@1i`*yY9~sQt~DPd?N82(+!JwgLr1s z(mC%FhgnHdh|h_~4Z0xlBiCh@-#8Z?7Ium6cZ^rk9qiq9DkBrGP%lN z+e)kfB~J-v9@P zz*v#8*FE0d+u65&Yg+yL0bDk6o`$^}Bme*a24YJ`L;&&t@c{NKZ}yA;000SaNLh0L z04^f{04^f|c%?sf00007bV*G`2j&J14I&`r=Zg6N000?uMObu0Z*6U5Zgc=ca%Ew3 zWn>_CX>@2HM@dakSAh-}0001uNkle(gpkWz! z!a@i*%#}D2f&`1f&tUGD8<-jV{H8r^)8w?Y^lJJ658QtZbQ3+s0oj+?1$Jq%pXeuU zxL_S?8IBlX)zsP)XXJQegFzDuB{_ imp /** Parent */ private final Composite parent; + /** Needs updates to include suppressed or null*/ + private final boolean dynamicallyIncludesSuppressedOrNull; + /** * Creates a new instance. * @@ -80,12 +83,14 @@ public abstract class ViewStatistics imp * @param target * @param reset * @param dependsOnAttribute + * @param dynamicallyIncludesSuppressedOrNull */ public ViewStatistics( final Composite parent, final Controller controller, final ModelPart target, final ModelPart reset, - final boolean dependsOnAttribute) { + final boolean dependsOnAttribute, + final boolean dynamicallyIncludesSuppressedOrNull) { // Register controller.addListener(ModelPart.SELECTED_ATTRIBUTE, this); @@ -96,6 +101,7 @@ public ViewStatistics( final Composite parent, controller.addListener(ModelPart.DATA_TYPE, this); controller.addListener(ModelPart.SELECTED_UTILITY_VISUALIZATION, this); controller.addListener(ModelPart.ATTRIBUTE_VALUE, this); + controller.addListener(ModelPart.SHOW_SPECIAL_VALUES, this); controller.addListener(target, this); if (reset != null) { controller.addListener(reset, this); @@ -106,6 +112,7 @@ public ViewStatistics( final Composite parent, this.reset = reset; this.target = target; this.dependsOnAttribute = dependsOnAttribute; + this.dynamicallyIncludesSuppressedOrNull = dynamicallyIncludesSuppressedOrNull; this.parent = parent; // Create controls @@ -200,6 +207,14 @@ public void update(final ModelEvent event) { return; } } + + // Invalidate + if (event.part == ModelPart.SHOW_SPECIAL_VALUES) { + if (dynamicallyIncludesSuppressedOrNull) { + this.triggerUpdate(); + return; + } + } // Potentially invalidate if (event.part == ModelPart.DATA_TYPE || diff --git a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsClassification.java b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsClassification.java index 93e7cac05..e69e8b8b9 100644 --- a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsClassification.java +++ b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsClassification.java @@ -138,7 +138,7 @@ public ViewStatisticsClassification(final Composite parent, final Controller controller, final ModelPart part) { - super(parent, controller, part, null, false); + super(parent, controller, part, null, false, false); this.manager = new AnalysisManager(parent.getDisplay()); this.isOutput = part != ModelPart.INPUT; this.rocCurves = new HashMap<>(); diff --git a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsContingencyHeatmap.java b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsContingencyHeatmap.java index f181b3272..f4f6fd70d 100644 --- a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsContingencyHeatmap.java +++ b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsContingencyHeatmap.java @@ -63,11 +63,11 @@ public class ViewStatisticsContingencyHeatmap extends ViewStatistics[] inputValues = new List[contingency.values1.length]; diff --git a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionHistogram.java b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionHistogram.java index 46726126f..1e232539f 100644 --- a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionHistogram.java +++ b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionHistogram.java @@ -85,7 +85,7 @@ public ViewStatisticsDistributionHistogram(final Composite parent, final ModelPart target, final ModelPart reset) { - super(parent, controller, target, reset, true); + super(parent, controller, target, reset, true, true); this.manager = new AnalysisManager(parent.getDisplay()); } @@ -344,7 +344,10 @@ public void run() throws InterruptedException { long time = System.currentTimeMillis(); // Perform work - this.distribution = builder.getFrequencyDistribution(column, hierarchy); + this.distribution = builder.getFrequencyDistribution(column, + hierarchy, + context.model.isShowSuppressedValues(), + context.model.isShowNullValues()); // Our users are patient while (System.currentTimeMillis() - time < MINIMAL_WORKING_TIME && !stopped){ diff --git a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionTable.java b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionTable.java index e98fec3a1..c8bb609e3 100644 --- a/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionTable.java +++ b/src/gui/org/deidentifier/arx/gui/view/impl/utility/ViewStatisticsDistributionTable.java @@ -57,11 +57,11 @@ public class ViewStatisticsDistributionTable extends ViewStatistics