From b596527c968e46b744689976bd6507a2978a1d2d Mon Sep 17 00:00:00 2001 From: iantei Date: Fri, 27 Sep 2024 16:32:47 -0700 Subject: [PATCH] Use 90% confidence threshold. Filter the expanded_labeled_inferred_ct with dropna. --- viz_scripts/scaffolding.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/viz_scripts/scaffolding.py b/viz_scripts/scaffolding.py index 3e46ebab..470820b0 100644 --- a/viz_scripts/scaffolding.py +++ b/viz_scripts/scaffolding.py @@ -161,15 +161,14 @@ def _select_max_label(row): if row['user_input']: return row['user_input'] max_entry = max(row['inferred_labels'], key=lambda x: x['p']) - return max_entry['labels'] if max_entry['p'] > row['confidence_threshold'] else { - 'mode_confirm': 'uncertain' - } + # Look up for the 'p' value > 90% + return max_entry['labels'] if (max_entry['p'] > .90) else {} labeled_inferred_labels = labeled_inferred_ct.apply(_select_max_label, axis=1).apply(pd.Series) disp.display(labeled_inferred_labels.head()) expanded_labeled_inferred_ct = pd.concat([labeled_inferred_ct, labeled_inferred_labels], axis=1) - # Filter out the dataframe in which mode_confirm is uncertain - expanded_labeled_inferred_ct = expanded_labeled_inferred_ct[(expanded_labeled_inferred_ct['mode_confirm'] != 'uncertain')] + # Filter out the dataframe + expanded_labeled_inferred_ct.dropna('index', how='all', inplace=True) disp.display(expanded_labeled_inferred_ct.head()) return expanded_labeled_inferred_ct