forked from monologg/JointBERT
-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
105 lines (77 loc) · 3.25 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
import os
import random
import logging
import torch
import numpy as np
from seqeval.metrics import precision_score, recall_score, f1_score
from transformers import BertTokenizer, BertConfig, DistilBertConfig, DistilBertTokenizer, AlbertConfig, AlbertTokenizer
from model import JointBERT, JointDistilBERT
MODEL_CLASSES = {
'bert': (BertConfig, JointBERT, BertTokenizer),
'distilbert': (DistilBertConfig, JointDistilBERT, DistilBertTokenizer),
'albert': (AlbertConfig, JointBERT, AlbertTokenizer)
}
MODEL_PATH_MAP = {
'bert': 'bert-base-uncased',
'distilbert': 'distilbert-base-uncased',
'albert': 'albert-xxlarge-v1'
}
def get_intent_labels(args):
return [label.strip() for label in open(os.path.join(args.data_dir, args.task, args.intent_label_file), 'r', encoding='utf-8')]
def get_slot_labels(args):
return [label.strip() for label in open(os.path.join(args.data_dir, args.task, args.slot_label_file), 'r', encoding='utf-8')]
def load_tokenizer(args):
return MODEL_CLASSES[args.model_type][2].from_pretrained(args.model_name_or_path)
def init_logger():
logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
def set_seed(args):
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if not args.no_cuda and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
def compute_metrics(intent_preds, intent_labels, slot_preds, slot_labels):
assert len(intent_preds) == len(intent_labels) == len(slot_preds) == len(slot_labels)
results = {}
intent_result = get_intent_acc(intent_preds, intent_labels)
slot_result = get_slot_metrics(slot_preds, slot_labels)
sementic_result = get_sentence_frame_acc(intent_preds, intent_labels, slot_preds, slot_labels)
results.update(intent_result)
results.update(slot_result)
results.update(sementic_result)
return results
def get_slot_metrics(preds, labels):
assert len(preds) == len(labels)
return {
"slot_precision": precision_score(labels, preds),
"slot_recall": recall_score(labels, preds),
"slot_f1": f1_score(labels, preds)
}
def get_intent_acc(preds, labels):
acc = (preds == labels).mean()
return {
"intent_acc": acc
}
def read_prediction_text(args):
return [text.strip() for text in open(os.path.join(args.pred_dir, args.pred_input_file), 'r', encoding='utf-8')]
def get_sentence_frame_acc(intent_preds, intent_labels, slot_preds, slot_labels):
"""For the cases that intent and all the slots are correct (in one sentence)"""
# Get the intent comparison result
intent_result = (intent_preds == intent_labels)
# Get the slot comparision result
slot_result = []
for preds, labels in zip(slot_preds, slot_labels):
assert len(preds) == len(labels)
one_sent_result = True
for p, l in zip(preds, labels):
if p != l:
one_sent_result = False
break
slot_result.append(one_sent_result)
slot_result = np.array(slot_result)
sementic_acc = np.multiply(intent_result, slot_result).mean()
return {
"sementic_frame_acc": sementic_acc
}