-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathexport_test.go
79 lines (68 loc) · 1.37 KB
/
export_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
package bpe
import (
"bytes"
"io"
"testing"
"github.com/pkg/errors"
)
type encoderMock struct {
data []byte
err error
}
func (e *encoderMock) Encode(w io.Writer, _ interface{}) error {
_, _ = w.Write(e.data)
return e.err
}
func TestExport(t *testing.T) {
tt := []struct {
name string
model *BPE
encoderMock *encoderMock
expected string
withError bool
}{
{
name: "test with default encoder",
model: newModelFromTokensFrequencyTable(
tokensFrequencyTable{
"foo": 1,
},
1,
),
expected: `{"max_token_length":3,"vocab":["foo"]}` + "\n",
},
{
name: "mocked encoder",
model: &BPE{},
encoderMock: &encoderMock{
data: []byte("123123"),
},
expected: "123123",
},
{
name: "encoder returns error",
model: &BPE{},
encoderMock: &encoderMock{
err: errors.New("Some error"),
},
expected: "",
withError: true,
},
}
for _, tc := range tt {
t.Run(tc.name, func(t *testing.T) {
var opts []ExportOption
if tc.encoderMock != nil {
opts = append(opts, WithEncoder(tc.encoderMock))
}
buf := bytes.NewBuffer(nil)
if err := Export(tc.model, buf, opts...); err != nil && !tc.withError {
t.Errorf("Unexpected Export error %v", err)
}
actual := buf.String()
if tc.expected != actual {
t.Errorf("Expected: %v\nGot: %v\n", tc.expected, actual)
}
})
}
}