Merge pull request #84 from go-ego/range-pr

add splitWord() function
go-ego · Jan 3, 2021 · dcdb98c · dcdb98c
2 parents 9538046 + d3b3c9b
commit dcdb98c
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 1 deletion.
diff --git a/segmenter.go b/segmenter.go
@@ -176,6 +176,12 @@ func updateJumper(jumper *jumper, baseDistance float32, token *Token) {
 	}
 }
 
+// SplitWords 将文本划分成字元
+func SplitWords(text Text) []Text {
+	var seg Segmenter
+	return seg.SplitTextToWords(text)
+}
+
 // SplitTextToWords 将文本划分成字元
 func (seg *Segmenter) SplitTextToWords(text Text) []Text {
 	output := make([]Text, 0, len(text)/3)

diff --git a/segmenter_test.go b/segmenter_test.go
@@ -30,7 +30,7 @@ func TestSplit(t *testing.T) {
 		bytesToString(seg1.SplitTextToWords([]byte("世界有七十亿人口"))))
 
 	tt.Expect(t, "github/ /is/ /a/ /web/-/based/ /hosting/ /service/,/ /for/ /software/ /development/ /projects/./",
-		bytesToString(seg1.SplitTextToWords([]byte(
+		bytesToString(SplitWords([]byte(
 			"GitHub is a web-based hosting service, for software development projects."))))
 
 	tt.Expect(t, "雅/虎/yahoo/!/ /致/力/于/，/领/先/的/门/户/网/站/。/",