diff --git a/init.go b/init.go index 5beeb0790..523a69f62 100644 --- a/init.go +++ b/init.go @@ -6,9 +6,9 @@ import ( "io/ioutil" "os" "path" - "strings" "github.com/cdipaolo/goml/base" + "github.com/cdipaolo/goml/text" ) const ( @@ -46,10 +46,7 @@ func RestoreModels(bytes []byte) (Models, error) { for i := range models { models[i].UpdateSanitize(base.OnlyWords) - models[i].UpdateTokenizer( - func(input string) []string { - return strings.Split(strings.ToLower(input), " ") - }) + models[i].UpdateTokenizer(&text.SimpleTokenizer{SplitOn: " "}) } return models, nil