From 2b8f1cef6b19fa920d290fad359afe051c49b831 Mon Sep 17 00:00:00 2001 From: Matt Piazza Date: Sun, 30 Oct 2016 16:30:07 -0400 Subject: [PATCH] use the new NaiveBayes UpdateTokenizer func --- init.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/init.go b/init.go index 5beeb0790..523a69f62 100644 --- a/init.go +++ b/init.go @@ -6,9 +6,9 @@ import ( "io/ioutil" "os" "path" - "strings" "github.com/cdipaolo/goml/base" + "github.com/cdipaolo/goml/text" ) const ( @@ -46,10 +46,7 @@ func RestoreModels(bytes []byte) (Models, error) { for i := range models { models[i].UpdateSanitize(base.OnlyWords) - models[i].UpdateTokenizer( - func(input string) []string { - return strings.Split(strings.ToLower(input), " ") - }) + models[i].UpdateTokenizer(&text.SimpleTokenizer{SplitOn: " "}) } return models, nil