Support :normalize option in docs-to-dataset

This commit is contained in:
Joshua Eckroth 2013-07-17 00:41:02 -04:00
parent 8d7c41c25b
commit b5f92c5ced

View file

@ -534,6 +534,7 @@ split immediately you can use do-split-dataset."
:lowercase (:lowercase parsed-opts true) :lowercase (:lowercase parsed-opts true)
:prefix "title-" :words-to-keep (:words-to-keep parsed-opts 1000) :prefix "title-" :words-to-keep (:words-to-keep parsed-opts 1000)
:counts (:counts parsed-opts false) :counts (:counts parsed-opts false)
:normalize (:normalize parsed-opts false)
:transform-tf (:transform-tf parsed-opts true) :transform-tf (:transform-tf parsed-opts true)
:transform-idf (:transform-idf parsed-opts true) :transform-idf (:transform-idf parsed-opts true)
:stemmer (if (:stemmer parsed-opts false) :stemmer (if (:stemmer parsed-opts false)
@ -549,6 +550,7 @@ split immediately you can use do-split-dataset."
:lowercase (:lowercase parsed-opts true) :lowercase (:lowercase parsed-opts true)
:prefix "fulltext-" :words-to-keep (:words-to-keep parsed-opts 1000) :prefix "fulltext-" :words-to-keep (:words-to-keep parsed-opts 1000)
:counts (:counts parsed-opts false) :counts (:counts parsed-opts false)
:normalize (:normalize parsed-opts false)
:transform-tf (:transform-tf parsed-opts true) :transform-tf (:transform-tf parsed-opts true)
:transform-idf (:transform-idf parsed-opts true) :transform-idf (:transform-idf parsed-opts true)
:stemmer (if (:stemmer parsed-opts false) :stemmer (if (:stemmer parsed-opts false)