diff --git a/.swp b/.swp new file mode 100644 index 0000000..2c83b2b Binary files /dev/null and b/.swp differ diff --git a/project.clj b/project.clj index 80c8f7d..4ba06c2 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject synaptic "0.3.0-SNAPSHOT" +(defproject keorn/synaptic "0.3.0-SNAPSHOT" :description "Neural Networks in Clojure" :url "https://github.com/japonophile/synaptic" :license {:name "Eclipse Public License" diff --git a/src/synaptic/core.clj b/src/synaptic/core.clj index b4dd330..1e919a2 100644 --- a/src/synaptic/core.clj +++ b/src/synaptic/core.clj @@ -39,6 +39,9 @@ (def load-training-set-header d/load-training-set-header) (alter-meta! #'load-training-set-header merge (select-keys (meta #'d/load-training-set-header) [:doc :arglists])) +(def dataset d/dataset) +(alter-meta! #'dataset merge + (select-keys (meta #'d/dataset) [:doc :arglists])) (def training-set d/training-set) (alter-meta! #'training-set merge (select-keys (meta #'d/training-set) [:doc :arglists])) diff --git a/src/synaptic/datasets.clj b/src/synaptic/datasets.clj index 0fd3ace..31493d2 100644 --- a/src/synaptic/datasets.clj +++ b/src/synaptic/datasets.clj @@ -80,10 +80,10 @@ (defn count-labels "Create a map with number of occurrence of each label." - [uniquelabels binlabels] - (let [binlb2cnt (reduce (fn [lbmap lb] (assoc lbmap lb (inc (get lbmap lb 0)))) - {} binlabels)] - (zipmap (u/frombinary uniquelabels (keys binlb2cnt)) (vals binlb2cnt)))) + [labeltranslator encodedlabels] + (let [translate-keys #(zipmap (mapv labeltranslator (keys %)) + (vals %))] + (translate-keys (frequencies encodedlabels)))) (defn training-set "Create a training set from samples and associated labels. @@ -91,22 +91,23 @@ It also has a map that will allow converting y's back to the original labels. Options: - :name - a name for the training set - :type - the type of training data (e.g. :binary-image, :grayscale-image ...) - :fieldsize - [width height] of each sample data (for images) - :nvalid - size of the validation set (default is 0, i.e. no validation set) - :batch - size of a mini-batch (default is the number of samples, after - having set apart the validation set) - :online true - set this flag for online training (same as batch size = 1) - :rand false - unset this flag to keep original ordering (by default, samples - will be shuffled before partitioning)." + :name - a name for the training set + :type - the type of training data (e.g. :binary-image, :grayscale-image ...) + :continuous true - set this flag to use continuous labels (auto-scaled to between 0 and 1) + :fieldsize - [width height] of each sample data (for images) + :nvalid - size of the validation set (default is 0, i.e. no validation set) + :batch - size of a mini-batch (default is the number of samples, after + having set apart the validation set) + :online true - set this flag for online training (same as batch size = 1) + :rand false - unset this flag to keep original ordering (by default, samples + will be shuffled before partitioning)." [samples labels & [options]] {:pre [(= (count samples) (count labels))]} (let [batchsize (if (:online options) 1 (:batch options)) trainsize (if (:nvalid options) (- (count samples) (:nvalid options))) randomize (if (nil? (:rand options)) true (:rand options)) - [binlb uniquelb] (u/tobinary labels) - [smp lb] (if randomize (shuffle-vecs samples binlb) [samples binlb]) + [reglb lbtranslator] (if (:continuous options) (u/tocontinuous labels) (u/tobinary labels)) + [smp lb] (if randomize (shuffle-vecs samples reglb) [samples reglb]) [trainsmp validsmp] (if trainsize (split-at trainsize smp) [smp nil]) [trainlb validlb] (if trainsize (split-at trainsize lb) [lb nil]) [batchsmp batchlb] (partition-vecs batchsize trainsmp trainlb) @@ -118,9 +119,9 @@ :type (:type options) :fieldsize (or (:fieldsize options) (u/divisors (count (first samples)))) - :batches (mapv (partial count-labels uniquelb) batchlb) - :valid (count-labels uniquelb validlb) - :labels uniquelb}] + :batches (mapv (partial count-labels lbtranslator) batchlb) + :valid (count-labels lbtranslator validlb) + :labeltranslator lbtranslator}] (TrainingSet. header trainsets validset))) (defn test-set @@ -129,7 +130,7 @@ Options: :name - a name for the test set - :type - the type of training data (e.g. :binary-image, :grayscale-image ...) + :type - the type of test data (e.g. :binary-image, :grayscale-image ...) :fieldsize - [width height] of each sample data (for images) :rand true - set this flag to shuffle samples." [samples & [options]] diff --git a/src/synaptic/net.clj b/src/synaptic/net.clj index 50476a9..2b94e53 100644 --- a/src/synaptic/net.clj +++ b/src/synaptic/net.clj @@ -111,6 +111,11 @@ [zs] (m/map (fn [z] (if (>= z 0) 1 -1)) zs)) +(defn relu + "Rectified linear unit." + [zs] + (m/map #(if (<= % 0.) 0. %) zs)) + (defn sigmoid "Sigmoid activation function. Computed as 1 / (1 + e^(-z))." @@ -256,16 +261,14 @@ as)))) (defn estimate - "Estimate classes for a given data set, by computing network output for each - sample of the data set, and returns the most probable class (label) - or its - index if labels are not defined." + "Estimate labels for a given data set, by computing network output for each + sample of the data set, and returns appropriately transformed result + - or its index if labels are not defined." [^Net nn ^DataSet dset] - (let [x (:x dset) - y (m/dense (:a (last (net-activities nn x)))) - n (count (first y)) - ci (mapv #(apply max-key % (range n)) y) - cs (-> nn :header :labels)] - (if cs - (mapv #(get cs %) ci) - ci))) - + (let [x (:x dset) + y (m/dense (:a (last (net-activities nn x)))) + label-size (count (first y)) + lbtranslator (-> nn :arch :labeltranslator)] + (if lbtranslator + (mapv lbtranslator y) + (mapv #(apply max-key % (range label-size)) y)))) diff --git a/src/synaptic/training.clj b/src/synaptic/training.clj index e1bd3d6..97d9665 100644 --- a/src/synaptic/training.clj +++ b/src/synaptic/training.clj @@ -111,7 +111,8 @@ (case actkind :softmax (fn [ys] (m/mult ys (m/- 1.0 ys))) :sigmoid (fn [ys] (m/mult ys (m/- 1.0 ys))) - :hyperbolic-tangent (fn [ys] (m/- 1.0 (m/mult ys ys))))) + :hyperbolic-tangent (fn [ys] (m/- 1.0 (m/mult ys ys))) + :relu (fn [ys] (m/map #(if (<= % 0) 0. 1.))))) (defn output-layer-error-deriv "Returns the function to compute the error derivative of the output layer @@ -623,12 +624,18 @@ as the training progresses." (fn [net _ _] (-> @net :training :algo))) +(defn initialize-train + "First step in train procedure" + [net ^TrainingSet trset] + (swap! net assoc-in [:training :state :state] :training) + (swap! net init-stats) + (swap! net assoc-in [:arch :labeltranslator] (-> trset :header :labeltranslator))) + (defmethod train :lbfgs [net ^TrainingSet trset nepochs] (future - (swap! net assoc-in [:training :state :state] :training) - (swap! net init-stats) + (initialize-train net trset) (let [l (-> @net :arch :layers) b (d/merge-batches (:batches trset)) w0 (weights-to-double-array (:weights @net)) @@ -654,8 +661,7 @@ :default [net ^TrainingSet trset nepochs] (future - (swap! net assoc-in [:training :state :state] :training) - (swap! net init-stats) + (initialize-train net trset) (let [maxep (+ nepochs (-> @net :training :stats :epochs)) all-batches (:batches trset)] (loop [batches all-batches] diff --git a/src/synaptic/util.clj b/src/synaptic/util.clj index b92dce3..4927099 100644 --- a/src/synaptic/util.clj +++ b/src/synaptic/util.clj @@ -146,13 +146,13 @@ (vec (for [i (range n)] (assoc (vec (repeat n 0)) i 1)))) (defn tobinary - "Encode labels to a vector with 0 and 1. Also returns the vector of + "Encode labels to a vector with 0 and 1. Also returns the map of unique labels to decode them." [labels] (let [uniquelabels (unique labels) lbcodes (bincodes (count uniquelabels)) lb2code (zipmap uniquelabels lbcodes)] - [(mapv lb2code labels) uniquelabels])) + [(mapv lb2code labels) (zipmap lbcodes uniquelabels)])) (defn frombinary "Decode a vector of 0 and 1 to the original label, based on a vector @@ -162,6 +162,19 @@ code2lb (zipmap lbcodes uniquelabels)] (mapv code2lb encodedlabels))) +; continuous scaling + +(defn tocontinuous + "Encode labels to vectors with numbers in range 0 to 1 + and return a function to decode them." + [labels] + (let [smallest-element (apply min (flatten labels)) + largest-element (apply max (flatten labels)) + scaling-factor (m/- largest-element smallest-element) + shifted-representation (m/- (m/matrix labels) smallest-element)] + [(m/to-vecs (m/div shifted-representation scaling-factor)) + #(m/+ smallest-element (m/mult (m/matrix %) scaling-factor))])) + ; Make clatrix matrices printable and readable in EDN format (defmethod print-method diff --git a/test/synaptic/datasets_test.clj b/test/synaptic/datasets_test.clj index fce0b8b..79f1e47 100644 --- a/test/synaptic/datasets_test.clj +++ b/test/synaptic/datasets_test.clj @@ -31,12 +31,12 @@ (is (= TrainingSet (type ts))) (let [bs (:batches ts) vs (:valid ts) - ulbs (-> ts :header :labels)] + ulbs (-> ts :header :labeltranslator)] (is (vector? bs)) (is (= 5 (count bs))) (is (every? #(= DataSet (type %)) bs)) (is (nil? vs)) - (is (= ["a" "b"] ulbs)) + (is (= {[0 1] "b", [1 0] "a"} ulbs)) (let [x (:x (first bs)) y (:y (first bs))] (is (m/matrix? x)) @@ -49,12 +49,12 @@ (is (= TrainingSet (type ts))) (let [bs (:batches ts) vs (:valid ts) - ulbs (-> ts :header :labels)] + ulbs (-> ts :header :labeltranslator)] (is (vector? bs)) (is (= 1 (count bs))) (is (= DataSet (type (first bs)))) (is (= DataSet (type vs))) - (is (= ["0" "1" "2" "3"] ulbs)) + (is (= {[0 0 0 1] "3", [0 0 1 0] "2", [0 1 0 0] "1", [1 0 0 0] "0"} ulbs)) (let [x (:x (first bs)) y (:y (first bs))] (is (m/matrix? x)) @@ -72,12 +72,12 @@ (is (= TrainingSet (type ts))) (let [bs (:batches ts) vs (:valid ts) - ulbs (-> ts :header :labels)] + ulbs (-> ts :header :labeltranslator)] (is (vector? bs)) (is (= 5 (count bs))) (is (= DataSet (type (first bs)))) (is (nil? vs)) - (is (= ["+" "-"] ulbs)) + (is (= {[0 1] "-", [1 0] "+"} ulbs)) (is (every? true? (map #(= [(map double %1)] (m/dense (:x %2))) smp bs))))))) diff --git a/test/synaptic/net_test.clj b/test/synaptic/net_test.clj index 6c1c3e3..f2aee1b 100644 --- a/test/synaptic/net_test.clj +++ b/test/synaptic/net_test.clj @@ -39,6 +39,9 @@ (testing "sigmoid" (is (m-quasi-equal? [[0.25 0.2]] (sigmoid (m/matrix [[(Math/log 1/3) (Math/log 1/4)]]))))) + (testing "relu" + (is (m-quasi-equal? [[0. 0.2]] + (relu (m/matrix [[-10. 0.2]]))))) (testing "hyperbolic-tangent" (is (m-quasi-equal? [[0.7615942 0.9640276 0.9950548]] (hyperbolic-tangent (m/matrix [[1 2 3]]))))) diff --git a/test/synaptic/util_test.clj b/test/synaptic/util_test.clj index 5380a94..e111945 100644 --- a/test/synaptic/util_test.clj +++ b/test/synaptic/util_test.clj @@ -104,7 +104,7 @@ (testing "tobinary should return the vector of unique labels and all labels encoded to binary vectors" (is (= [[[0 0 0 1] [1 0 0 0] [0 1 0 0] [0 0 0 1] [0 0 1 0] [1 0 0 0] [0 0 1 0]] - ["1" "2" "3" "8"]] + {[0 0 0 1] "8", [0 0 1 0] "3", [0 1 0 0] "2", [1 0 0 0] "1"}] (tobinary ["8" "1" "2" "8" "3" "1" "3"])))) (testing "frombinary should decode each label to its original value, based on a vector of unique labels" @@ -113,6 +113,14 @@ [[0 0 0 1] [1 0 0 0] [0 1 0 0] [0 0 0 1] [0 0 1 0] [1 0 0 0] [0 0 1 0]]))))) +(deftest test-continuous-scaling + (testing "tocontinuous should return the vector of unique labels and all labels + scaled to vectors with values in range 0 to 1, and a function to scale them back" + (is (= [[0.4 0.6] [0.8 1.0] [0.0 0.2]] + (first (tocontinuous [[1 2] [3 4] [-1 0]])))) + (is (= (m/matrix [[-1 4]]) + ((second (tocontinuous [[1 2] [3 4] [-1 0]])) [[0 1]]))))) + (deftest test-data-manipulation (testing "unique should return a sorted vector of unique values" (is (= ["a" "b" "c" "d" "x" "y" "z"]