From 7e4ff01e6498b0b9a2c4150836980ad8179c7f6e Mon Sep 17 00:00:00 2001 From: Ehud Ben-Reuven Date: Mon, 1 Dec 2014 11:11:52 -0500 Subject: [PATCH 1/6] ipython notebooks --- .gitignore | 3 + load.py | 10 +- notebooks/0_multiply.ipynb | 143 ++++ notebooks/1_linear_regression.ipynb | 240 ++++++ notebooks/2_logistic_regression.ipynb | 1095 +++++++++++++++++++++++++ notebooks/3_net.ipynb | 1002 ++++++++++++++++++++++ notebooks/4_modern_net.ipynb | 1075 ++++++++++++++++++++++++ notebooks/5_convolutional_net.ipynb | 303 +++++++ notebooks/KNN.ipynb | 102 +++ notebooks/index.ipynb | 292 +++++++ 10 files changed, 4260 insertions(+), 5 deletions(-) create mode 100644 notebooks/0_multiply.ipynb create mode 100644 notebooks/1_linear_regression.ipynb create mode 100644 notebooks/2_logistic_regression.ipynb create mode 100644 notebooks/3_net.ipynb create mode 100644 notebooks/4_modern_net.ipynb create mode 100644 notebooks/5_convolutional_net.ipynb create mode 100644 notebooks/KNN.ipynb create mode 100644 notebooks/index.ipynb diff --git a/.gitignore b/.gitignore index db4561e..ead0dec 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ docs/_build/ # PyBuilder target/ + +#ipython notebook +.ipynb_checkpoints/ diff --git a/load.py b/load.py index 25d931d..cae4cd5 100644 --- a/load.py +++ b/load.py @@ -13,19 +13,19 @@ def one_hot(x,n): def mnist(ntrain=60000,ntest=10000,onehot=True): data_dir = os.path.join(datasets_dir,'mnist/') - fd = open(os.path.join(data_dir,'train-images.idx3-ubyte')) + fd = open(os.path.join(data_dir,'train-images-idx3-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) trX = loaded[16:].reshape((60000,28*28)).astype(float) - fd = open(os.path.join(data_dir,'train-labels.idx1-ubyte')) + fd = open(os.path.join(data_dir,'train-labels-idx1-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) trY = loaded[8:].reshape((60000)) - fd = open(os.path.join(data_dir,'t10k-images.idx3-ubyte')) + fd = open(os.path.join(data_dir,'t10k-images-idx3-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) teX = loaded[16:].reshape((10000,28*28)).astype(float) - fd = open(os.path.join(data_dir,'t10k-labels.idx1-ubyte')) + fd = open(os.path.join(data_dir,'t10k-labels-idx1-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) teY = loaded[8:].reshape((10000)) @@ -45,4 +45,4 @@ def mnist(ntrain=60000,ntest=10000,onehot=True): trY = np.asarray(trY) teY = np.asarray(teY) - return trX,teX,trY,teY \ No newline at end of file + return trX,teX,trY,teY diff --git a/notebooks/0_multiply.ipynb b/notebooks/0_multiply.ipynb new file mode 100644 index 0000000..fb96d08 --- /dev/null +++ b/notebooks/0_multiply.ipynb @@ -0,0 +1,143 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:f13eb3b414e3e34aa7531d120d38fcd6602672de65630585a7952395a63c3f7e" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "import Theano" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "tensor is the section of theano that deals with data (like numpy)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "create Theano symbolic variables" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "a = T.scalar()\n", + "b = T.scalar()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "our model" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "y = a * b" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "compile to a python function (to CPU or GPU depending on configuration)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "multiply = theano.function(inputs=[a, b], outputs=y)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and use the function" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "multiply(1, 2)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 5, + "text": [ + "array(2.0)" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "multiply(3, 3)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 6, + "text": [ + "array(9.0)" + ] + } + ], + "prompt_number": 6 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/1_linear_regression.ipynb b/notebooks/1_linear_regression.ipynb new file mode 100644 index 0000000..2f8e70b --- /dev/null +++ b/notebooks/1_linear_regression.ipynb @@ -0,0 +1,240 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:65a56707d32e8e45ec327145d45b5a9a273bc13b6c18fc9b7e7bf8c84fdf53ba" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "imports" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "import numpy as np" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "generate data with noise" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trX = np.linspace(-1, 1, 101)\n", + "trY = 2 * trX + np.random.randn(*trX.shape) * 0.33" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "symbolic vriable initalization" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.scalar()\n", + "Y = T.scalar()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "our model" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w):\n", + " return X * w" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "model parameter initalization. hyper variables, with real value" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w = theano.shared(np.asarray(0., dtype=theano.config.floatX))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "y = model(X, w)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "metric to be optimized by model" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "cost = T.mean(T.sqr(y - Y))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "learning signal for parameters. Computes the gradient symbolically" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "gradient = T.grad(cost=cost, wrt=w)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 9 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "how to update in each step. 0.01 is the learning rate" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "updates = [[w, w - gradient * 0.01]]" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 10 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "compile to a python function. `allow_input_downcast=True` is set to ignore typing issue with Theano on different platforms (GPU handle only 32bit)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 11 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "iterate 100 times over the entire data (epoch.) In each epoch iterate over all the data samples" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "for i in range(100):\n", + " for x, y in zip(trX, trY):\n", + " train(x, y)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we should get something close to the true weight of the data: 2" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w.get_value()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 13, + "text": [ + "array(1.9382810308787022)" + ] + } + ], + "prompt_number": 13 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/2_logistic_regression.ipynb b/notebooks/2_logistic_regression.ipynb new file mode 100644 index 0000000..568c985 --- /dev/null +++ b/notebooks/2_logistic_regression.ipynb @@ -0,0 +1,1095 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:8e7467687cabf437572893b54a343b6de5be3c2f06c6247823de13c0201bf14a" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our input is now the handwritten digits, each is a vector" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "import numpy as np" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GeForce GT 650M\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Convert to correct dtype" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "initalize model parameters (small random guassian distribution)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In our model we will replace scalars with vectors" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.fmatrix()\n", + "Y = T.fmatrix()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and weights with matrices" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w = init_weights((784, 10))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and use softmax convert our weighted input to probability" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w):\n", + " return T.nnet.softmax(T.dot(X, w))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "probability output" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "py_x = model(X, w)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 9 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "maxima prediction" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "y_pred = T.argmax(py_x, axis=1)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 10 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "different cost function, maximize the value that is correct and minize the others" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n", + "gradient = T.grad(cost=cost, wrt=w)\n", + "update = [[w, w - gradient * 0.05]]\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=update, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 11 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "now we also have a predict function that will be used on the test data" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "predict = theano.function(inputs=[X], outputs=y_pred, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "train on mini-batches of 128 samples each" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0 0.918269230769\n", + "1" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.918569711538\n", + "2" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.918970352564\n", + "3" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919070512821\n", + "4" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919170673077\n", + "5" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919671474359\n", + "6" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919871794872\n", + "7" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920372596154\n", + "8" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920572916667\n", + "9" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920372596154\n", + "10" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920572916667\n", + "11" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920673076923\n", + "12" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920572916667\n", + "13" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920673076923\n", + "14" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921274038462\n", + "15" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "16" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921674679487\n", + "17" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921875\n", + "18" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921674679487\n", + "19" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "20" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "21" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921774839744\n", + "22" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921975160256\n", + "23" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921875\n", + "24" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921574519231\n", + "25" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921875\n", + "26" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922075320513\n", + "27" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922275641026\n", + "28" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922275641026\n", + "29" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922375801282\n", + "30" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922576121795\n", + "31" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922375801282\n", + "32" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922576121795\n", + "33" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922576121795\n", + "34" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922676282051\n", + "35" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922976762821\n", + "36" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922776442308\n", + "37" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922776442308\n", + "38" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922876602564\n", + "39" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923076923077\n", + "40" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923076923077\n", + "41" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.92327724359\n", + "42" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923377403846\n", + "43" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923677884615\n", + "44" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923778044872\n", + "45" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "46" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "47" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "48" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "49" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923878205128\n", + "50" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "51" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924078525641\n", + "52" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "53" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924078525641\n", + "54" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924078525641\n", + "55" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924178685897\n", + "56" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924479166667\n", + "57" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "58" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "59" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "60" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "61" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "62" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "63" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "64" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "65" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "66" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "67" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "68" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "69" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "70" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "71" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "72" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "73" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "74" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "75" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "76" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "77" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "78" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "79" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "80" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "81" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "82" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "83" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "84" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924979967949\n", + "85" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "86" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "87" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "88" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "89" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "90" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "91" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "92" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "93" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "94" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "95" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "96" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "97" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "98" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "99" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "CPU times: user 1min 17s, sys: 4.96 s, total: 1min 22s\n", + "Wall time: 1min 23s\n" + ] + } + ], + "prompt_number": 14 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "prediction is not as good as kNN" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "28.5 sec for CPU. GPU was 1min23s or x3 slower :-(" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/3_net.ipynb b/notebooks/3_net.ipynb new file mode 100644 index 0000000..56f4443 --- /dev/null +++ b/notebooks/3_net.ipynb @@ -0,0 +1,1002 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:fc684fd1168901d7d18baf83d926aaf0042beea79127f8b118f4da677b8dc70b" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An \"old\" network before 2006" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "import numpy as np\n", + "\n", + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)\n", + "\n", + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GeForce GT 650M\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generalize to compute stochastic gradient descent (SGD) on all model parametes. Note that SGD has a fixed learning rate which slow things down as you near the goal" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def sgd(cost, params, lr=0.05):\n", + " grads = T.grad(cost=cost, wrt=params)\n", + " updates = []\n", + " for p, g in zip(params, grads):\n", + " updates.append([p, p - g * lr])\n", + " return updates" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "logistic regression (LR) model is replaced with a neural network (NN) model with one hidden layer" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w_h, w_o):\n", + " h = T.nnet.sigmoid(T.dot(X, w_h))\n", + " pyx = T.nnet.softmax(T.dot(h, w_o))\n", + " return pyx" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.fmatrix()\n", + "Y = T.fmatrix()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have 625 hidden units" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w_h = init_weights((784, 625))\n", + "w_o = init_weights((625, 10))\n", + "\n", + "py_x = model(X, w_h, w_o)\n", + "y_x = T.argmax(py_x, axis=1)\n", + "\n", + "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n", + "params = [w_h, w_o]\n", + "updates = sgd(cost, params)\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n", + "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0 0.708233173077\n", + "1" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.831129807692\n", + "2" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.868189102564\n", + "3" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.883413461538\n", + "4" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.889623397436\n", + "5" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.894931891026\n", + "6" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.8984375\n", + "7" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.901642628205\n", + "8" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.90484775641\n", + "9" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.907251602564\n", + "10" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.909154647436\n", + "11" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.911157852564\n", + "12" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.912960737179\n", + "13" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.913361378205\n", + "14" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.914763621795\n", + "15" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.915765224359\n", + "16" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.916466346154\n", + "17" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.916967147436\n", + "18" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.917367788462\n", + "19" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.918269230769\n", + "20" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919070512821\n", + "21" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919571314103\n", + "22" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919571314103\n", + "23" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919571314103\n", + "24" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920272435897\n", + "25" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920873397436\n", + "26" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921073717949\n", + "27" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "28" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921774839744\n", + "29" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922475961538\n", + "30" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922475961538\n", + "31" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922776442308\n", + "32" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923677884615\n", + "33" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "34" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924979967949\n", + "35" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.925280448718\n", + "36" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.926081730769\n", + "37" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.926181891026\n", + "38" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.926682692308\n", + "39" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.927483974359\n", + "40" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.927984775641\n", + "41" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.928485576923\n", + "42" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.928886217949\n", + "43" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.929487179487\n", + "44" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.929887820513\n", + "45" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.930588942308\n", + "46" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.931189903846\n", + "47" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.93108974359\n", + "48" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.931690705128\n", + "49" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.932391826923\n", + "50" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.933994391026\n", + "51" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.934495192308\n", + "52" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.93499599359\n", + "53" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.935697115385\n", + "54" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.936698717949\n", + "55" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.937600160256\n", + "56" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.938000801282\n", + "57" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.938401442308\n", + "58" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.939002403846\n", + "59" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.939503205128\n", + "60" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.939903846154\n", + "61" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.940604967949\n", + "62" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.941205929487\n", + "63" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.941706730769\n", + "64" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.941907051282\n", + "65" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.942608173077\n", + "66" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.943309294872\n", + "67" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.943709935897\n", + "68" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.944110576923\n", + "69" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.945212339744\n", + "70" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.945913461538\n", + "71" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.946314102564\n", + "72" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.946614583333\n", + "73" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.94671474359\n", + "74" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.946915064103\n", + "75" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.947215544872\n", + "76" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.947315705128\n", + "77" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.94781650641\n", + "78" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.948617788462\n", + "79" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.949018429487\n", + "80" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.949719551282\n", + "81" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.950721153846\n", + "82" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.951021634615\n", + "83" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.951522435897\n", + "84" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.951923076923\n", + "85" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.952223557692\n", + "86" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.952924679487\n", + "87" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.953525641026\n", + "88" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.953826121795\n", + "89" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.954427083333\n", + "90" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955028044872\n", + "91" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955228365385\n", + "92" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955528846154\n", + "93" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955729166667\n", + "94" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956029647436\n", + "95" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956330128205\n", + "96" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956330128205\n", + "97" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956730769231\n", + "98" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956931089744\n", + "99" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.957331730769\n", + "CPU times: user 2min 9s, sys: 5.21 s, total: 2min 14s\n", + "Wall time: 2min 14s\n" + ] + } + ], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "CPU 2min 56s, GPU 2min 14s. The result are 0.957 and not 0.98 as discussed in the video lecture" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/4_modern_net.ipynb b/notebooks/4_modern_net.ipynb new file mode 100644 index 0000000..acc84a2 --- /dev/null +++ b/notebooks/4_modern_net.ipynb @@ -0,0 +1,1075 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:a24946d3f359951c3345d365cd78ed271a7c2fe1ce2db20406363ca25c9b1806" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "two main changes: replace sigmoid with rectifier (LeRU). The problem of overfitting of old NN is solved by adding noise. This allows to add hidden layers (in the fast this caused overfitting.)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n", + "import numpy as np\n", + "\n", + "srng = RandomStreams()\n", + "\n", + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)\n", + "\n", + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "replacement for sigmoid function" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def rectify(X):\n", + " return T.maximum(X, 0.)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "modify softmax to remove max value to avoid explosition of exp (numeric stability)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def softmax(X):\n", + " e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n", + " return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we use a running RMS average of the gradient to scale the gradient size" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n", + " grads = T.grad(cost=cost, wrt=params)\n", + " updates = []\n", + " for p, g in zip(params, grads):\n", + " acc = theano.shared(p.get_value() * 0.) # acc is allocated for each parameter (p) with 0 values with the shape of p\n", + " acc_new = rho * acc + (1 - rho) * g ** 2\n", + " gradient_scaling = T.sqrt(acc_new + epsilon)\n", + " g = g / gradient_scaling\n", + " updates.append((acc, acc_new))\n", + " updates.append((p, p - lr * g))\n", + " return updates" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "inject noise with dropout. randomly set to zero some (p) of the nodes. We have to compensate by scaling the nodes that were not dropped out." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def dropout(X, p=0.):\n", + " if p > 0:\n", + " retain_prob = 1 - p\n", + " X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n", + " X /= retain_prob\n", + " return X" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "new model has two hidden layers of 625 nodes" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden):\n", + " X = dropout(X, p_drop_input)\n", + " h = rectify(T.dot(X, w_h))\n", + "\n", + " h = dropout(h, p_drop_hidden)\n", + " h2 = rectify(T.dot(h, w_h2))\n", + "\n", + " h2 = dropout(h2, p_drop_hidden)\n", + " py_x = softmax(T.dot(h2, w_o))\n", + " return h, h2, py_x" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.fmatrix()\n", + "Y = T.fmatrix()\n", + "\n", + "w_h = init_weights((784, 625))\n", + "w_h2 = init_weights((625, 625))\n", + "w_o = init_weights((625, 10))\n", + "\n", + "noise_h, noise_h2, noise_py_x = model(X, w_h, w_h2, w_o, 0.2, 0.5)\n", + "h, h2, py_x = model(X, w_h, w_h2, w_o, 0., 0.)\n", + "y_x = T.argmax(py_x, axis=1)\n", + "\n", + "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n", + "params = [w_h, w_h2, w_o]\n", + "updates = RMSprop(cost, params, lr=0.001)\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n", + "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "/Users/udi/anaconda/lib/python2.7/site-packages/theano/sandbox/rng_mrg.py:1195: UserWarning: MRG_RandomStreams Can't determine #streams from size (Shape.0), guessing 60*256\n", + " nstreams = self.n_streams(size)\n" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0 0.940104166667\n", + "1" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.965344551282\n", + "2" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.972856570513\n", + "3" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.973858173077\n", + "4" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.975260416667\n", + "5" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.978665865385\n", + "6" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.978866185897\n", + "7" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.979166666667\n", + "8" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.982171474359\n", + "9" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.980268429487\n", + "10" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.982171474359\n", + "11" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98297275641\n", + "12" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983072916667\n", + "13" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983673878205\n", + "14" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984074519231\n", + "15" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983673878205\n", + "16" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983373397436\n", + "17" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984074519231\n", + "18" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984775641026\n", + "19" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983874198718\n", + "20" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985977564103\n", + "21" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984174679487\n", + "22" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984675480769\n", + "23" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984274839744\n", + "24" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984775641026\n", + "25" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "26" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985176282051\n", + "27" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984575320513\n", + "28" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985076121795\n", + "29" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985576923077\n", + "30" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98577724359\n", + "31" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98577724359\n", + "32" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98687900641\n", + "33" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985376602564\n", + "34" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986077724359\n", + "35" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985877403846\n", + "36" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985677083333\n", + "37" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98687900641\n", + "38" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "39" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98577724359\n", + "40" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985376602564\n", + "41" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986578525641\n", + "42" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986478365385\n", + "43" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985677083333\n", + "44" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986278044872\n", + "45" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "46" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "47" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.988581730769\n", + "48" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986378205128\n", + "49" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987479967949\n", + "50" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985977564103\n", + "51" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98687900641\n", + "52" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987580128205\n", + "53" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "54" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985877403846\n", + "55" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986077724359\n", + "56" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "57" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986077724359\n", + "58" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "59" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985877403846\n", + "60" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "61" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "62" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "63" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "64" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987780448718\n", + "65" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "66" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987079326923\n", + "67" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987079326923\n", + "68" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "69" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "70" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986177884615\n", + "71" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986378205128\n", + "72" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "73" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "74" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987780448718\n", + "75" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "76" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987179487179\n", + "77" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987980769231\n", + "78" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985977564103\n", + "79" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986578525641\n", + "80" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "81" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.988782051282\n", + "82" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "83" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "84" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987479967949\n", + "85" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987980769231\n", + "86" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987479967949\n", + "87" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987580128205\n", + "88" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986578525641\n", + "89" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987680288462\n", + "90" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987880608974\n", + "91" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "92" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "93" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "94" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986778846154\n", + "95" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987780448718\n", + "96" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987580128205\n", + "97" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987980769231\n", + "98" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "99" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987079326923\n", + "CPU times: user 4min 53s, sys: 30.4 s, total: 5min 23s\n", + "Wall time: 5min 23s\n" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "CPU 17min 4s, GPU 5min 23s" + ], + "language": "python", + "metadata": {}, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "98.8% not 99% in video" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb new file mode 100644 index 0000000..c640612 --- /dev/null +++ b/notebooks/5_convolutional_net.ipynb @@ -0,0 +1,303 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:008f881ba39f22a33c3616104c79c772db5c6b9ec1eff75607de7c23b1a078e4" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use 2D convolution - respect the 2D structure of the data by running the same network (same weights) on small 2D patches of the image." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "import os\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n", + "import numpy as np\n", + "from theano.tensor.nnet.conv import conv2d\n", + "from theano.tensor.signal.downsample import max_pool_2d\n", + "\n", + "srng = RandomStreams()\n", + "\n", + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)\n", + "\n", + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))\n", + "\n", + "def rectify(X):\n", + " return T.maximum(X, 0.)\n", + "\n", + "def softmax(X):\n", + " e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n", + " return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n", + "\n", + "def dropout(X, p=0.):\n", + " if p > 0:\n", + " retain_prob = 1 - p\n", + " X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX, nstreams=60*256)\n", + " X /= retain_prob\n", + " return X\n", + "\n", + "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n", + " grads = T.grad(cost=cost, wrt=params)\n", + " updates = []\n", + " for p, g in zip(params, grads):\n", + " acc = theano.shared(p.get_value() * 0.)\n", + " acc_new = rho * acc + (1 - rho) * g ** 2\n", + " gradient_scaling = T.sqrt(acc_new + epsilon)\n", + " g = g / gradient_scaling\n", + " updates.append((acc, acc_new))\n", + " updates.append((p, p - lr * g))\n", + " return updates" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GeForce GT 650M\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "block to compute conv->activate->pool->noise" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden):\n", + " # block to compute conv->activate->pool->noise\n", + " l1a = rectify(conv2d(X, w, border_mode='full'))\n", + " l1 = max_pool_2d(l1a, (2, 2))\n", + " l1 = dropout(l1, p_drop_conv)\n", + "\n", + " l2a = rectify(conv2d(l1, w2))\n", + " l2 = max_pool_2d(l2a, (2, 2))\n", + " l2 = dropout(l2, p_drop_conv)\n", + "\n", + " l3a = rectify(conv2d(l2, w3))\n", + " l3b = max_pool_2d(l3a, (2, 2))\n", + " # return back to original vector represnation\n", + " l3 = T.flatten(l3b, outdim=2)\n", + " l3 = dropout(l3, p_drop_conv)\n", + "\n", + " l4 = rectify(T.dot(l3, w4))\n", + " l4 = dropout(l4, p_drop_hidden)\n", + "\n", + " pyx = softmax(T.dot(l4, w_o))\n", + " return l1, l2, l3, l4, pyx" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "convert data from vector to color images, but we have just one color channel " + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trX = trX.reshape(-1, 1, 28, 28)\n", + "teX = teX.reshape(-1, 1, 28, 28)\n", + "\n", + "X = T.ftensor4()\n", + "Y = T.fmatrix()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "covolution weights (n_kernels, n_channels, kernel_w, kernel_h). We will use 3x3 kernel. Smallest possible. Instead of bigger kernels use more layers. It looks as if each convolution layer is twice as big as the previous one, but keep in mind that we perform maxpool which will reduce the dimension by factor of two" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w = init_weights((32, 1, 3, 3))\n", + "w2 = init_weights((64, 32, 3, 3))\n", + "w3 = init_weights((128, 64, 3, 3))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "last layer is flat but its input size must match the size of the previous convolution layer" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w4 = init_weights((128 * 3 * 3, 625))\n", + "w_o = init_weights((625, 10))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "noise during training" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, w_o, 0.2, 0.5)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "noiseless model for testing" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, w_o, 0., 0.)\n", + "y_x = T.argmax(py_x, axis=1)\n", + "\n", + "\n", + "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n", + "params = [w, w2, w3, w4, w_o]\n", + "updates = RMSprop(cost, params, lr=0.001)\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n", + "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " print '%d:%.4f'%(start/128,cost),\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " print\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print '###',i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0:2.3026 " + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "1:2.3026 " + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "2:2.3026 " + ] + } + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/KNN.ipynb b/notebooks/KNN.ipynb new file mode 100644 index 0000000..6fbe425 --- /dev/null +++ b/notebooks/KNN.ipynb @@ -0,0 +1,102 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:2a506df8c7b7baaab1282610827bb2333815c5150a1098d957f2d4d75d49ef8b" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simplest way to perform classification of the handwritten digits is to use nearest neighbor" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "import os\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trLabel, teLabel = load.mnist(onehot=False)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import sklearn.neighbors\n", + "clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=1)\n", + "clf.fit(trX,trLabel)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 3, + "text": [ + "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n", + " metric_params=None, n_neighbors=1, p=2, weights='uniform')" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "tePredict = clf.predict(teX)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "np.mean(tePredict == teLabel)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 5, + "text": [ + "0.96909999999999996" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which works very well but not good enough (0.995 or more)" + ] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/index.ipynb b/notebooks/index.ipynb new file mode 100644 index 0000000..be232ed --- /dev/null +++ b/notebooks/index.ipynb @@ -0,0 +1,292 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:cb17786a324ccc4066b6446428b8d06831376bd8ca1f416866ee0d68af35f208" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Follow the lecture [Introduction to Deep Learning with Python](https://www.youtube.com/watch?v=S75EdAcXHKk)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from IPython.display import YouTubeVideo\n", + "YouTubeVideo('S75EdAcXHKk')" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "html": [ + "\n", + " \n", + " " + ], + "metadata": {}, + "output_type": "pyout", + "prompt_number": 1, + "text": [ + "" + ] + } + ], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "after performing the set described below you can walk through the notebooks:\n", + " \n", + "* [KNN](./KNN.ipynb)\n", + "* [Mulitply two numbers with Theano](./0_multiply.ipynb)\n", + "* [Linear regression](./1_linear_regression.ipynb)\n", + "* [Logistic Regression](./2_logistic_regression.ipynb)\n", + "* [Neural Network](./3_net.ipynb)\n", + "* [Modern Neural Network](./4_modern_net.ipynb)\n", + "* [Convolution Neural Network](./5_convolutional_net.ipynb)" + ] + }, + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Get Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "download mnist files from http://yann.lecun.com/exdb/mnist/ and open the zipped file into `~/Downloads/lisa/data/mnist/`" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have 60K images of handwritten numbers for training and 10K for testing. Each image has 28*28 gray-level pixels" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trX.shape, teX.shape, trY.shape, teY.shape" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 3, + "text": [ + "((60000, 784), (10000, 784), (60000, 10), (10000, 10))" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "np.min(trX),np.max(trX),np.min(teX),np.max(teX)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 4, + "text": [ + "(0.0, 1.0, 0.0, 1.0)" + ] + } + ], + "prompt_number": 4 + }, + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Installation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install Theano, it is highly recommended to [install using anaconda](http://deeplearning.net/software/theano/install.html#anaconda-1-5). If you also [installed MKL](http://continuum.io/blog/mkl-optimizations), which is also highly recommended, you will need to export the following environment variables before running `ipython notebook`\n", + "\n", + " export DYLD_FALLBACK_LIBRARY_PATH=$HOME/anaconda/lib:$DYLD_FALLBACK_LIBRARY_PATH" + ] + }, + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Using GPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need to install CUDA and export the following environment variables before running `ipython notebook`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "export DYLD_FALLBACK_LIBRARY_PATH=/Developer/NVIDIA/CUDA-6.5/lib:$DYLD_FALLBACK_LIBRARY_PATH\n", + "export PATH=/Developer/NVIDIA/CUDA-6.5/bin:$PATH\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "you can control if CPU or GPU is used inside the notebook, however, you can only set it once. You need to restart the notebook if you want to change it" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and check if you are using CPU or GPU" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from theano import function, config, shared, sandbox\n", + "import theano.tensor as T\n", + "import numpy\n", + "import time\n", + "\n", + "vlen = 10 * 30 * 768 # 10 x #cores x # threads per core\n", + "iters = 1000\n", + "\n", + "rng = numpy.random.RandomState(22)\n", + "x = shared(numpy.asarray(rng.rand(vlen), config.floatX))\n", + "f = function([], T.exp(x))\n", + "print f.maker.fgraph.toposort()\n", + "t0 = time.time()\n", + "for i in xrange(iters):\n", + " r = f()\n", + "t1 = time.time()\n", + "print 'Looping %d times took' % iters, t1 - t0, 'seconds'\n", + "print 'Result is', r\n", + "if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]):\n", + " print 'Used the cpu'\n", + "else:\n", + " print 'Used the gpu'" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "[GpuElemwise{exp,no_inplace}(), HostFromGpu(GpuElemwise{exp,no_inplace}.0)]\n", + "Looping 1000 times took" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.656645059586 seconds\n", + "Result is [ 1.23178029 1.61879349 1.52278066 ..., 2.20771813 2.29967761\n", + " 1.62323296]\n", + "Used the gpu\n" + ] + }, + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GeForce GT 650M\n" + ] + } + ], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this example the GPU is x4-x5 faster than CPU" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file From 5f273a0c085c61daf703981cadb853a3851ce61e Mon Sep 17 00:00:00 2001 From: Ehud Ben-Reuven Date: Mon, 1 Dec 2014 11:15:29 -0500 Subject: [PATCH 2/6] link to nbviewer --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0db9e9f..f879126 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,6 @@ Theano-Tutorials ================ Bare bones introduction to machine learning from linear regression to convolutional neural networks using Theano. + +See [ipython notebooks](http://nbviewer.ipython.org/github/udibr/Theano-Tutorials/blob/master/notebooks/index.ipynb) + From 3025daa443575e00344be5c136af450a5bb9f567 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 1 Dec 2014 17:03:22 +0000 Subject: [PATCH 3/6] use AWS --- notebooks/5_convolutional_net.ipynb | 53 ++- notebooks/index.ipynb | 607 +++++++++++++++++++++++++++- 2 files changed, 621 insertions(+), 39 deletions(-) diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb index c640612..bb38200 100644 --- a/notebooks/5_convolutional_net.ipynb +++ b/notebooks/5_convolutional_net.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:008f881ba39f22a33c3616104c79c772db5c6b9ec1eff75607de7c23b1a078e4" + "signature": "sha256:a9995f8de8a5cadbb7f85f9979e6849bb9c8ffecda36e7002b4346a743a4004d" }, "nbformat": 3, "nbformat_minor": 0, @@ -34,6 +34,13 @@ "outputs": [], "prompt_number": 1 }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On OSX this example is really slow and the GPU cause the Mac to shutdown! So run it on AWS with GPU" + ] + }, { "cell_type": "code", "collapsed": false, @@ -97,7 +104,7 @@ "output_type": "stream", "stream": "stderr", "text": [ - "Using gpu device 0: GeForce GT 650M\n" + "Using gpu device 0: GRID K520\n" ] } ], @@ -244,7 +251,19 @@ ], "language": "python", "metadata": {}, - "outputs": [] + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "/usr/local/lib/python2.7/dist-packages/theano/tensor/subtensor.py:110: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n", + " start in [None, 0] or\n", + "/usr/local/lib/python2.7/dist-packages/theano/tensor/subtensor.py:114: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n", + " stop in [None, length, maxsize] or\n" + ] + } + ], + "prompt_number": 9 }, { "cell_type": "code", @@ -254,9 +273,9 @@ "for i in range(100):\n", " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", " cost = train(trX[start:end], trY[start:end])\n", - " print '%d:%.4f'%(start/128,cost),\n", + " #print '%d:%.4f'%(start/128,cost),\n", + " #print\n", " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", - " print\n", " errs = []\n", " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", @@ -264,29 +283,7 @@ ], "language": "python", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "0:2.3026 " - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "1:2.3026 " - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - "2:2.3026 " - ] - } - ] + "outputs": [] }, { "cell_type": "code", diff --git a/notebooks/index.ipynb b/notebooks/index.ipynb index be232ed..0a74ee8 100644 --- a/notebooks/index.ipynb +++ b/notebooks/index.ipynb @@ -77,6 +77,44 @@ "download mnist files from http://yann.lecun.com/exdb/mnist/ and open the zipped file into `~/Downloads/lisa/data/mnist/`" ] }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%bash\n", + "mkdir -p ~/Downloads/lisa/data/mnist/\n", + "cd ~/Downloads/lisa/data/mnist/\n", + "#wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n", + "#wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n", + "#wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n", + "#wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n", + "gunzip *.gz" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "!ls ~/Downloads/lisa/data/mnist/" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "t10k-images-idx3-ubyte\ttrain-images-idx3-ubyte\r\n", + "t10k-labels-idx1-ubyte\ttrain-labels-idx1-ubyte\r\n" + ] + } + ], + "prompt_number": 5 + }, { "cell_type": "code", "collapsed": false, @@ -94,7 +132,7 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 2 + "prompt_number": 6 }, { "cell_type": "markdown", @@ -115,18 +153,19 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 3, + "prompt_number": 7, "text": [ "((60000, 784), (10000, 784), (60000, 10), (10000, 10))" ] } ], - "prompt_number": 3 + "prompt_number": 7 }, { "cell_type": "code", "collapsed": false, "input": [ + "import numpy as np\n", "np.min(trX),np.max(trX),np.min(teX),np.max(teX)" ], "language": "python", @@ -135,13 +174,13 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 4, + "prompt_number": 9, "text": [ "(0.0, 1.0, 0.0, 1.0)" ] } ], - "prompt_number": 4 + "prompt_number": 9 }, { "cell_type": "heading", @@ -155,11 +194,546 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install Theano, it is highly recommended to [install using anaconda](http://deeplearning.net/software/theano/install.html#anaconda-1-5). If you also [installed MKL](http://continuum.io/blog/mkl-optimizations), which is also highly recommended, you will need to export the following environment variables before running `ipython notebook`\n", + "Install Theano. On OS X it is highly recommended to [install using anaconda](http://deeplearning.net/software/theano/install.html#anaconda-1-5). If you also [installed MKL](http://continuum.io/blog/mkl-optimizations), which is also highly recommended, you will need to export the following environment variables before running `ipython notebook`\n", "\n", " export DYLD_FALLBACK_LIBRARY_PATH=$HOME/anaconda/lib:$DYLD_FALLBACK_LIBRARY_PATH" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On AWS I started from the following community image \"Ubuntu-14.04-Caffe-GPU - ami-588d0030\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "install Theano" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "!sudo pip install theano" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Downloading/unpacking theano\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " Downloading Theano-0.6.0.tar.gz (1.8MB): \r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 4.1kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 8.2kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 12kB \r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 16kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 20kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 24kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 28kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 32kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 36kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 40kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 45kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 49kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 53kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 57kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 61kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 65kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 69kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 73kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 77kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 81kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 86kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 90kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 94kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 98kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 102kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 106kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 110kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 114kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 118kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 122kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 126kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 131kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 135kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 139kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 143kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 147kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 151kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 155kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 159kB" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 163kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 167kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 172kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 176kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 180kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 184kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 188kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 192kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 196kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 200kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 204kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 208kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 212kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 217kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 221kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 225kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 229kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 233kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 237kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 241kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 245kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 249kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 253kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 258kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 262kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 266kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 270kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 274kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 278kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 282kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 286kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 290kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 294kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 299kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 303kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 307kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 311kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 315kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 319kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 323kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 327kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 331kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 335kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 339kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 344kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 348kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 352kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 356kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 360kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 364kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 368kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 372kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 376kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 380kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 385kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 389kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 393kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 397kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 401kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 405kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 409kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 413kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 417kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 421kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 425kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 430kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 434kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 438kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 442kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 446kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 450kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 454kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 458kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 462kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 466kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 471kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 475kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 479kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 483kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 487kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 491kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 495kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 499kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 503kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 507kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 512kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 516kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 520kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 524kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 528kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 532kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 536kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 540kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 544kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 548kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 552kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 557kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 561kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 565kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 569kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 573kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 577kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 581kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 585kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 589kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 593kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 598kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 602kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 606kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 610kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 614kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 618kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 622kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 626kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 630kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 634kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 638kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 643kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 647kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 651kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 655kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 659kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 663kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 667kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 671kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 675kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 679kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 684kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 688kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 692kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 696kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 700kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 704kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 708kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 712kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 716kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 720kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 724kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 729kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 733kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 737kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 741kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 745kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 749kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 753kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 757kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 761kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 765kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 770kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 774kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 778kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 782kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 786kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 790kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 794kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 798kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 802kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 806kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 811kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 815kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 819kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 823kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 827kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 831kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 835kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 839kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 843kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 847kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 851kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 856kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 860kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 864kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 868kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 872kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 876kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 880kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 884kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 888kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 892kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 897kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 901kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 905kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 909kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 913kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 917kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 921kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 925kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 929kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 933kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 937kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 942kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 946kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 950kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 954kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 958kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 962kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 966kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 970kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 974kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 978kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 983kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 987kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 991kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 995kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 999kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 100% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): \r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1.8MB downloaded\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " Running setup.py (path:/tmp/pip_build_root/theano/setup.py) egg_info for package theano\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " \r\n", + " warning: manifest_maker: MANIFEST.in, line 7: 'recursive-include' expects ...\r\n", + " \r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Requirement already satisfied (use --upgrade to upgrade): numpy>=1.5.0 in /usr/local/lib/python2.7/dist-packages/numpy-1.9.0-py2.7-linux-x86_64.egg (from theano)\r\n", + "Requirement already satisfied (use --upgrade to upgrade): scipy>=0.7.2 in /usr/local/lib/python2.7/dist-packages (from theano)\r\n", + "Installing collected packages: theano\r\n", + " Running setup.py install for theano\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " changing mode of build/scripts-2.7/theano-cache from 644 to 755\r\n", + " changing mode of build/scripts-2.7/theano-nose from 644 to 755\r\n", + " changing mode of build/scripts-2.7/theano-test from 644 to 755\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " \r\n", + " warning: manifest_maker: MANIFEST.in, line 7: 'recursive-include' expects ...\r\n", + " \r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " changing mode of /usr/local/bin/theano-nose to 755\r\n", + " changing mode of /usr/local/bin/theano-cache to 755\r\n", + " changing mode of /usr/local/bin/theano-test to 755\r\n", + "Successfully installed theano\r\n", + "Cleaning up...\r\n" + ] + } + ], + "prompt_number": 14 + }, { "cell_type": "heading", "level": 1, @@ -179,12 +753,23 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "On OSX\n", "```bash\n", "export DYLD_FALLBACK_LIBRARY_PATH=/Developer/NVIDIA/CUDA-6.5/lib:$DYLD_FALLBACK_LIBRARY_PATH\n", "export PATH=/Developer/NVIDIA/CUDA-6.5/bin:$PATH\n", "```" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On AWS\n", + "```bash\n", + "export PATH=/usr/local/cuda-6.5/bin/:$PATH\n", + "```" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -203,7 +788,7 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 5 + "prompt_number": 1 }, { "cell_type": "markdown", @@ -254,7 +839,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.656645059586 seconds\n", + " 0.738588094711 seconds\n", "Result is [ 1.23178029 1.61879349 1.52278066 ..., 2.20771813 2.29967761\n", " 1.62323296]\n", "Used the gpu\n" @@ -264,17 +849,17 @@ "output_type": "stream", "stream": "stderr", "text": [ - "Using gpu device 0: GeForce GT 650M\n" + "Using gpu device 0: GRID K520\n" ] } ], - "prompt_number": 6 + "prompt_number": 2 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "In this example the GPU is x4-x5 faster than CPU" + "On OSX, for this example, the GPU is x4-x5 faster than CPU" ] }, { From 4cf1d52e348ffdfde0e17bde2e8169fdf79f03ff Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 1 Dec 2014 19:13:20 +0000 Subject: [PATCH 4/6] use AWS --- notebooks/5_convolutional_net.ipynb | 599 +++++++++++++++++++++++++++- 1 file changed, 596 insertions(+), 3 deletions(-) diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb index bb38200..f92ca4e 100644 --- a/notebooks/5_convolutional_net.ipynb +++ b/notebooks/5_convolutional_net.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:a9995f8de8a5cadbb7f85f9979e6849bb9c8ffecda36e7002b4346a743a4004d" + "signature": "sha256:c49098d0b225cb300e6bb8f70e3b23c1d99940dd9fb0281aec2ab86ae0ad5f90" }, "nbformat": 3, "nbformat_minor": 0, @@ -38,7 +38,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "On OSX this example is really slow and the GPU cause the Mac to shutdown! So run it on AWS with GPU" + "On OSX this example is really slow and the GPU cause the Mac to shutdown! So run it on AWS with GPU (e.g. g2.2xlarge )" ] }, { @@ -283,7 +283,600 @@ ], "language": "python", "metadata": {}, - "outputs": [] + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "### 0 0.934795673077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 1 0.973657852564\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 2 0.982872596154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 3 0.987179487179\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 4 0.987479967949\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 5 0.989783653846\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 6 0.987379807692\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 7 0.991987179487\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 8 0.991786858974\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 9 0.991786858974\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 10 0.992287660256\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 11 0.992487980769\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 12 0.991786858974\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 13 0.9921875\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 14 0.992588141026\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 15 0.993689903846\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 16 0.992788461538\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 17 0.993289262821\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 18 0.992287660256\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 19 0.992588141026\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 20 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 21 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 22 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 23 0.993289262821\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 24 0.993189102564\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 25 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 26 0.993389423077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 27 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 28 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 29 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 30 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 31 0.993289262821\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 32 0.993189102564\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 33 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 34 0.994190705128\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 35 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 36 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 37 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 38 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 39 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 40 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 41 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 42 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 43 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 44 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 45 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 46 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 47 0.993389423077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 48 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 49 0.993389423077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 50 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 51 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 52 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 53 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 54 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 55 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 56 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 57 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 58 0.99469150641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 59 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 60 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 61 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 62 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 63 0.994190705128\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 64 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 65 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 66 0.994491185897\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 67 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 68 0.995092147436\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 69 0.995192307692\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 70 0.99469150641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 71 0.994491185897\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 72 0.995092147436\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 73 0.994891826923\n", + "###" + ] + } + ] }, { "cell_type": "code", From 59a5c53e183d024c542a4bce6f311a6c5859cd93 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 1 Dec 2014 19:24:50 +0000 Subject: [PATCH 5/6] use AWS --- notebooks/5_convolutional_net.ipynb | 218 +++++++++++++++++++++++++++- 1 file changed, 217 insertions(+), 1 deletion(-) diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb index f92ca4e..9503c00 100644 --- a/notebooks/5_convolutional_net.ipynb +++ b/notebooks/5_convolutional_net.ipynb @@ -1,7 +1,7 @@ { "metadata": { "name": "", - "signature": "sha256:c49098d0b225cb300e6bb8f70e3b23c1d99940dd9fb0281aec2ab86ae0ad5f90" + "signature": "sha256:5d9f76b1d1dc426c1b07e4480adfabee73388a181a66dd7bbdf1e9c1e0896efe" }, "nbformat": 3, "nbformat_minor": 0, @@ -875,9 +875,225 @@ " 73 0.994891826923\n", "###" ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 74 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 75 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 76 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 77 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 78 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 79 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 80 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 81 0.994491185897\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 82 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 83 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 84 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 85 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 86 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 87 0.994891826923\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 88 0.994991987179\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 89 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 90 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 91 0.995092147436\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 92 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 93 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 94 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 95 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 96 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 97 0.994190705128\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 98 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 99 0.994391025641\n", + "CPU times: user 1h 34min 43s, sys: 45min 13s, total: 2h 19min 57s\n", + "Wall time: 2h 19min 56s\n" + ] } ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "GPU 2h 19min 56s 0.994-0.995" + ] + }, { "cell_type": "code", "collapsed": false, From 2cccba7cfabc5f5b7ca743e530e77ab9d7698cfb Mon Sep 17 00:00:00 2001 From: Madison May Date: Wed, 21 Jan 2015 01:35:07 -0500 Subject: [PATCH 6/6] Modified ipython notebooks -- general cleanup --- notebooks/0_multiply.ipynb | 27 +- notebooks/1_linear_regression.ipynb | 51 +- notebooks/2_logistic_regression.ipynb | 898 ++------------------------ notebooks/3_net.ipynb | 222 +++---- notebooks/4_modern_net.ipynb | 265 ++++---- notebooks/5_convolutional_net.ipynb | 17 +- notebooks/KNN.ipynb | 4 +- 7 files changed, 316 insertions(+), 1168 deletions(-) diff --git a/notebooks/0_multiply.ipynb b/notebooks/0_multiply.ipynb index fb96d08..d6e9955 100644 --- a/notebooks/0_multiply.ipynb +++ b/notebooks/0_multiply.ipynb @@ -12,14 +12,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "import Theano" + "Theano Basics\n", + "-------------" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "tensor is the section of theano that deals with data (like numpy)" + "`theano.tensor` is the module that deals most directly with the manipulation of data." ] }, { @@ -38,7 +39,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "create Theano symbolic variables" + "Instation of Theano symbolic variables is simple." ] }, { @@ -57,7 +58,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "our model" + "Relationships between variables are defined in a functional manner." ] }, { @@ -75,7 +76,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "compile to a python function (to CPU or GPU depending on configuration)" + "You can then compile to an input/output mapping to a python function (executed on the CPU or GPU depending on configuration). " ] }, { @@ -93,7 +94,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "and use the function" + "Finally, you can call the function as normal." ] }, { @@ -106,15 +107,17 @@ "metadata": {}, "outputs": [ { - "metadata": {}, - "output_type": "pyout", - "prompt_number": 5, - "text": [ - "array(2.0)" + "ename": "NameError", + "evalue": "name 'multiply' is not defined", + "output_type": "pyerr", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmultiply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'multiply' is not defined" ] } ], - "prompt_number": 5 + "prompt_number": 1 }, { "cell_type": "code", diff --git a/notebooks/1_linear_regression.ipynb b/notebooks/1_linear_regression.ipynb index 2f8e70b..e5f8664 100644 --- a/notebooks/1_linear_regression.ipynb +++ b/notebooks/1_linear_regression.ipynb @@ -12,7 +12,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "imports" + "Gradient Descent with Theano\n", + "-----------------------------" ] }, { @@ -26,13 +27,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 2 + "prompt_number": 1 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "generate data with noise" + "First let's generate data with noise." ] }, { @@ -45,13 +46,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 3 + "prompt_number": 2 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "symbolic vriable initalization" + "And initialize our symbolic variables." ] }, { @@ -64,13 +65,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 4 + "prompt_number": 3 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "our model" + "Then define our model (linear regression without a bias term)." ] }, { @@ -83,13 +84,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 5 + "prompt_number": 4 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "model parameter initalization. hyper variables, with real value" + "Next initialize model parameters. " ] }, { @@ -101,7 +102,7 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 6 + "prompt_number": 5 }, { "cell_type": "code", @@ -112,13 +113,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 7 + "prompt_number": 6 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "metric to be optimized by model" + "Set the metric (cost function) to be optimized by model." ] }, { @@ -130,13 +131,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 8 + "prompt_number": 7 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "learning signal for parameters. Computes the gradient symbolically" + "And use some build in theano magic (symbolic gradient calculations) to get a learning signal for parameters. " ] }, { @@ -148,13 +149,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 9 + "prompt_number": 8 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "how to update in each step. 0.01 is the learning rate" + "Next, define how the gradient is used to modify parameters. " ] }, { @@ -166,13 +167,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 10 + "prompt_number": 9 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "compile to a python function. `allow_input_downcast=True` is set to ignore typing issue with Theano on different platforms (GPU handle only 32bit)" + "Compile the theano updates to a python function. `allow_input_downcast=True` is set to ignore typing issue with Theano on different platforms (GPU computation requires float32s)." ] }, { @@ -184,13 +185,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 11 + "prompt_number": 10 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "iterate 100 times over the entire data (epoch.) In each epoch iterate over all the data samples" + "Iterate 100 times over the entire data (each iteration is known as an epoch)." ] }, { @@ -204,13 +205,13 @@ "language": "python", "metadata": {}, "outputs": [], - "prompt_number": 12 + "prompt_number": 11 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "we should get something close to the true weight of the data: 2" + "The result should be something close to the expected value, 2." ] }, { @@ -225,13 +226,13 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 13, + "prompt_number": 12, "text": [ - "array(1.9382810308787022)" + "array(2.0196163191213174)" ] } ], - "prompt_number": 13 + "prompt_number": 12 } ], "metadata": {} diff --git a/notebooks/2_logistic_regression.ipynb b/notebooks/2_logistic_regression.ipynb index 568c985..17fc87b 100644 --- a/notebooks/2_logistic_regression.ipynb +++ b/notebooks/2_logistic_regression.ipynb @@ -25,7 +25,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Our input is now the handwritten digits, each is a vector" + "Our input is now the handwritten digits, each is a vector. The MNIST dataset is available from http://yann.lecun.com/exdb/mnist/." ] }, { @@ -44,7 +44,19 @@ ], "language": "python", "metadata": {}, - "outputs": [], + "outputs": [ + { + "ename": "IOError", + "evalue": "[Errno 2] No such file or directory: '/home/mmay/Downloads/lisa/data/mnist/train-images-idx3-ubyte'", + "output_type": "pyerr", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mIOError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpanduser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"~/Downloads/lisa/data/\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mtrX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mteX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mteY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmnist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0monehot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/home/mmay/indico/Theano-Tutorials/load.pyc\u001b[0m in \u001b[0;36mmnist\u001b[0;34m(ntrain, ntest, onehot)\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mntrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m60000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mntest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0monehot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0mdata_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatasets_dir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'mnist/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m \u001b[0mfd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'train-images-idx3-ubyte'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 17\u001b[0m \u001b[0mloaded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfromfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0mtrX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloaded\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m60000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mIOError\u001b[0m: [Errno 2] No such file or directory: '/home/mmay/Downloads/lisa/data/mnist/train-images-idx3-ubyte'" + ] + } + ], "prompt_number": 2 }, { @@ -57,22 +69,13 @@ ], "language": "python", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stderr", - "text": [ - "Using gpu device 0: GeForce GT 650M\n" - ] - } - ], - "prompt_number": 3 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Convert to correct dtype" + "Convert to the correct dtype." ] }, { @@ -84,14 +87,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 4 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "initalize model parameters (small random guassian distribution)" + "Initalize model parameters (small random guassian distribution)" ] }, { @@ -103,14 +105,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 5 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "In our model we will replace scalars with vectors" + "In our new model we will replace scalars with matrices." ] }, { @@ -122,14 +123,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 6 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "and weights with matrices" + "The same goes for our weights." ] }, { @@ -140,14 +140,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 7 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "and use softmax convert our weighted input to probability" + "We use the softmax activation function to convert our weighted input to probabilities." ] }, { @@ -159,14 +158,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 8 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "probability output" + "And define a variable for the probability output." ] }, { @@ -177,14 +175,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 9 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "maxima prediction" + "Adding in a convenience function to return the class with highest probability." ] }, { @@ -195,14 +192,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 10 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "different cost function, maximize the value that is correct and minize the others" + "We use a different cost function than last time -- categorical crossentropy is more appropriate for classification problems." ] }, { @@ -217,14 +213,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 11 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "now we also have a predict function that will be used on the test data" + "Now we also have a predict function that will be used on the test data." ] }, { @@ -235,14 +230,13 @@ ], "language": "python", "metadata": {}, - "outputs": [], - "prompt_number": 12 + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "train on mini-batches of 128 samples each" + "Let's train on mini-batches of 128 samples each." ] }, { @@ -253,7 +247,7 @@ "for i in range(100):\n", " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", " cost = train(trX[start:end], trY[start:end])\n", - " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " \n", " errs = []\n", " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", @@ -261,832 +255,14 @@ ], "language": "python", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0 0.918269230769\n", - "1" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.918569711538\n", - "2" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.918970352564\n", - "3" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.919070512821\n", - "4" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.919170673077\n", - "5" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.919671474359\n", - "6" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.919871794872\n", - "7" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920372596154\n", - "8" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920572916667\n", - "9" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920372596154\n", - "10" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920572916667\n", - "11" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920673076923\n", - "12" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920572916667\n", - "13" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.920673076923\n", - "14" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921274038462\n", - "15" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921474358974\n", - "16" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921674679487\n", - "17" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921875\n", - "18" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921674679487\n", - "19" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921474358974\n", - "20" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921474358974\n", - "21" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921774839744\n", - "22" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921975160256\n", - "23" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921875\n", - "24" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921574519231\n", - "25" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.921875\n", - "26" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922075320513\n", - "27" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922275641026\n", - "28" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922275641026\n", - "29" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922375801282\n", - "30" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922576121795\n", - "31" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922375801282\n", - "32" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922576121795\n", - "33" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922576121795\n", - "34" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922676282051\n", - "35" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922976762821\n", - "36" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922776442308\n", - "37" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922776442308\n", - "38" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.922876602564\n", - "39" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923076923077\n", - "40" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923076923077\n", - "41" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.92327724359\n", - "42" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923377403846\n", - "43" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923677884615\n", - "44" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923778044872\n", - "45" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923978365385\n", - "46" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923978365385\n", - "47" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923978365385\n", - "48" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923978365385\n", - "49" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923878205128\n", - "50" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923978365385\n", - "51" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924078525641\n", - "52" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.923978365385\n", - "53" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924078525641\n", - "54" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924078525641\n", - "55" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924178685897\n", - "56" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924479166667\n", - "57" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "58" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "59" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "60" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "61" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "62" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "63" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "64" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "65" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "66" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "67" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "68" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "69" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "70" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "71" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "72" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "73" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "74" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "75" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "76" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "77" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924879807692\n", - "78" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924879807692\n", - "79" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924879807692\n", - "80" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "81" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "82" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "83" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924879807692\n", - "84" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924979967949\n", - "85" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "86" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "87" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "88" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "89" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "90" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924879807692\n", - "91" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "92" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924779647436\n", - "93" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "94" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "95" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "96" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "97" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "98" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924579326923\n", - "99" - ] - }, - { - "output_type": "stream", - "stream": "stdout", - "text": [ - " 0.924679487179\n", - "CPU times: user 1min 17s, sys: 4.96 s, total: 1min 22s\n", - "Wall time: 1min 23s\n" - ] - } - ], - "prompt_number": 14 - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "prediction is not as good as kNN" - ] + "outputs": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "28.5 sec for CPU. GPU was 1min23s or x3 slower :-(" + "With this naive model, predictions are poorer than those produced by the kNN model." ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [], - "language": "python", - "metadata": {}, - "outputs": [] } ], "metadata": {} diff --git a/notebooks/3_net.ipynb b/notebooks/3_net.ipynb index 56f4443..37b6cb4 100644 --- a/notebooks/3_net.ipynb +++ b/notebooks/3_net.ipynb @@ -20,7 +20,7 @@ "collapsed": false, "input": [ "import os\n", - "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "# os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" ], "language": "python", @@ -63,22 +63,14 @@ ], "language": "python", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stderr", - "text": [ - "Using gpu device 0: GeForce GT 650M\n" - ] - } - ], + "outputs": [], "prompt_number": 3 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Generalize to compute stochastic gradient descent (SGD) on all model parametes. Note that SGD has a fixed learning rate which slow things down as you near the goal" + "Generalize to compute stochastic gradient descent (SGD) on all model parametes. Note that SGD has a fixed learning rate which slow things down as you near the goal." ] }, { @@ -101,7 +93,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "logistic regression (LR) model is replaced with a neural network (NN) model with one hidden layer" + "The logistic regression (LR) model is replaced with a neural network (NN) model with one hidden layer." ] }, { @@ -134,7 +126,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "we have 625 hidden units" + "We have 625 hidden units." ] }, { @@ -180,7 +172,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - "0 0.708233173077\n", + "0 0.703625801282\n", "1" ] }, @@ -188,7 +180,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.831129807692\n", + " 0.829126602564\n", "2" ] }, @@ -196,7 +188,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.868189102564\n", + " 0.867988782051\n", "3" ] }, @@ -204,7 +196,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.883413461538\n", + " 0.882512019231\n", "4" ] }, @@ -212,7 +204,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.889623397436\n", + " 0.889723557692\n", "5" ] }, @@ -220,7 +212,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.894931891026\n", + " 0.895633012821\n", "6" ] }, @@ -228,7 +220,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.8984375\n", + " 0.899138621795\n", "7" ] }, @@ -236,7 +228,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.901642628205\n", + " 0.902043269231\n", "8" ] }, @@ -244,7 +236,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.90484775641\n", + " 0.904747596154\n", "9" ] }, @@ -252,7 +244,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.907251602564\n", + " 0.907151442308\n", "10" ] }, @@ -260,7 +252,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.909154647436\n", + " 0.908954326923\n", "11" ] }, @@ -268,7 +260,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.911157852564\n", + " 0.910957532051\n", "12" ] }, @@ -276,7 +268,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.912960737179\n", + " 0.912760416667\n", "13" ] }, @@ -284,7 +276,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.913361378205\n", + " 0.913962339744\n", "14" ] }, @@ -292,7 +284,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.914763621795\n", + " 0.914463141026\n", "15" ] }, @@ -300,7 +292,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.915765224359\n", + " 0.915564903846\n", "16" ] }, @@ -316,7 +308,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.916967147436\n", + " 0.917467948718\n", "18" ] }, @@ -324,7 +316,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.917367788462\n", + " 0.918269230769\n", "19" ] }, @@ -332,7 +324,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.918269230769\n", + " 0.918569711538\n", "20" ] }, @@ -348,7 +340,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.919571314103\n", + " 0.919170673077\n", "22" ] }, @@ -356,7 +348,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.919571314103\n", + " 0.919471153846\n", "23" ] }, @@ -364,7 +356,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.919571314103\n", + " 0.919871794872\n", "24" ] }, @@ -372,7 +364,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.920272435897\n", + " 0.92047275641\n", "25" ] }, @@ -380,7 +372,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.920873397436\n", + " 0.921574519231\n", "26" ] }, @@ -388,7 +380,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.921073717949\n", + " 0.922475961538\n", "27" ] }, @@ -396,7 +388,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.921474358974\n", + " 0.922375801282\n", "28" ] }, @@ -404,7 +396,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.921774839744\n", + " 0.923076923077\n", "29" ] }, @@ -412,7 +404,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.922475961538\n", + " 0.923477564103\n", "30" ] }, @@ -420,7 +412,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.922475961538\n", + " 0.924078525641\n", "31" ] }, @@ -428,7 +420,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.922776442308\n", + " 0.924479166667\n", "32" ] }, @@ -436,7 +428,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.923677884615\n", + " 0.924579326923\n", "33" ] }, @@ -444,7 +436,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.924579326923\n", + " 0.925480769231\n", "34" ] }, @@ -452,7 +444,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.924979967949\n", + " 0.925981570513\n", "35" ] }, @@ -460,7 +452,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.925280448718\n", + " 0.926482371795\n", "36" ] }, @@ -468,7 +460,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.926081730769\n", + " 0.926983173077\n", "37" ] }, @@ -476,7 +468,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.926181891026\n", + " 0.927383814103\n", "38" ] }, @@ -484,7 +476,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.926682692308\n", + " 0.927884615385\n", "39" ] }, @@ -492,7 +484,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.927483974359\n", + " 0.928685897436\n", "40" ] }, @@ -500,7 +492,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.927984775641\n", + " 0.928886217949\n", "41" ] }, @@ -508,7 +500,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.928485576923\n", + " 0.929387019231\n", "42" ] }, @@ -516,7 +508,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.928886217949\n", + " 0.929887820513\n", "43" ] }, @@ -524,7 +516,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.929487179487\n", + " 0.930488782051\n", "44" ] }, @@ -532,7 +524,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.929887820513\n", + " 0.93108974359\n", "45" ] }, @@ -540,7 +532,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.930588942308\n", + " 0.932091346154\n", "46" ] }, @@ -548,7 +540,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.931189903846\n", + " 0.932892628205\n", "47" ] }, @@ -556,7 +548,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.93108974359\n", + " 0.933493589744\n", "48" ] }, @@ -564,7 +556,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.931690705128\n", + " 0.934395032051\n", "49" ] }, @@ -572,7 +564,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.932391826923\n", + " 0.93499599359\n", "50" ] }, @@ -580,7 +572,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.933994391026\n", + " 0.936197916667\n", "51" ] }, @@ -588,7 +580,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.934495192308\n", + " 0.936598557692\n", "52" ] }, @@ -596,7 +588,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.93499599359\n", + " 0.936598557692\n", "53" ] }, @@ -604,7 +596,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.935697115385\n", + " 0.937199519231\n", "54" ] }, @@ -612,7 +604,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.936698717949\n", + " 0.9375\n", "55" ] }, @@ -620,7 +612,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.937600160256\n", + " 0.938401442308\n", "56" ] }, @@ -628,7 +620,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.938000801282\n", + " 0.939002403846\n", "57" ] }, @@ -636,7 +628,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.938401442308\n", + " 0.939403044872\n", "58" ] }, @@ -644,7 +636,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.939002403846\n", + " 0.939503205128\n", "59" ] }, @@ -652,7 +644,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.939503205128\n", + " 0.940204326923\n", "60" ] }, @@ -660,7 +652,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.939903846154\n", + " 0.940604967949\n", "61" ] }, @@ -668,7 +660,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.940604967949\n", + " 0.941205929487\n", "62" ] }, @@ -676,7 +668,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.941205929487\n", + " 0.941706730769\n", "63" ] }, @@ -684,7 +676,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.941706730769\n", + " 0.942107371795\n", "64" ] }, @@ -692,7 +684,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.941907051282\n", + " 0.943008814103\n", "65" ] }, @@ -700,7 +692,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.942608173077\n", + " 0.943309294872\n", "66" ] }, @@ -708,7 +700,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.943309294872\n", + " 0.944210737179\n", "67" ] }, @@ -716,7 +708,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.943709935897\n", + " 0.944711538462\n", "68" ] }, @@ -724,7 +716,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.944110576923\n", + " 0.945412660256\n", "69" ] }, @@ -732,7 +724,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.945212339744\n", + " 0.945713141026\n", "70" ] }, @@ -740,7 +732,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.945913461538\n", + " 0.946414262821\n", "71" ] }, @@ -748,7 +740,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.946314102564\n", + " 0.947315705128\n", "72" ] }, @@ -756,7 +748,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.946614583333\n", + " 0.947716346154\n", "73" ] }, @@ -764,7 +756,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.94671474359\n", + " 0.948116987179\n", "74" ] }, @@ -772,7 +764,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.946915064103\n", + " 0.948617788462\n", "75" ] }, @@ -780,7 +772,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.947215544872\n", + " 0.948818108974\n", "76" ] }, @@ -788,7 +780,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.947315705128\n", + " 0.949118589744\n", "77" ] }, @@ -796,7 +788,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.94781650641\n", + " 0.949719551282\n", "78" ] }, @@ -804,7 +796,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.948617788462\n", + " 0.950220352564\n", "79" ] }, @@ -812,7 +804,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.949018429487\n", + " 0.950520833333\n", "80" ] }, @@ -820,7 +812,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.949719551282\n", + " 0.951021634615\n", "81" ] }, @@ -828,7 +820,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.950721153846\n", + " 0.951622596154\n", "82" ] }, @@ -836,7 +828,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.951021634615\n", + " 0.952023237179\n", "83" ] }, @@ -844,7 +836,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.951522435897\n", + " 0.952223557692\n", "84" ] }, @@ -852,7 +844,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.951923076923\n", + " 0.952624198718\n", "85" ] }, @@ -860,7 +852,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.952223557692\n", + " 0.953425480769\n", "86" ] }, @@ -868,7 +860,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.952924679487\n", + " 0.953525641026\n", "87" ] }, @@ -876,7 +868,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.953525641026\n", + " 0.954126602564\n", "88" ] }, @@ -884,7 +876,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.953826121795\n", + " 0.954427083333\n", "89" ] }, @@ -892,7 +884,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.954427083333\n", + " 0.954627403846\n", "90" ] }, @@ -900,7 +892,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.955028044872\n", + " 0.954927884615\n", "91" ] }, @@ -908,7 +900,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.955228365385\n", + " 0.955128205128\n", "92" ] }, @@ -924,7 +916,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.955729166667\n", + " 0.955428685897\n", "94" ] }, @@ -932,7 +924,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.956029647436\n", + " 0.955729166667\n", "95" ] }, @@ -940,7 +932,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.956330128205\n", + " 0.955929487179\n", "96" ] }, @@ -964,7 +956,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.956931089744\n", + " 0.957131410256\n", "99" ] }, @@ -972,28 +964,22 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.957331730769\n", - "CPU times: user 2min 9s, sys: 5.21 s, total: 2min 14s\n", - "Wall time: 2min 14s\n" + " 0.957231570513\n", + "CPU times: user 18min 9s, sys: 8min 9s, total: 26min 18s\n", + "Wall time: 6min 36s\n" ] } ], "prompt_number": 8 }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "CPU 2min 56s, GPU 2min 14s. The result are 0.957 and not 0.98 as discussed in the video lecture" - ] - }, { "cell_type": "code", "collapsed": false, "input": [], "language": "python", "metadata": {}, - "outputs": [] + "outputs": [], + "prompt_number": 8 } ], "metadata": {} diff --git a/notebooks/4_modern_net.ipynb b/notebooks/4_modern_net.ipynb index acc84a2..395bbc1 100644 --- a/notebooks/4_modern_net.ipynb +++ b/notebooks/4_modern_net.ipynb @@ -12,7 +12,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "two main changes: replace sigmoid with rectifier (LeRU). The problem of overfitting of old NN is solved by adding noise. This allows to add hidden layers (in the fast this caused overfitting.)" + "A more modern neural net\n", + "------------------------" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Three main changes: replace sigmoid with rectifier (LeRU), add dropout regularization, and replace SGD with a more modern learning rule (RMSProp). Dropout regularization allows us to add hidden layers (in the past this caused overfitting.)" ] }, { @@ -66,14 +74,29 @@ ], "language": "python", "metadata": {}, - "outputs": [], + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "ERROR (theano.sandbox.cuda): nvcc compiler not found on $PATH. Check your nvcc installation and try again.\n" + ] + }, + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "ERROR:theano.sandbox.cuda:nvcc compiler not found on $PATH. Check your nvcc installation and try again.\n" + ] + } + ], "prompt_number": 3 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "replacement for sigmoid function" + "`rectify` is our replacement for the sigmoid function." ] }, { @@ -92,7 +115,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "modify softmax to remove max value to avoid explosition of exp (numeric stability)" + "Modify `softmax` to remove max value to avoid explosition of exp (numeric stability)" ] }, { @@ -112,7 +135,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "we use a running RMS average of the gradient to scale the gradient size" + "We use a running RMS average of the gradient to scale the gradient size" ] }, { @@ -140,7 +163,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "inject noise with dropout. randomly set to zero some (p) of the nodes. We have to compensate by scaling the nodes that were not dropped out." + "Dropout regularization will randomly set to zero some of the nodes (with probability `p`). We have to compensate by scaling the nodes that were not dropped out." ] }, { @@ -163,7 +186,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "new model has two hidden layers of 625 nodes" + "Our new model has two hidden layers of 625 nodes." ] }, { @@ -210,16 +233,7 @@ ], "language": "python", "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "stream": "stderr", - "text": [ - "/Users/udi/anaconda/lib/python2.7/site-packages/theano/sandbox/rng_mrg.py:1195: UserWarning: MRG_RandomStreams Can't determine #streams from size (Shape.0), guessing 60*256\n", - " nstreams = self.n_streams(size)\n" - ] - } - ], + "outputs": [], "prompt_number": 9 }, { @@ -243,7 +257,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - "0 0.940104166667\n", + "0 0.943810096154\n", "1" ] }, @@ -251,7 +265,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.965344551282\n", + " 0.962239583333\n", "2" ] }, @@ -259,7 +273,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.972856570513\n", + " 0.974258814103\n", "3" ] }, @@ -267,7 +281,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.973858173077\n", + " 0.974559294872\n", "4" ] }, @@ -275,7 +289,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.975260416667\n", + " 0.975761217949\n", "5" ] }, @@ -283,7 +297,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.978665865385\n", + " 0.97796474359\n", "6" ] }, @@ -291,7 +305,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.978866185897\n", + " 0.978966346154\n", "7" ] }, @@ -299,7 +313,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.979166666667\n", + " 0.978866185897\n", "8" ] }, @@ -307,7 +321,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.982171474359\n", + " 0.982271634615\n", "9" ] }, @@ -315,7 +329,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.980268429487\n", + " 0.983173076923\n", "10" ] }, @@ -331,7 +345,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98297275641\n", + " 0.984575320513\n", "12" ] }, @@ -339,7 +353,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.983072916667\n", + " 0.982872596154\n", "13" ] }, @@ -355,7 +369,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984074519231\n", + " 0.983373397436\n", "15" ] }, @@ -363,7 +377,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.983673878205\n", + " 0.983874198718\n", "16" ] }, @@ -371,7 +385,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.983373397436\n", + " 0.983673878205\n", "17" ] }, @@ -379,7 +393,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984074519231\n", + " 0.983774038462\n", "18" ] }, @@ -387,7 +401,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984775641026\n", + " 0.983673878205\n", "19" ] }, @@ -395,7 +409,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.983874198718\n", + " 0.983273237179\n", "20" ] }, @@ -403,7 +417,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985977564103\n", + " 0.983974358974\n", "21" ] }, @@ -411,7 +425,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984174679487\n", + " 0.983673878205\n", "22" ] }, @@ -419,7 +433,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984675480769\n", + " 0.984174679487\n", "23" ] }, @@ -427,7 +441,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984274839744\n", + " 0.984875801282\n", "24" ] }, @@ -435,7 +449,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984775641026\n", + " 0.984274839744\n", "25" ] }, @@ -443,7 +457,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986979166667\n", + " 0.984875801282\n", "26" ] }, @@ -451,7 +465,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985176282051\n", + " 0.985376602564\n", "27" ] }, @@ -459,7 +473,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.984575320513\n", + " 0.985176282051\n", "28" ] }, @@ -467,7 +481,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985076121795\n", + " 0.984975961538\n", "29" ] }, @@ -475,7 +489,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985576923077\n", + " 0.985977564103\n", "30" ] }, @@ -483,7 +497,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98577724359\n", + " 0.985677083333\n", "31" ] }, @@ -491,7 +505,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98577724359\n", + " 0.984775641026\n", "32" ] }, @@ -499,7 +513,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98687900641\n", + " 0.985376602564\n", "33" ] }, @@ -507,7 +521,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985376602564\n", + " 0.986478365385\n", "34" ] }, @@ -515,7 +529,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986077724359\n", + " 0.985076121795\n", "35" ] }, @@ -523,7 +537,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985877403846\n", + " 0.986578525641\n", "36" ] }, @@ -531,7 +545,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985677083333\n", + " 0.984975961538\n", "37" ] }, @@ -539,7 +553,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98687900641\n", + " 0.984975961538\n", "38" ] }, @@ -547,7 +561,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986678685897\n", + " 0.98577724359\n", "39" ] }, @@ -555,7 +569,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98577724359\n", + " 0.986177884615\n", "40" ] }, @@ -563,7 +577,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985376602564\n", + " 0.98687900641\n", "41" ] }, @@ -571,7 +585,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986578525641\n", + " 0.985677083333\n", "42" ] }, @@ -579,7 +593,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986478365385\n", + " 0.986378205128\n", "43" ] }, @@ -587,7 +601,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985677083333\n", + " 0.986478365385\n", "44" ] }, @@ -595,7 +609,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986278044872\n", + " 0.986478365385\n", "45" ] }, @@ -603,7 +617,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986979166667\n", + " 0.986378205128\n", "46" ] }, @@ -611,7 +625,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987379807692\n", + " 0.987179487179\n", "47" ] }, @@ -619,7 +633,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.988581730769\n", + " 0.986979166667\n", "48" ] }, @@ -627,7 +641,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986378205128\n", + " 0.987479967949\n", "49" ] }, @@ -635,7 +649,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987479967949\n", + " 0.986278044872\n", "50" ] }, @@ -643,7 +657,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985977564103\n", + " 0.986278044872\n", "51" ] }, @@ -651,7 +665,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.98687900641\n", + " 0.986979166667\n", "52" ] }, @@ -659,7 +673,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987580128205\n", + " 0.986478365385\n", "53" ] }, @@ -675,7 +689,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985877403846\n", + " 0.98687900641\n", "55" ] }, @@ -683,7 +697,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986077724359\n", + " 0.987980769231\n", "56" ] }, @@ -691,7 +705,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.987179487179\n", "57" ] }, @@ -699,7 +713,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986077724359\n", + " 0.986778846154\n", "58" ] }, @@ -707,7 +721,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.986778846154\n", "59" ] }, @@ -715,7 +729,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985877403846\n", + " 0.986578525641\n", "60" ] }, @@ -723,7 +737,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.986578525641\n", "61" ] }, @@ -739,7 +753,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.986578525641\n", "63" ] }, @@ -755,7 +769,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987780448718\n", + " 0.986077724359\n", "65" ] }, @@ -763,7 +777,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986678685897\n", + " 0.987880608974\n", "66" ] }, @@ -771,7 +785,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987079326923\n", + " 0.986478365385\n", "67" ] }, @@ -779,7 +793,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987079326923\n", + " 0.985677083333\n", "68" ] }, @@ -787,7 +801,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986678685897\n", + " 0.986778846154\n", "69" ] }, @@ -795,7 +809,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987379807692\n", + " 0.986478365385\n", "70" ] }, @@ -803,7 +817,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986177884615\n", + " 0.986578525641\n", "71" ] }, @@ -811,7 +825,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986378205128\n", + " 0.986478365385\n", "72" ] }, @@ -827,7 +841,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987379807692\n", + " 0.98687900641\n", "74" ] }, @@ -835,7 +849,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987780448718\n", + " 0.987179487179\n", "75" ] }, @@ -843,7 +857,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986678685897\n", + " 0.986778846154\n", "76" ] }, @@ -851,7 +865,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987179487179\n", + " 0.987379807692\n", "77" ] }, @@ -859,7 +873,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987980769231\n", + " 0.986478365385\n", "78" ] }, @@ -867,7 +881,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.985977564103\n", + " 0.987780448718\n", "79" ] }, @@ -875,7 +889,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986578525641\n", + " 0.986478365385\n", "80" ] }, @@ -883,7 +897,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.987379807692\n", "81" ] }, @@ -891,7 +905,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.988782051282\n", + " 0.986278044872\n", "82" ] }, @@ -899,7 +913,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987379807692\n", + " 0.987279647436\n", "83" ] }, @@ -907,7 +921,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.987580128205\n", "84" ] }, @@ -915,7 +929,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987479967949\n", + " 0.987580128205\n", "85" ] }, @@ -923,7 +937,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987980769231\n", + " 0.987379807692\n", "86" ] }, @@ -931,7 +945,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987479967949\n", + " 0.987980769231\n", "87" ] }, @@ -939,7 +953,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987580128205\n", + " 0.987179487179\n", "88" ] }, @@ -947,7 +961,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986578525641\n", + " 0.986678685897\n", "89" ] }, @@ -955,7 +969,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987680288462\n", + " 0.986578525641\n", "90" ] }, @@ -963,7 +977,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987880608974\n", + " 0.987580128205\n", "91" ] }, @@ -971,7 +985,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986979166667\n", + " 0.987880608974\n", "92" ] }, @@ -979,7 +993,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986979166667\n", + " 0.987079326923\n", "93" ] }, @@ -987,7 +1001,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.986979166667\n", "94" ] }, @@ -995,7 +1009,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.986778846154\n", + " 0.986979166667\n", "95" ] }, @@ -1003,7 +1017,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987780448718\n", + " 0.987880608974\n", "96" ] }, @@ -1011,7 +1025,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987580128205\n", + " 0.986979166667\n", "97" ] }, @@ -1019,7 +1033,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987980769231\n", + " 0.987479967949\n", "98" ] }, @@ -1027,7 +1041,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987279647436\n", + " 0.987479967949\n", "99" ] }, @@ -1035,38 +1049,13 @@ "output_type": "stream", "stream": "stdout", "text": [ - " 0.987079326923\n", - "CPU times: user 4min 53s, sys: 30.4 s, total: 5min 23s\n", - "Wall time: 5min 23s\n" + " 0.986678685897\n", + "CPU times: user 1h 8min 35s, sys: 44min 35s, total: 1h 53min 10s\n", + "Wall time: 30min 27s\n" ] } ], "prompt_number": 10 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "CPU 17min 4s, GPU 5min 23s" - ], - "language": "python", - "metadata": {}, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "98.8% not 99% in video" - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [], - "language": "python", - "metadata": {}, - "outputs": [] } ], "metadata": {} diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb index 9503c00..e75d672 100644 --- a/notebooks/5_convolutional_net.ipynb +++ b/notebooks/5_convolutional_net.ipynb @@ -114,7 +114,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "block to compute conv->activate->pool->noise" + "Convolution --> activation --> pooling --> dropout" ] }, { @@ -152,7 +152,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "convert data from vector to color images, but we have just one color channel " + "Reshape operation to conform to resize to single channel of 28 * 28 pixels." ] }, { @@ -174,7 +174,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "covolution weights (n_kernels, n_channels, kernel_w, kernel_h). We will use 3x3 kernel. Smallest possible. Instead of bigger kernels use more layers. It looks as if each convolution layer is twice as big as the previous one, but keep in mind that we perform maxpool which will reduce the dimension by factor of two" + "Convolution weights (n_kernels, n_channels, kernel_w, kernel_h). Moving up the heirarchy we move towards deeper, narrower representations thanks to increasing numbers of filters and maxpooling operations." ] }, { @@ -190,13 +190,6 @@ "outputs": [], "prompt_number": 6 }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "last layer is flat but its input size must match the size of the previous convolution layer" - ] - }, { "cell_type": "code", "collapsed": false, @@ -213,7 +206,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "noise during training" + "Apply dropout during training." ] }, { @@ -231,7 +224,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "noiseless model for testing" + "Use a noiseless model for prediction." ] }, { diff --git a/notebooks/KNN.ipynb b/notebooks/KNN.ipynb index 6fbe425..1d0c291 100644 --- a/notebooks/KNN.ipynb +++ b/notebooks/KNN.ipynb @@ -12,7 +12,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Simplest way to perform classification of the handwritten digits is to use nearest neighbor" + "One of the simplest ways to perform classification of the handwritten digits is to use a nearest neighbor classifier." ] }, { @@ -92,7 +92,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "which works very well but not good enough (0.995 or more)" + "This method works quite well, but prediction on new examples is computationally expensive." ] } ],