diff --git a/.gitignore b/.gitignore
index db4561e..ead0dec 100644
--- a/.gitignore
+++ b/.gitignore
@@ -52,3 +52,6 @@ docs/_build/
 
 # PyBuilder
 target/
+
+#ipython notebook
+.ipynb_checkpoints/
diff --git a/README.md b/README.md
index 0db9e9f..f879126 100644
--- a/README.md
+++ b/README.md
@@ -2,3 +2,6 @@ Theano-Tutorials
 ================
 
 Bare bones introduction to machine learning from linear regression to convolutional neural networks using Theano.
+
+See [ipython notebooks](http://nbviewer.ipython.org/github/udibr/Theano-Tutorials/blob/master/notebooks/index.ipynb)
+
diff --git a/load.py b/load.py
index 25d931d..cae4cd5 100644
--- a/load.py
+++ b/load.py
@@ -13,19 +13,19 @@ def one_hot(x,n):
 
 def mnist(ntrain=60000,ntest=10000,onehot=True):
 	data_dir = os.path.join(datasets_dir,'mnist/')
-	fd = open(os.path.join(data_dir,'train-images.idx3-ubyte'))
+	fd = open(os.path.join(data_dir,'train-images-idx3-ubyte'))
 	loaded = np.fromfile(file=fd,dtype=np.uint8)
 	trX = loaded[16:].reshape((60000,28*28)).astype(float)
 
-	fd = open(os.path.join(data_dir,'train-labels.idx1-ubyte'))
+	fd = open(os.path.join(data_dir,'train-labels-idx1-ubyte'))
 	loaded = np.fromfile(file=fd,dtype=np.uint8)
 	trY = loaded[8:].reshape((60000))
 
-	fd = open(os.path.join(data_dir,'t10k-images.idx3-ubyte'))
+	fd = open(os.path.join(data_dir,'t10k-images-idx3-ubyte'))
 	loaded = np.fromfile(file=fd,dtype=np.uint8)
 	teX = loaded[16:].reshape((10000,28*28)).astype(float)
 
-	fd = open(os.path.join(data_dir,'t10k-labels.idx1-ubyte'))
+	fd = open(os.path.join(data_dir,'t10k-labels-idx1-ubyte'))
 	loaded = np.fromfile(file=fd,dtype=np.uint8)
 	teY = loaded[8:].reshape((10000))
 
@@ -45,4 +45,4 @@ def mnist(ntrain=60000,ntest=10000,onehot=True):
 		trY = np.asarray(trY)
 		teY = np.asarray(teY)
 
-	return trX,teX,trY,teY
\ No newline at end of file
+	return trX,teX,trY,teY
diff --git a/notebooks/0_multiply.ipynb b/notebooks/0_multiply.ipynb
new file mode 100644
index 0000000..d6e9955
--- /dev/null
+++ b/notebooks/0_multiply.ipynb
@@ -0,0 +1,146 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:f13eb3b414e3e34aa7531d120d38fcd6602672de65630585a7952395a63c3f7e"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Theano Basics\n",
+      "-------------"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "`theano.tensor` is the module that deals most directly with the manipulation of data."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import theano\n",
+      "from theano import tensor as T"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Instation of Theano symbolic variables is simple."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "a = T.scalar()\n",
+      "b = T.scalar()"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Relationships between variables are defined in a functional manner."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "y = a * b"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 3
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "You can then compile to an input/output mapping to a python function (executed on the CPU or GPU depending on configuration).  "
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "multiply = theano.function(inputs=[a, b], outputs=y)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Finally, you can call the function as normal."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "multiply(1, 2)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "ename": "NameError",
+       "evalue": "name 'multiply' is not defined",
+       "output_type": "pyerr",
+       "traceback": [
+        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+        "\u001b[0;32m<ipython-input-1-2c7a8e9cf618>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmultiply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+        "\u001b[0;31mNameError\u001b[0m: name 'multiply' is not defined"
+       ]
+      }
+     ],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "multiply(3, 3)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 6,
+       "text": [
+        "array(9.0)"
+       ]
+      }
+     ],
+     "prompt_number": 6
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/1_linear_regression.ipynb b/notebooks/1_linear_regression.ipynb
new file mode 100644
index 0000000..e5f8664
--- /dev/null
+++ b/notebooks/1_linear_regression.ipynb
@@ -0,0 +1,241 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:65a56707d32e8e45ec327145d45b5a9a273bc13b6c18fc9b7e7bf8c84fdf53ba"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Gradient Descent with Theano\n",
+      "-----------------------------"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import theano\n",
+      "from theano import tensor as T\n",
+      "import numpy as np"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "First let's generate data with noise."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "trX = np.linspace(-1, 1, 101)\n",
+      "trY = 2 * trX + np.random.randn(*trX.shape) * 0.33"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "And initialize our symbolic variables."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "X = T.scalar()\n",
+      "Y = T.scalar()"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 3
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Then define our model (linear regression without a bias term)."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def model(X, w):\n",
+      "    return X * w"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Next initialize model parameters. "
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "w = theano.shared(np.asarray(0., dtype=theano.config.floatX))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 5
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "y = model(X, w)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Set the metric (cost function) to be optimized by model."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "cost = T.mean(T.sqr(y - Y))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 7
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "And use some build in theano magic (symbolic gradient calculations) to get a learning signal for parameters. "
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "gradient = T.grad(cost=cost, wrt=w)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 8
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Next, define how the gradient is used to modify parameters.  "
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "updates = [[w, w - gradient * 0.01]]"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 9
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Compile the theano updates to a python function. `allow_input_downcast=True` is set to ignore typing issue with Theano on different platforms (GPU computation requires float32s)."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 10
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Iterate 100 times over the entire data (each iteration is known as an epoch)."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "for i in range(100):\n",
+      "    for x, y in zip(trX, trY):\n",
+      "        train(x, y)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 11
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "The result should be something close to the expected value, 2."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "w.get_value()"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 12,
+       "text": [
+        "array(2.0196163191213174)"
+       ]
+      }
+     ],
+     "prompt_number": 12
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/2_logistic_regression.ipynb b/notebooks/2_logistic_regression.ipynb
new file mode 100644
index 0000000..17fc87b
--- /dev/null
+++ b/notebooks/2_logistic_regression.ipynb
@@ -0,0 +1,271 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:8e7467687cabf437572893b54a343b6de5be3c2f06c6247823de13c0201bf14a"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os\n",
+      "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n",
+      "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Our input is now the handwritten digits, each is a vector.  The MNIST dataset is available from http://yann.lecun.com/exdb/mnist/."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os,sys,inspect\n",
+      "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+      "parentdir = os.path.dirname(currentdir)\n",
+      "sys.path.insert(0,parentdir) \n",
+      "import load\n",
+      "\n",
+      "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n",
+      "\n",
+      "trX, teX, trY, teY = load.mnist(onehot=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "ename": "IOError",
+       "evalue": "[Errno 2] No such file or directory: '/home/mmay/Downloads/lisa/data/mnist/train-images-idx3-ubyte'",
+       "output_type": "pyerr",
+       "traceback": [
+        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mIOError\u001b[0m                                   Traceback (most recent call last)",
+        "\u001b[0;32m<ipython-input-2-d1f8ab0d1f51>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatasets_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpanduser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"~/Downloads/lisa/data/\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mtrX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mteX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrY\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mteY\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mload\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmnist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0monehot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+        "\u001b[0;32m/home/mmay/indico/Theano-Tutorials/load.pyc\u001b[0m in \u001b[0;36mmnist\u001b[0;34m(ntrain, ntest, onehot)\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmnist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mntrain\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m60000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mntest\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0monehot\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mdata_dir\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdatasets_dir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'mnist/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m         \u001b[0mfd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_dir\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'train-images-idx3-ubyte'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m         \u001b[0mloaded\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfromfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfd\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mtrX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloaded\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m16\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m60000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m28\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+        "\u001b[0;31mIOError\u001b[0m: [Errno 2] No such file or directory: '/home/mmay/Downloads/lisa/data/mnist/train-images-idx3-ubyte'"
+       ]
+      }
+     ],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import theano\n",
+      "from theano import tensor as T\n",
+      "import numpy as np"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Convert to the correct dtype."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def floatX(X):\n",
+      "    return np.asarray(X, dtype=theano.config.floatX)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Initalize model parameters (small random guassian distribution)"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def init_weights(shape):\n",
+      "    return theano.shared(floatX(np.random.randn(*shape) * 0.01))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "In our new model we will replace scalars with matrices."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "X = T.fmatrix()\n",
+      "Y = T.fmatrix()"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "The same goes for our weights."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "w = init_weights((784, 10))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "We use the softmax activation function to convert our weighted input to probabilities."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def model(X, w):\n",
+      "    return T.nnet.softmax(T.dot(X, w))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "And define a variable for the probability output."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "py_x = model(X, w)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Adding in a convenience function to return the class with highest probability."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "y_pred = T.argmax(py_x, axis=1)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "We use a different cost function than last time -- categorical crossentropy is more appropriate for classification problems."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n",
+      "gradient = T.grad(cost=cost, wrt=w)\n",
+      "update = [[w, w - gradient * 0.05]]\n",
+      "\n",
+      "train = theano.function(inputs=[X, Y], outputs=cost, updates=update, allow_input_downcast=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Now we also have a predict function that will be used on the test data."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "predict = theano.function(inputs=[X], outputs=y_pred, allow_input_downcast=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Let's train on mini-batches of 128 samples each."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%time\n",
+      "for i in range(100):\n",
+      "    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n",
+      "        cost = train(trX[start:end], trY[start:end])\n",
+      "        \n",
+      "    errs = []\n",
+      "    for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n",
+      "        errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n",
+      "    print i, np.mean(np.concatenate(errs))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "With this naive model, predictions are poorer than those produced by the kNN model."
+     ]
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/3_net.ipynb b/notebooks/3_net.ipynb
new file mode 100644
index 0000000..37b6cb4
--- /dev/null
+++ b/notebooks/3_net.ipynb
@@ -0,0 +1,988 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:fc684fd1168901d7d18baf83d926aaf0042beea79127f8b118f4da677b8dc70b"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "An \"old\" network before 2006"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os\n",
+      "# os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n",
+      "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os,sys,inspect\n",
+      "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+      "parentdir = os.path.dirname(currentdir)\n",
+      "sys.path.insert(0,parentdir) \n",
+      "import load\n",
+      "\n",
+      "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n",
+      "\n",
+      "trX, teX, trY, teY = load.mnist(onehot=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import theano\n",
+      "from theano import tensor as T\n",
+      "import numpy as np\n",
+      "\n",
+      "def floatX(X):\n",
+      "    return np.asarray(X, dtype=theano.config.floatX)\n",
+      "\n",
+      "def init_weights(shape):\n",
+      "    return theano.shared(floatX(np.random.randn(*shape) * 0.01))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 3
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Generalize to compute stochastic gradient descent (SGD) on all model parametes. Note that SGD has a fixed learning rate which slow things down as you near the goal."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def sgd(cost, params, lr=0.05):\n",
+      "    grads = T.grad(cost=cost, wrt=params)\n",
+      "    updates = []\n",
+      "    for p, g in zip(params, grads):\n",
+      "        updates.append([p, p - g * lr])\n",
+      "    return updates"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "The logistic regression (LR) model is replaced with a neural network (NN) model with one hidden layer."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def model(X, w_h, w_o):\n",
+      "    h = T.nnet.sigmoid(T.dot(X, w_h))\n",
+      "    pyx = T.nnet.softmax(T.dot(h, w_o))\n",
+      "    return pyx"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 5
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "X = T.fmatrix()\n",
+      "Y = T.fmatrix()"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "We have 625 hidden units."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "w_h = init_weights((784, 625))\n",
+      "w_o = init_weights((625, 10))\n",
+      "\n",
+      "py_x = model(X, w_h, w_o)\n",
+      "y_x = T.argmax(py_x, axis=1)\n",
+      "\n",
+      "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n",
+      "params = [w_h, w_o]\n",
+      "updates = sgd(cost, params)\n",
+      "\n",
+      "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
+      "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 7
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%time\n",
+      "for i in range(100):\n",
+      "    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n",
+      "        cost = train(trX[start:end], trY[start:end])\n",
+      "    # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n",
+      "    errs = []\n",
+      "    for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n",
+      "        errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n",
+      "    print i, np.mean(np.concatenate(errs))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "0 0.703625801282\n",
+        "1"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.829126602564\n",
+        "2"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.867988782051\n",
+        "3"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.882512019231\n",
+        "4"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.889723557692\n",
+        "5"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.895633012821\n",
+        "6"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.899138621795\n",
+        "7"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.902043269231\n",
+        "8"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.904747596154\n",
+        "9"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.907151442308\n",
+        "10"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.908954326923\n",
+        "11"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.910957532051\n",
+        "12"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.912760416667\n",
+        "13"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.913962339744\n",
+        "14"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.914463141026\n",
+        "15"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.915564903846\n",
+        "16"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.916466346154\n",
+        "17"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.917467948718\n",
+        "18"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.918269230769\n",
+        "19"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.918569711538\n",
+        "20"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.919070512821\n",
+        "21"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.919170673077\n",
+        "22"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.919471153846\n",
+        "23"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.919871794872\n",
+        "24"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.92047275641\n",
+        "25"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.921574519231\n",
+        "26"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.922475961538\n",
+        "27"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.922375801282\n",
+        "28"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.923076923077\n",
+        "29"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.923477564103\n",
+        "30"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.924078525641\n",
+        "31"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.924479166667\n",
+        "32"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.924579326923\n",
+        "33"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.925480769231\n",
+        "34"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.925981570513\n",
+        "35"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.926482371795\n",
+        "36"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.926983173077\n",
+        "37"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.927383814103\n",
+        "38"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.927884615385\n",
+        "39"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.928685897436\n",
+        "40"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.928886217949\n",
+        "41"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.929387019231\n",
+        "42"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.929887820513\n",
+        "43"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.930488782051\n",
+        "44"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.93108974359\n",
+        "45"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.932091346154\n",
+        "46"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.932892628205\n",
+        "47"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.933493589744\n",
+        "48"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.934395032051\n",
+        "49"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.93499599359\n",
+        "50"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.936197916667\n",
+        "51"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.936598557692\n",
+        "52"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.936598557692\n",
+        "53"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.937199519231\n",
+        "54"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.9375\n",
+        "55"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.938401442308\n",
+        "56"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.939002403846\n",
+        "57"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.939403044872\n",
+        "58"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.939503205128\n",
+        "59"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.940204326923\n",
+        "60"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.940604967949\n",
+        "61"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.941205929487\n",
+        "62"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.941706730769\n",
+        "63"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.942107371795\n",
+        "64"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.943008814103\n",
+        "65"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.943309294872\n",
+        "66"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.944210737179\n",
+        "67"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.944711538462\n",
+        "68"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.945412660256\n",
+        "69"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.945713141026\n",
+        "70"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.946414262821\n",
+        "71"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.947315705128\n",
+        "72"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.947716346154\n",
+        "73"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.948116987179\n",
+        "74"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.948617788462\n",
+        "75"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.948818108974\n",
+        "76"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.949118589744\n",
+        "77"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.949719551282\n",
+        "78"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.950220352564\n",
+        "79"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.950520833333\n",
+        "80"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.951021634615\n",
+        "81"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.951622596154\n",
+        "82"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.952023237179\n",
+        "83"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.952223557692\n",
+        "84"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.952624198718\n",
+        "85"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.953425480769\n",
+        "86"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.953525641026\n",
+        "87"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.954126602564\n",
+        "88"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.954427083333\n",
+        "89"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.954627403846\n",
+        "90"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.954927884615\n",
+        "91"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.955128205128\n",
+        "92"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.955528846154\n",
+        "93"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.955428685897\n",
+        "94"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.955729166667\n",
+        "95"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.955929487179\n",
+        "96"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.956330128205\n",
+        "97"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.956730769231\n",
+        "98"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.957131410256\n",
+        "99"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.957231570513\n",
+        "CPU times: user 18min 9s, sys: 8min 9s, total: 26min 18s\n",
+        "Wall time: 6min 36s\n"
+       ]
+      }
+     ],
+     "prompt_number": 8
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 8
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/4_modern_net.ipynb b/notebooks/4_modern_net.ipynb
new file mode 100644
index 0000000..395bbc1
--- /dev/null
+++ b/notebooks/4_modern_net.ipynb
@@ -0,0 +1,1064 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:a24946d3f359951c3345d365cd78ed271a7c2fe1ce2db20406363ca25c9b1806"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "A more modern neural net\n",
+      "------------------------"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Three main changes: replace sigmoid with rectifier (LeRU), add dropout regularization, and replace SGD with a more modern learning rule (RMSProp). Dropout regularization allows us to add hidden layers (in the past this caused overfitting.)"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os\n",
+      "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n",
+      "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os,sys,inspect\n",
+      "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+      "parentdir = os.path.dirname(currentdir)\n",
+      "sys.path.insert(0,parentdir) \n",
+      "import load\n",
+      "\n",
+      "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n",
+      "\n",
+      "trX, teX, trY, teY = load.mnist(onehot=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import theano\n",
+      "from theano import tensor as T\n",
+      "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
+      "import numpy as np\n",
+      "\n",
+      "srng = RandomStreams()\n",
+      "\n",
+      "def floatX(X):\n",
+      "    return np.asarray(X, dtype=theano.config.floatX)\n",
+      "\n",
+      "def init_weights(shape):\n",
+      "    return theano.shared(floatX(np.random.randn(*shape) * 0.01))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stderr",
+       "text": [
+        "ERROR (theano.sandbox.cuda): nvcc compiler not found on $PATH. Check your nvcc installation and try again.\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stderr",
+       "text": [
+        "ERROR:theano.sandbox.cuda:nvcc compiler not found on $PATH. Check your nvcc installation and try again.\n"
+       ]
+      }
+     ],
+     "prompt_number": 3
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "`rectify` is our replacement for the sigmoid function."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def rectify(X):\n",
+      "    return T.maximum(X, 0.)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Modify `softmax` to remove max value to avoid explosition of exp (numeric stability)"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def softmax(X):\n",
+      "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
+      "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 5
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "We use a running RMS average of the gradient to scale the gradient size"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n",
+      "    grads = T.grad(cost=cost, wrt=params)\n",
+      "    updates = []\n",
+      "    for p, g in zip(params, grads):\n",
+      "        acc = theano.shared(p.get_value() * 0.) # acc is allocated for each parameter (p) with 0 values with the shape of p\n",
+      "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
+      "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
+      "        g = g / gradient_scaling\n",
+      "        updates.append((acc, acc_new))\n",
+      "        updates.append((p, p - lr * g))\n",
+      "    return updates"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Dropout regularization will randomly set to zero some of the nodes (with probability `p`). We have to compensate by scaling the nodes that were not dropped out."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def dropout(X, p=0.):\n",
+      "    if p > 0:\n",
+      "        retain_prob = 1 - p\n",
+      "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n",
+      "        X /= retain_prob\n",
+      "    return X"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 7
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Our new model has two hidden layers of 625 nodes."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden):\n",
+      "    X = dropout(X, p_drop_input)\n",
+      "    h = rectify(T.dot(X, w_h))\n",
+      "\n",
+      "    h = dropout(h, p_drop_hidden)\n",
+      "    h2 = rectify(T.dot(h, w_h2))\n",
+      "\n",
+      "    h2 = dropout(h2, p_drop_hidden)\n",
+      "    py_x = softmax(T.dot(h2, w_o))\n",
+      "    return h, h2, py_x"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 8
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "X = T.fmatrix()\n",
+      "Y = T.fmatrix()\n",
+      "\n",
+      "w_h = init_weights((784, 625))\n",
+      "w_h2 = init_weights((625, 625))\n",
+      "w_o = init_weights((625, 10))\n",
+      "\n",
+      "noise_h, noise_h2, noise_py_x = model(X, w_h, w_h2, w_o, 0.2, 0.5)\n",
+      "h, h2, py_x = model(X, w_h, w_h2, w_o, 0., 0.)\n",
+      "y_x = T.argmax(py_x, axis=1)\n",
+      "\n",
+      "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
+      "params = [w_h, w_h2, w_o]\n",
+      "updates = RMSprop(cost, params, lr=0.001)\n",
+      "\n",
+      "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
+      "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 9
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%time\n",
+      "for i in range(100):\n",
+      "    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n",
+      "        cost = train(trX[start:end], trY[start:end])\n",
+      "    # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n",
+      "    errs = []\n",
+      "    for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n",
+      "        errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n",
+      "    print i, np.mean(np.concatenate(errs))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "0 0.943810096154\n",
+        "1"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.962239583333\n",
+        "2"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.974258814103\n",
+        "3"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.974559294872\n",
+        "4"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.975761217949\n",
+        "5"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.97796474359\n",
+        "6"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.978966346154\n",
+        "7"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.978866185897\n",
+        "8"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.982271634615\n",
+        "9"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983173076923\n",
+        "10"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.982171474359\n",
+        "11"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984575320513\n",
+        "12"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.982872596154\n",
+        "13"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983673878205\n",
+        "14"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983373397436\n",
+        "15"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983874198718\n",
+        "16"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983673878205\n",
+        "17"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983774038462\n",
+        "18"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983673878205\n",
+        "19"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983273237179\n",
+        "20"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983974358974\n",
+        "21"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.983673878205\n",
+        "22"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984174679487\n",
+        "23"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984875801282\n",
+        "24"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984274839744\n",
+        "25"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984875801282\n",
+        "26"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985376602564\n",
+        "27"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985176282051\n",
+        "28"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984975961538\n",
+        "29"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985977564103\n",
+        "30"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985677083333\n",
+        "31"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984775641026\n",
+        "32"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985376602564\n",
+        "33"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "34"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985076121795\n",
+        "35"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986578525641\n",
+        "36"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984975961538\n",
+        "37"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.984975961538\n",
+        "38"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.98577724359\n",
+        "39"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986177884615\n",
+        "40"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.98687900641\n",
+        "41"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985677083333\n",
+        "42"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986378205128\n",
+        "43"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "44"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "45"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986378205128\n",
+        "46"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987179487179\n",
+        "47"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "48"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987479967949\n",
+        "49"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986278044872\n",
+        "50"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986278044872\n",
+        "51"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "52"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "53"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987279647436\n",
+        "54"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.98687900641\n",
+        "55"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987980769231\n",
+        "56"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987179487179\n",
+        "57"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986778846154\n",
+        "58"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986778846154\n",
+        "59"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986578525641\n",
+        "60"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986578525641\n",
+        "61"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "62"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986578525641\n",
+        "63"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "64"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986077724359\n",
+        "65"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987880608974\n",
+        "66"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "67"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.985677083333\n",
+        "68"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986778846154\n",
+        "69"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "70"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986578525641\n",
+        "71"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "72"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987379807692\n",
+        "73"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.98687900641\n",
+        "74"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987179487179\n",
+        "75"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986778846154\n",
+        "76"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987379807692\n",
+        "77"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "78"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987780448718\n",
+        "79"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986478365385\n",
+        "80"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987379807692\n",
+        "81"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986278044872\n",
+        "82"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987279647436\n",
+        "83"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987580128205\n",
+        "84"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987580128205\n",
+        "85"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987379807692\n",
+        "86"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987980769231\n",
+        "87"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987179487179\n",
+        "88"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986678685897\n",
+        "89"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986578525641\n",
+        "90"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987580128205\n",
+        "91"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987880608974\n",
+        "92"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987079326923\n",
+        "93"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "94"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "95"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987880608974\n",
+        "96"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986979166667\n",
+        "97"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987479967949\n",
+        "98"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.987479967949\n",
+        "99"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.986678685897\n",
+        "CPU times: user 1h 8min 35s, sys: 44min 35s, total: 1h 53min 10s\n",
+        "Wall time: 30min 27s\n"
+       ]
+      }
+     ],
+     "prompt_number": 10
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb
new file mode 100644
index 0000000..e75d672
--- /dev/null
+++ b/notebooks/5_convolutional_net.ipynb
@@ -0,0 +1,1102 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:5d9f76b1d1dc426c1b07e4480adfabee73388a181a66dd7bbdf1e9c1e0896efe"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Use 2D convolution - respect the 2D structure of the data by running the same network (same weights) on small 2D patches of the image."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os,sys,inspect\n",
+      "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+      "parentdir = os.path.dirname(currentdir)\n",
+      "sys.path.insert(0,parentdir) \n",
+      "import load\n",
+      "import os\n",
+      "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n",
+      "\n",
+      "trX, teX, trY, teY = load.mnist(onehot=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "On OSX this example is really slow and the GPU cause the Mac to shutdown! So run it on AWS with GPU (e.g. g2.2xlarge )"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import theano\n",
+      "from theano import tensor as T\n",
+      "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n",
+      "import numpy as np\n",
+      "from theano.tensor.nnet.conv import conv2d\n",
+      "from theano.tensor.signal.downsample import max_pool_2d\n",
+      "\n",
+      "srng = RandomStreams()\n",
+      "\n",
+      "def floatX(X):\n",
+      "    return np.asarray(X, dtype=theano.config.floatX)\n",
+      "\n",
+      "def init_weights(shape):\n",
+      "    return theano.shared(floatX(np.random.randn(*shape) * 0.01))\n",
+      "\n",
+      "def rectify(X):\n",
+      "    return T.maximum(X, 0.)\n",
+      "\n",
+      "def softmax(X):\n",
+      "    e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n",
+      "    return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n",
+      "\n",
+      "def dropout(X, p=0.):\n",
+      "    if p > 0:\n",
+      "        retain_prob = 1 - p\n",
+      "        X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX, nstreams=60*256)\n",
+      "        X /= retain_prob\n",
+      "    return X\n",
+      "\n",
+      "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n",
+      "    grads = T.grad(cost=cost, wrt=params)\n",
+      "    updates = []\n",
+      "    for p, g in zip(params, grads):\n",
+      "        acc = theano.shared(p.get_value() * 0.)\n",
+      "        acc_new = rho * acc + (1 - rho) * g ** 2\n",
+      "        gradient_scaling = T.sqrt(acc_new + epsilon)\n",
+      "        g = g / gradient_scaling\n",
+      "        updates.append((acc, acc_new))\n",
+      "        updates.append((p, p - lr * g))\n",
+      "    return updates"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stderr",
+       "text": [
+        "Using gpu device 0: GRID K520\n"
+       ]
+      }
+     ],
+     "prompt_number": 3
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Convolution --> activation --> pooling --> dropout"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "def model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden):\n",
+      "    # block to compute conv->activate->pool->noise\n",
+      "    l1a = rectify(conv2d(X, w, border_mode='full'))\n",
+      "    l1 = max_pool_2d(l1a, (2, 2))\n",
+      "    l1 = dropout(l1, p_drop_conv)\n",
+      "\n",
+      "    l2a = rectify(conv2d(l1, w2))\n",
+      "    l2 = max_pool_2d(l2a, (2, 2))\n",
+      "    l2 = dropout(l2, p_drop_conv)\n",
+      "\n",
+      "    l3a = rectify(conv2d(l2, w3))\n",
+      "    l3b = max_pool_2d(l3a, (2, 2))\n",
+      "    # return back to original vector represnation\n",
+      "    l3 = T.flatten(l3b, outdim=2)\n",
+      "    l3 = dropout(l3, p_drop_conv)\n",
+      "\n",
+      "    l4 = rectify(T.dot(l3, w4))\n",
+      "    l4 = dropout(l4, p_drop_hidden)\n",
+      "\n",
+      "    pyx = softmax(T.dot(l4, w_o))\n",
+      "    return l1, l2, l3, l4, pyx"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Reshape operation to conform to resize to single channel of 28 * 28 pixels."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "trX = trX.reshape(-1, 1, 28, 28)\n",
+      "teX = teX.reshape(-1, 1, 28, 28)\n",
+      "\n",
+      "X = T.ftensor4()\n",
+      "Y = T.fmatrix()"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 5
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Convolution weights (n_kernels, n_channels, kernel_w, kernel_h).  Moving up the heirarchy we move towards deeper, narrower representations thanks to increasing numbers of filters and maxpooling operations."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "w = init_weights((32, 1, 3, 3))\n",
+      "w2 = init_weights((64, 32, 3, 3))\n",
+      "w3 = init_weights((128, 64, 3, 3))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "w4 = init_weights((128 * 3 * 3, 625))\n",
+      "w_o = init_weights((625, 10))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 7
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Apply dropout during training."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, w_o, 0.2, 0.5)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 8
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Use a noiseless model for prediction."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, w_o, 0., 0.)\n",
+      "y_x = T.argmax(py_x, axis=1)\n",
+      "\n",
+      "\n",
+      "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n",
+      "params = [w, w2, w3, w4, w_o]\n",
+      "updates = RMSprop(cost, params, lr=0.001)\n",
+      "\n",
+      "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n",
+      "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stderr",
+       "text": [
+        "/usr/local/lib/python2.7/dist-packages/theano/tensor/subtensor.py:110: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n",
+        "  start in [None, 0] or\n",
+        "/usr/local/lib/python2.7/dist-packages/theano/tensor/subtensor.py:114: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n",
+        "  stop in [None, length, maxsize] or\n"
+       ]
+      }
+     ],
+     "prompt_number": 9
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%time\n",
+      "for i in range(100):\n",
+      "    for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n",
+      "        cost = train(trX[start:end], trY[start:end])\n",
+      "        #print '%d:%.4f'%(start/128,cost),\n",
+      "    #print\n",
+      "    # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n",
+      "    errs = []\n",
+      "    for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n",
+      "        errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n",
+      "    print '###',i, np.mean(np.concatenate(errs))"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "### 0 0.934795673077\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 1 0.973657852564\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 2 0.982872596154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 3 0.987179487179\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 4 0.987479967949\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 5 0.989783653846\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 6 0.987379807692\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 7 0.991987179487\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 8 0.991786858974\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 9 0.991786858974\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 10 0.992287660256\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 11 0.992487980769\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 12 0.991786858974\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 13 0.9921875\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 14 0.992588141026\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 15 0.993689903846\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 16 0.992788461538\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 17 0.993289262821\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 18 0.992287660256\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 19 0.992588141026\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 20 0.99358974359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 21 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 22 0.99358974359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 23 0.993289262821\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 24 0.993189102564\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 25 0.993489583333\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 26 0.993389423077\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 27 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 28 0.993790064103\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 29 0.993489583333\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 30 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 31 0.993289262821\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 32 0.993189102564\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 33 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 34 0.994190705128\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 35 0.994090544872\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 36 0.993489583333\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 37 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 38 0.993790064103\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 39 0.99358974359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 40 0.994090544872\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 41 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 42 0.993790064103\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 43 0.99358974359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 44 0.994090544872\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 45 0.99358974359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 46 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 47 0.993389423077\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 48 0.994090544872\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 49 0.993389423077\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 50 0.994290865385\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 51 0.994290865385\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 52 0.994391025641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 53 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 54 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 55 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 56 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 57 0.994090544872\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 58 0.99469150641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 59 0.993990384615\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 60 0.994391025641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 61 0.994591346154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 62 0.994391025641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 63 0.994190705128\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 64 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 65 0.994391025641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 66 0.994491185897\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 67 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 68 0.995092147436\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 69 0.995192307692\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 70 0.99469150641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 71 0.994491185897\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 72 0.995092147436\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 73 0.994891826923\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 74 0.993489583333\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 75 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 76 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 77 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 78 0.994591346154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 79 0.993790064103\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 80 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 81 0.994491185897\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 82 0.994591346154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 83 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 84 0.994290865385\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 85 0.994591346154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 86 0.994591346154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 87 0.994891826923\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 88 0.994991987179\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 89 0.99358974359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 90 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 91 0.995092147436\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 92 0.994290865385\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 93 0.994791666667\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 94 0.994391025641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 95 0.994391025641\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 96 0.994591346154\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 97 0.994190705128\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 98 0.993890224359\n",
+        "###"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 99 0.994391025641\n",
+        "CPU times: user 1h 34min 43s, sys: 45min 13s, total: 2h 19min 57s\n",
+        "Wall time: 2h 19min 56s\n"
+       ]
+      }
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "GPU 2h 19min 56s 0.994-0.995"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/KNN.ipynb b/notebooks/KNN.ipynb
new file mode 100644
index 0000000..1d0c291
--- /dev/null
+++ b/notebooks/KNN.ipynb
@@ -0,0 +1,102 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:2a506df8c7b7baaab1282610827bb2333815c5150a1098d957f2d4d75d49ef8b"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "One of the simplest ways to perform classification of the handwritten digits is to use a nearest neighbor classifier."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os,sys,inspect\n",
+      "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+      "parentdir = os.path.dirname(currentdir)\n",
+      "sys.path.insert(0,parentdir) \n",
+      "import load\n",
+      "import os\n",
+      "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n",
+      "\n",
+      "trX, teX, trLabel, teLabel = load.mnist(onehot=False)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import sklearn.neighbors\n",
+      "clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=1)\n",
+      "clf.fit(trX,trLabel)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 3,
+       "text": [
+        "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
+        "           metric_params=None, n_neighbors=1, p=2, weights='uniform')"
+       ]
+      }
+     ],
+     "prompt_number": 3
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "tePredict = clf.predict(teX)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "np.mean(tePredict == teLabel)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 5,
+       "text": [
+        "0.96909999999999996"
+       ]
+      }
+     ],
+     "prompt_number": 5
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "This method works quite well, but prediction on new examples is computationally expensive."
+     ]
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file
diff --git a/notebooks/index.ipynb b/notebooks/index.ipynb
new file mode 100644
index 0000000..0a74ee8
--- /dev/null
+++ b/notebooks/index.ipynb
@@ -0,0 +1,877 @@
+{
+ "metadata": {
+  "name": "",
+  "signature": "sha256:cb17786a324ccc4066b6446428b8d06831376bd8ca1f416866ee0d68af35f208"
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Follow the lecture [Introduction to Deep Learning with Python](https://www.youtube.com/watch?v=S75EdAcXHKk)"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "from IPython.display import YouTubeVideo\n",
+      "YouTubeVideo('S75EdAcXHKk')"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "html": [
+        "\n",
+        "        <iframe\n",
+        "            width=\"400\"\n",
+        "            height=300\"\n",
+        "            src=\"https://www.youtube.com/embed/S75EdAcXHKk\"\n",
+        "            frameborder=\"0\"\n",
+        "            allowfullscreen\n",
+        "        ></iframe>\n",
+        "        "
+       ],
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 1,
+       "text": [
+        "<IPython.lib.display.YouTubeVideo at 0x109e76fd0>"
+       ]
+      }
+     ],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "after performing the set described below you can walk through the notebooks:\n",
+      "    \n",
+      "* [KNN](./KNN.ipynb)\n",
+      "* [Mulitply two numbers with Theano](./0_multiply.ipynb)\n",
+      "* [Linear regression](./1_linear_regression.ipynb)\n",
+      "* [Logistic Regression](./2_logistic_regression.ipynb)\n",
+      "* [Neural Network](./3_net.ipynb)\n",
+      "* [Modern Neural Network](./4_modern_net.ipynb)\n",
+      "* [Convolution Neural Network](./5_convolutional_net.ipynb)"
+     ]
+    },
+    {
+     "cell_type": "heading",
+     "level": 1,
+     "metadata": {},
+     "source": [
+      "Get Data"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "download mnist files from http://yann.lecun.com/exdb/mnist/ and open the zipped file into `~/Downloads/lisa/data/mnist/`"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%bash\n",
+      "mkdir -p ~/Downloads/lisa/data/mnist/\n",
+      "cd ~/Downloads/lisa/data/mnist/\n",
+      "#wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
+      "#wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
+      "#wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
+      "#wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
+      "gunzip *.gz"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 4
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "!ls ~/Downloads/lisa/data/mnist/"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "t10k-images-idx3-ubyte\ttrain-images-idx3-ubyte\r\n",
+        "t10k-labels-idx1-ubyte\ttrain-labels-idx1-ubyte\r\n"
+       ]
+      }
+     ],
+     "prompt_number": 5
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os,sys,inspect\n",
+      "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n",
+      "parentdir = os.path.dirname(currentdir)\n",
+      "sys.path.insert(0,parentdir) \n",
+      "import load\n",
+      "\n",
+      "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n",
+      "\n",
+      "trX, teX, trY, teY = load.mnist(onehot=True)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "we have 60K images of handwritten numbers for training and 10K for testing. Each image has 28*28 gray-level pixels"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "trX.shape, teX.shape, trY.shape, teY.shape"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 7,
+       "text": [
+        "((60000, 784), (10000, 784), (60000, 10), (10000, 10))"
+       ]
+      }
+     ],
+     "prompt_number": 7
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import numpy as np\n",
+      "np.min(trX),np.max(trX),np.min(teX),np.max(teX)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 9,
+       "text": [
+        "(0.0, 1.0, 0.0, 1.0)"
+       ]
+      }
+     ],
+     "prompt_number": 9
+    },
+    {
+     "cell_type": "heading",
+     "level": 1,
+     "metadata": {},
+     "source": [
+      "Installation"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "Install Theano. On OS X it is highly recommended to [install using anaconda](http://deeplearning.net/software/theano/install.html#anaconda-1-5). If you also [installed MKL](http://continuum.io/blog/mkl-optimizations), which is also highly recommended, you will need to export the following environment variables before running `ipython notebook`\n",
+      "\n",
+      "    export DYLD_FALLBACK_LIBRARY_PATH=$HOME/anaconda/lib:$DYLD_FALLBACK_LIBRARY_PATH"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "On AWS I started from the following community image \"Ubuntu-14.04-Caffe-GPU - ami-588d0030\""
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "install Theano"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "!sudo pip install theano"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "Downloading/unpacking theano\r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB): \r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   0%  4.1kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   0%  8.2kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   0%  12kB \r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   0%  16kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   1%  20kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   1%  24kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   1%  28kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   1%  32kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   2%  36kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   2%  40kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   2%  45kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   2%  49kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   3%  53kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   3%  57kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   3%  61kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   3%  65kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   3%  69kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   4%  73kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   4%  77kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   4%  81kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   4%  86kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   5%  90kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   5%  94kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   5%  98kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   5%  102kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   6%  106kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   6%  110kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   6%  114kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   6%  118kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   6%  122kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   7%  126kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   7%  131kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   7%  135kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   7%  139kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   8%  143kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   8%  147kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   8%  151kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   8%  155kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   9%  159kB"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   9%  163kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   9%  167kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   9%  172kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):   9%  176kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  10%  180kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  10%  184kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  10%  188kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  10%  192kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  11%  196kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  11%  200kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  11%  204kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  11%  208kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  12%  212kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  12%  217kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  12%  221kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  12%  225kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  13%  229kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  13%  233kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  13%  237kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  13%  241kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  13%  245kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  14%  249kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  14%  253kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  14%  258kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  14%  262kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  15%  266kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  15%  270kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  15%  274kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  15%  278kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  16%  282kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  16%  286kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  16%  290kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  16%  294kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  16%  299kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  17%  303kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  17%  307kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  17%  311kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  17%  315kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  18%  319kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  18%  323kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  18%  327kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  18%  331kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  19%  335kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  19%  339kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  19%  344kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  19%  348kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  19%  352kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  20%  356kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  20%  360kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  20%  364kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  20%  368kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  21%  372kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  21%  376kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  21%  380kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  21%  385kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  22%  389kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  22%  393kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  22%  397kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  22%  401kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  22%  405kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  23%  409kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  23%  413kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  23%  417kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  23%  421kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  24%  425kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  24%  430kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  24%  434kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  24%  438kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  25%  442kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  25%  446kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  25%  450kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  25%  454kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  26%  458kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  26%  462kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  26%  466kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  26%  471kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  26%  475kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  27%  479kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  27%  483kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  27%  487kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  27%  491kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  28%  495kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  28%  499kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  28%  503kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  28%  507kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  29%  512kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  29%  516kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  29%  520kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  29%  524kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  29%  528kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  30%  532kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  30%  536kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  30%  540kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  30%  544kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  31%  548kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  31%  552kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  31%  557kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  31%  561kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  32%  565kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  32%  569kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  32%  573kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  32%  577kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  32%  581kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  33%  585kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  33%  589kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  33%  593kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  33%  598kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  34%  602kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  34%  606kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  34%  610kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  34%  614kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  35%  618kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  35%  622kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  35%  626kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  35%  630kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  35%  634kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  36%  638kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  36%  643kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  36%  647kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  36%  651kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  37%  655kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  37%  659kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  37%  663kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  37%  667kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  38%  671kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  38%  675kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  38%  679kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  38%  684kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  39%  688kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  39%  692kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  39%  696kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  39%  700kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  39%  704kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  40%  708kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  40%  712kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  40%  716kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  40%  720kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  41%  724kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  41%  729kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  41%  733kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  41%  737kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  42%  741kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  42%  745kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  42%  749kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  42%  753kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  42%  757kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  43%  761kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  43%  765kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  43%  770kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  43%  774kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  44%  778kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  44%  782kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  44%  786kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  44%  790kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  45%  794kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  45%  798kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  45%  802kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  45%  806kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  45%  811kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  46%  815kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  46%  819kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  46%  823kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  46%  827kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  47%  831kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  47%  835kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  47%  839kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  47%  843kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  48%  847kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  48%  851kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  48%  856kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  48%  860kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  49%  864kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  49%  868kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  49%  872kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  49%  876kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  49%  880kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  50%  884kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  50%  888kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  50%  892kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  50%  897kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  51%  901kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  51%  905kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  51%  909kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  51%  913kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  52%  917kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  52%  921kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  52%  925kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  52%  929kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  52%  933kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  53%  937kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  53%  942kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  53%  946kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  53%  950kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  54%  954kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  54%  958kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  54%  962kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  54%  966kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  55%  970kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  55%  974kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  55%  978kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  55%  983kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  55%  987kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  56%  991kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  56%  995kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  56%  999kB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  56%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  57%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  57%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  57%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  57%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  58%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  58%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  58%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  58%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  58%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  59%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  59%  1.0MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  59%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  59%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  60%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  60%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  60%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  60%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  61%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  61%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  61%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  61%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  62%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  62%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  62%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  62%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  62%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  63%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  63%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  63%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  63%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  64%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  64%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  64%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  64%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  65%  1.1MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  65%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  65%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  65%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  65%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  66%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  66%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  66%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  66%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  67%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  67%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  67%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  67%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  68%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  68%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  68%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  68%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  68%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  69%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  69%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  69%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  69%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  70%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  70%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  70%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  70%  1.2MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  71%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  71%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  71%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  71%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  71%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  72%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  72%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  72%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  72%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  73%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  73%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  73%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  73%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  74%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  74%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  74%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  74%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  75%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  75%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  75%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  75%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  75%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  76%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  76%  1.3MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  76%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  76%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  77%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  77%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  77%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  77%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  78%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  78%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  78%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  78%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  78%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  79%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  79%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  79%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  79%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  80%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  80%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  80%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  80%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  81%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  81%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  81%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  81%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  81%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  82%  1.4MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  82%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  82%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  82%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  83%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  83%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  83%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  83%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  84%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  84%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  84%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  84%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  84%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  85%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  85%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  85%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  85%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  86%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  86%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  86%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  86%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  87%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  87%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  87%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  87%  1.5MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  88%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  88%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  88%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  88%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  88%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  89%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  89%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  89%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  89%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  90%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  90%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  90%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  90%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  91%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  91%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  91%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  91%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  91%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  92%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  92%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  92%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  92%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  93%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  93%  1.6MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  93%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  93%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  94%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  94%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  94%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  94%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  94%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  95%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  95%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  95%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  95%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  96%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  96%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  96%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  96%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  97%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  97%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  97%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  97%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  98%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  98%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  98%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  98%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  98%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  99%  1.7MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  99%  1.8MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  99%  1.8MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):  99%  1.8MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB): 100%  1.8MB\r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB):            \r",
+        "  Downloading Theano-0.6.0.tar.gz (1.8MB): 1.8MB downloaded\r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "  Running setup.py (path:/tmp/pip_build_root/theano/setup.py) egg_info for package theano\r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "    \r\n",
+        "    warning: manifest_maker: MANIFEST.in, line 7: 'recursive-include' expects <dir> <pattern1> <pattern2> ...\r\n",
+        "    \r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "Requirement already satisfied (use --upgrade to upgrade): numpy>=1.5.0 in /usr/local/lib/python2.7/dist-packages/numpy-1.9.0-py2.7-linux-x86_64.egg (from theano)\r\n",
+        "Requirement already satisfied (use --upgrade to upgrade): scipy>=0.7.2 in /usr/local/lib/python2.7/dist-packages (from theano)\r\n",
+        "Installing collected packages: theano\r\n",
+        "  Running setup.py install for theano\r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "    changing mode of build/scripts-2.7/theano-cache from 644 to 755\r\n",
+        "    changing mode of build/scripts-2.7/theano-nose from 644 to 755\r\n",
+        "    changing mode of build/scripts-2.7/theano-test from 644 to 755\r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "    \r\n",
+        "    warning: manifest_maker: MANIFEST.in, line 7: 'recursive-include' expects <dir> <pattern1> <pattern2> ...\r\n",
+        "    \r\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "    changing mode of /usr/local/bin/theano-nose to 755\r\n",
+        "    changing mode of /usr/local/bin/theano-cache to 755\r\n",
+        "    changing mode of /usr/local/bin/theano-test to 755\r\n",
+        "Successfully installed theano\r\n",
+        "Cleaning up...\r\n"
+       ]
+      }
+     ],
+     "prompt_number": 14
+    },
+    {
+     "cell_type": "heading",
+     "level": 1,
+     "metadata": {},
+     "source": [
+      "Using GPU"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "You will need to install CUDA and export the following environment variables before running `ipython notebook`"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "On OSX\n",
+      "```bash\n",
+      "export DYLD_FALLBACK_LIBRARY_PATH=/Developer/NVIDIA/CUDA-6.5/lib:$DYLD_FALLBACK_LIBRARY_PATH\n",
+      "export PATH=/Developer/NVIDIA/CUDA-6.5/bin:$PATH\n",
+      "```"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "On AWS\n",
+      "```bash\n",
+      "export PATH=/usr/local/cuda-6.5/bin/:$PATH\n",
+      "```"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "you can control if CPU or GPU is used inside the notebook, however, you can only set it once. You need to restart the notebook if you want to change it"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "import os\n",
+      "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n",
+      "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "and check if you are using CPU or GPU"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "from theano import function, config, shared, sandbox\n",
+      "import theano.tensor as T\n",
+      "import numpy\n",
+      "import time\n",
+      "\n",
+      "vlen = 10 * 30 * 768  # 10 x #cores x # threads per core\n",
+      "iters = 1000\n",
+      "\n",
+      "rng = numpy.random.RandomState(22)\n",
+      "x = shared(numpy.asarray(rng.rand(vlen), config.floatX))\n",
+      "f = function([], T.exp(x))\n",
+      "print f.maker.fgraph.toposort()\n",
+      "t0 = time.time()\n",
+      "for i in xrange(iters):\n",
+      "    r = f()\n",
+      "t1 = time.time()\n",
+      "print 'Looping %d times took' % iters, t1 - t0, 'seconds'\n",
+      "print 'Result is', r\n",
+      "if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]):\n",
+      "    print 'Used the cpu'\n",
+      "else:\n",
+      "    print 'Used the gpu'"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "[GpuElemwise{exp,no_inplace}(<CudaNdarrayType(float32, vector)>), HostFromGpu(GpuElemwise{exp,no_inplace}.0)]\n",
+        "Looping 1000 times took"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        " 0.738588094711 seconds\n",
+        "Result is [ 1.23178029  1.61879349  1.52278066 ...,  2.20771813  2.29967761\n",
+        "  1.62323296]\n",
+        "Used the gpu\n"
+       ]
+      },
+      {
+       "output_type": "stream",
+       "stream": "stderr",
+       "text": [
+        "Using gpu device 0: GRID K520\n"
+       ]
+      }
+     ],
+     "prompt_number": 2
+    },
+    {
+     "cell_type": "markdown",
+     "metadata": {},
+     "source": [
+      "On OSX, for this example, the GPU is x4-x5 faster than CPU"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [],
+     "language": "python",
+     "metadata": {},
+     "outputs": []
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
\ No newline at end of file