diff --git a/CNN_DEMO/README.md b/CNN_DEMO/README.md new file mode 100644 index 0000000..7f01036 --- /dev/null +++ b/CNN_DEMO/README.md @@ -0,0 +1,43 @@ +# Step 0: Get going +To get going right away, train a LeNet-5 model on MNIST using Alt-Min by executing: +``` +python train_cnn_14jan.py +``` +# Step 1: HYPERPARAMETER TESTING +Write a hyperparameter grid search in `experiments` sub-folder. +e.g.: +* `./all_lenetFashionMnist_18jan.sh` does both alt-min (am) and SGD for fashion-mnist +* `./am_lenetMnist_18jan.sh` does am on mnist +* `./sgd_lenetMnist_18jan.sh` does sgd on mnist + +These files call `../train_cnn_14jan.py` in a loop, which trains LeNet architectures on training sets and record error on validation set. Make sure to set "savePath" etc. within that file to an appropriate sub-directory. + + +# Step 2: ANALYSIS +1. Still from the experiments folder, use `cnnMnistViz19jan.py` to determine the hyperparameters which performed the best, on average. E.g.: +``` +python -i ../cnnMnistViz19jan.py --loadPath='./lenetMnist_18jan/' +``` + +This will automatically save the results in `bestHypers` and `ims` subfolders within the folder specific via argument `--loadPath`. + +2. Copy the values saved in `bestHypers` subfolder (in `.pt` format) to the corresponding subfolder of `am2-paper-results/postHPexps/`. `postHPexps` stands for "POST-HYPER-PARAMETER-EXPERIMENTS", e.g. testing and visualization. + +3. To print the selected hyperparameters to terminal, execute from `am2-paper-results` e.g.: +``` +python -i printHPOrez_cnn.py --loadPath='postHPexps/lenet/lenet_mnist/' +``` + +# Step 3: TESTING +In the `postHPexps` subfolders, run the executables to initiate testing. Each one executes: +1. Load the best hyperparameters determined in Step (2a) and copied in Step (2b). +2. Train corresponding network using entire training set. +3. Evaluate on the test. +4. Repeat for many random intializations. + +# Step 4: VISUALIZATION +From the `postHPexps` subfolder, use `testRezViz_19jan.py` to generate figures displaying testing results, e.g. +``` +python ../../../testRezViz_19jan.py --loadPath='./testResults/' +``` + diff --git a/CNN_DEMO/__pycache__/altmin14jan.cpython-36.pyc b/CNN_DEMO/__pycache__/altmin14jan.cpython-36.pyc new file mode 100644 index 0000000..be3b10e Binary files /dev/null and b/CNN_DEMO/__pycache__/altmin14jan.cpython-36.pyc differ diff --git a/CNN_DEMO/__pycache__/models.cpython-36.pyc b/CNN_DEMO/__pycache__/models.cpython-36.pyc new file mode 100644 index 0000000..86c26d5 Binary files /dev/null and b/CNN_DEMO/__pycache__/models.cpython-36.pyc differ diff --git a/CNN_DEMO/altmin14jan.py b/CNN_DEMO/altmin14jan.py new file mode 100644 index 0000000..2f84a8b --- /dev/null +++ b/CNN_DEMO/altmin14jan.py @@ -0,0 +1,405 @@ +""" +@author Mattia Rigotti (augmented by Benjamin Cowen) +@date 15 Jan 2019 +""" +from __future__ import print_function, division +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import numpy as np + +from utils.nn import Flatten + + +def compute_codes_loss(code, nmod, lin, loss_fn, codes_target, mu, lambda_c): + ''' codes: outputs of the linear modules + nmod: non-linear module downstream from linear module + lin: linear module (Conv2d or Linear) + ''' + output = lin(nmod(code)) + loss = (1/mu)*loss_fn(output) + F.mse_loss(code, codes_target) + if lambda_c>0.0: + loss += (lambda_c/mu)*code.abs().mean() + return loss + + +def update_memory(As, Bs, inputs, codes, model_mods, eta=0.0): + "Updates the bookkeeping matrices using codes as in Mairal2009" + if hasattr(model_mods, 'n_inputs'): + x = inputs.view(-1, model_mods.n_inputs) + else: + x = inputs + + with torch.no_grad(): + id_codes = [i for i,m in enumerate(model_mods) if hasattr(m, 'has_codes') and getattr(m, 'has_codes')] + for i, (idx, c_in, c_out) in enumerate(zip(id_codes, [x]+codes[:-1], codes)): + try: + nmod = model_mods[idx-1] + except IndexError: + nmod = lambda x: x + + a = nmod(c_in) + if eta == 0.0: + As[i] += a.t().mm(a) + Bs[i] += c_out.t().mm(a) + else: + As[i] = (1-eta)*As[i] + eta*a.t().mm(a) + Bs[i] = (1-eta)*Bs[i] + eta*c_out.t().mm(a) + return As, Bs + + +def update_hidden_weights_bcd_(model_mods, As, Bs, lambda_w): + # Use BCD to update intermediate weights + id_codes = [i for i,m in enumerate(model_mods) if hasattr(m, 'has_codes') and getattr(m, 'has_codes')] + for i, A, B in zip(id_codes, As, Bs): + model_mods[i].weight.data = BCD(model_mods[i].weight.data, A, B, lambda_w) + + +def BCD(w, A, B, lambda_w, eps=1e-3, max_iter=20, return_errors=False): + # lambda_w is referenced to A_jj in every column + B = B.div(A.diag() + 1e-10) + A = A.div(A.diag() + 1e-10) + + errors = [] + with torch.no_grad(): + for i in range(max_iter): + w_pre = w.clone() + error = 0 + for j in range(A.shape[1]): + delta_j = (B[:,j] - w.mv(A[:,j])) + w[:,j].add_(delta_j) + # u_j /= max(u_j.norm(), 1.0) # This was in Mairal2009, but assumes that B has spectral radius smaller than A + # Shrinkage step (sparsity regularizer) + if lambda_w > 0.0: + sign_w = w[:,j].sign() + w[:,j].abs_().add_(-lambda_w).clamp_(min=0.0).mul_(sign_w) + error += delta_j.abs().mean().item() + errors.append(error) + # Converged is there is no change between steps + if (w - w_pre).abs().max().item() < eps: + break + + if return_errors: + return w, errors + else: + return w + + +def post_processing_step(model, data, target, criterion, lambda_w, n_iter=1): + with torch.no_grad(): + output, codes = get_codes(model, data) + + update_last_layer_(model, codes[-1], target, criterion, lambda_w=args.lambda_w, n_iter=n_iter) + + +def insert_mod(model_mods, mod, has_codes): + "If a mod is not empty, close it, include it, and start a new mod" + if len(mod) == 1: + model_mods.add_module(str(len(model_mods)), mod[0]) + model_mods[-1].has_codes = has_codes + elif len(mod) > 1: + model_mods.add_module(str(len(model_mods)), mod) + model_mods[-1].has_codes = has_codes + mod = nn.Sequential() + return mod + +# EXTRACTED FROM get_mods 2 Jan 2019: +def set_mod_optimizers_(model_mods, optimizer=None, optimizer_params={}, data_parallel=False): + ''' + Sets/resets model optimizer(s). + ''' + if optimizer is not None: + # Include an optimizer in modules with codes + for m in model_mods: + if m.has_codes: + m.optimizer = getattr(optim, optimizer)(m.parameters(), **optimizer_params) + + # Add optimizer to the last layer + model_mods[-1].optimizer = getattr(optim, optimizer)(model_mods[-1].parameters(), **optimizer_params) + + if data_parallel: + data_parallel_mods_(model_mods) + + return model_mods + +def get_mods(model, optimizer=None, optimizer_params={}, data_parallel=False): + '''Returns all the modules in a nn.Sequential alternating linear and non-linear modules + Arguments: + optimizer: if not None, each module will be given an optimizer of the indicated type + with parameters in the dictionary optimizer_params + EDIT BY BEN: 2 Jan 2019 + --- if model is already a nn.Sequential, just resets the optimizers. + ''' + if not isinstance(model, nn.Sequential): + model_mods = nn.Sequential() + if hasattr(model, 'n_inputs'): + model_mods.n_inputs = model.n_inputs + + nmod, lmod = nn.Sequential(), nn.Sequential() + for m in list(model.features) + [Flatten()] + list(model.classifier): + if any([isinstance(m, t) for t in [nn.Conv2d, nn.Linear, nn.BatchNorm2d, nn.BatchNorm1d]]) or hasattr(m, 'has_codes'): + nmod = insert_mod(model_mods, nmod, has_codes=False) + lmod.add_module(str(len(lmod)), m) + else: + lmod = insert_mod(model_mods, lmod, has_codes=True) + nmod.add_module(str(len(nmod)), m) + + insert_mod(model_mods, nmod, has_codes=False) + insert_mod(model_mods, lmod, has_codes=True) + + # Last layer that generates codes is lumped together with adjacent modules to produce the last layer + id_codes = [i for i,m in enumerate(model_mods) if hasattr(m, 'has_codes') and getattr(m, 'has_codes')] + + model_tmp = model_mods[:id_codes[-2]+1] + model_tmp.add_module(str(len(model_tmp)), model_mods[id_codes[-2]+1:]) + model_tmp[-1].has_codes = False + model_mods = model_tmp + + # Added by Ben. + if optimizer is not None: + set_mod_optimizers_(model, optimizer = optimizer, + optimizer_params = optimizer_params, data_parallel=data_parallel) + return model_mods + +def data_parallel_mods_(model_mods): + for i,m in enumerate(model_mods): + model_mods[i] = torch.nn.DataParallel(m) + model_mods[i].has_codes = m.has_codes + if hasattr(m, 'optimizer'): + model_mods[i].optimizer = m.optimizer + + +# EDIT BY BEN 12/12/2018: +# all RNN stuff... +def get_codes(model_mods, inputs): + '''Runs the architecture forward using `inputs` as inputs, and returns outputs and intermediate codes + ''' + # First mess with the input shape like this. + # Probably have to change for the RNN. + if hasattr(model_mods, 'n_inputs'): + x = inputs.view(-1, model_mods.n_inputs) + else: + x = inputs + + # If doing RNN, codes have to be computed a certain way. + if hasattr(model_mods, 'isRNN') and getattr(model_mods, 'isRNN'): + codes = model_mods.forward(x, get_codes=True) + + else: + # As codes only return outputs of linear layers + codes = [] + for m in model_mods: + x = m(x) + if hasattr(m, 'has_codes') and getattr(m, 'has_codes'): + codes.append(x.clone()) + # Do not include output of very last linear layer (not counted among codes) + return x, codes + + +def update_codes(codes, model_mods, targets, criterion, mu, lambda_c, n_iter, lr): + id_codes = [i for i,m in enumerate(model_mods) if hasattr(m, 'has_codes') and getattr(m, 'has_codes')] + for l in range(1, len(codes)+1): + idx = id_codes[-l] + + codes[-l].requires_grad_(True) + optimizer = optim.SGD([codes[-l]], lr=lr) + codes_initial = codes[-l].clone() + + try: + nmod = model_mods[idx+1] + except IndexError: + nmod = lambda x: x + try: + lin = model_mods[idx+2] + except IndexError: + lin = lambda x: x + + if l == 1: # last layer + loss_fn = lambda x: criterion(x, targets) + else: # intermediate layers + loss_fn = lambda x: F.mse_loss(x, codes[-l+1].detach()) + + # EDIT BY BEN 12/6: + # This optionally exits if converged. +# chgTol = 1e-4 +# chg = chgTol+1 + # first iteration is out here. + optimizer.zero_grad() + loss = compute_codes_loss(codes[-l], nmod, lin, loss_fn, codes_initial, mu, lambda_c) + loss.backward() + optimizer.step() + it = 1 + while ( (it < n_iter)):# and (chg>chgTol)): + it +=1 + # For tracking convergence: + prevLoss = loss.item() + # Gradient Descent Step: + optimizer.zero_grad() + loss = compute_codes_loss(codes[-l], nmod, lin, loss_fn, codes_initial, mu, lambda_c) + loss.backward() + optimizer.step() + # Check convergence: +# chg = (np.abs(loss.item() - prevLoss)/np.abs(prevLoss)) + return codes + + +def update_last_layer_(mod_out, inputs, targets, criterion, n_iter): + for it in range(n_iter): + mod_out.optimizer.zero_grad() + outputs = mod_out(inputs) + loss = criterion(outputs, targets) + loss.backward() + mod_out.optimizer.step() + + +def update_hidden_weights_adam_(model_mods, inputs, codes, lambda_w, n_iter, dropout_perc=0): + id_codes = [i for i,m in enumerate(model_mods) if hasattr(m, 'has_codes') and getattr(m, 'has_codes')] + + if hasattr(model_mods, 'n_inputs'): + x = inputs.view(-1, model_mods.n_inputs) + else: + x = inputs + + for idx, c_in, c_out in zip(id_codes, [x]+codes[:-1], codes): + # Randomly skip this layer according to dropout_perc: + if (torch.rand(1).item() <= dropout_perc) and dropout_perc>0.00001: + continue + + lin = model_mods[idx] + if idx >= 1: + nmod = model_mods[idx-1] + else: + nmod = lambda x: x + + # EDIT BY BEN 12/6: + # This optionally exits if converged. +# chgTol = 1e-4 +# chg = chgTol+1 + # first iteration is out here. + lin.optimizer.zero_grad() + loss = F.mse_loss(lin(nmod(c_in)), c_out.detach()) + if lambda_w > 0.0: + loss += lambda_w*lin.weight.abs().mean() + loss.backward() + lin.optimizer.step() + it = 1 + while ( (it < n_iter)):# and (chg>chgTol)): + it +=1 + # For tracking convergence: + prevLoss = loss.item() + # Gradient Descent Step: + lin.optimizer.zero_grad() + loss = F.mse_loss(lin(nmod(c_in)), c_out.detach()) + if lambda_w > 0.0: + loss += lambda_w*lin.weight.abs().mean() + loss.backward() + lin.optimizer.step() + # Check convergence: +# chg = (np.abs(loss.item() - prevLoss)/np.abs(prevLoss)) + + + +# ------------------------------------------------------------------------ +# Misc +# ------------------------------------------------------------------------ +def update_codes_TANH(model, codes, targets, mu, n_iter, lr): + '''Update codes approximating sign non-linearity with the soft version tanh + ''' + # Last layer codes + codes[-1].requires_grad_(True) + codes_target = codes[-1].clone() + optimizer = optim.SGD([codes[-1]], lr=lr) + for it in range(n_iter): + optimizer.zero_grad() + loss = compute_codes_loss(codes[-1], F.nll_loss, targets, torch.tanh, model.w_out, codes_target, mu, 0) + loss.backward() + optimizer.step() + + # Hidden layers codes + for l in range(2, len(codes)): + codes[-l].requires_grad_(True) + codes_target = codes[-l].clone() + optimizer = optim.SGD([codes[-l]], lr=lr) + for it in range(n_iter): + optimizer.zero_grad() + loss = compute_codes_loss(codes[-l], F.mse_loss, codes[-l+1].data, torch.tanh, model.w[-l+1], codes_target, mu, 0) + loss.backward() + optimizer.step() + + return codes + + +class OnlineCov(nn.Module): + '''Online covariance class + Online calcularion of covariance matrix between a stream x and y, updated as in the Online paragraph of https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Covariance" + ''' + def __init__(self, in_features, out_features=None): + super(OnlineCov, self).__init__() + if out_features is None: + out_features = in_features + # Covariance + self.register_buffer('cc', torch.zeros(out_features, in_features)) + # Means + self.register_buffer('mX', torch.zeros(in_features)) + self.register_buffer('mY', torch.zeros(out_features)) + # Iteration number + self.n_iter = 0 + + self.in_features = in_features + self.out_features = out_features + + def reset(self): + self.cc.zero_() + self.mX.zero_() + self.mY.zero_() + self.n_iter = 0 + + def update_cov(self, x, y): + # Checks + if x.dim() < 2: + x = x.view(1,-1) + if y.dim() < 2: + y = y.view(1,-1) + assert x.size(0) == y.size(0) + + # Start update + self.n_iter += 1 + + self.mX += (x - self.mX).mean(0)/self.n_iter + dx = x - self.mX + dy = y - self.mY # Note: it's correct that dy is computed with previous mean + + self.cc += dy.t().mm(dx) + + self.mY += (y - self.mY).mean(0)/self.n_iter + + def get_cc(self): + return self.cc/self.n_iter + + def __len__(self): + return self.n_iter + + def extra_repr(self): + return '{in_features}, {out_features}, n_iter={n_iter}'.format(**self.__dict__) + + +def update_memory_cov(As, Bs, codes, nlin=F.relu): + '''Updates the bookkeeping matrices using codes as in Mairal2009, but using centered variance and covariance + Arguments: + As, Bs: lists of OnlineCov objects + ''' + a = codes[0].data + for i in range(len(codes)-1): + As[i].update_cov(a, a) + Bs[i].update_cov(a, codes[i+1].data) + a = nlin(codes[i+1].data) + return As, Bs + + +def update_hidden_weights_cov_(Ws, As, Bs, lambda_w): + # Use BCD to update intermediate weights + for i, (A, B) in enumerate(zip(As, Bs)): + Ws[i].weight.data = BCD(Ws[i].weight.data, A.get_cc(), B.get_cc(), lambda_w) + Ws[i].bias.data = B.mY - Ws[i].weight.data.mv(B.mX) + diff --git a/CNN_DEMO/cnnMnistViz19jan.py b/CNN_DEMO/cnnMnistViz19jan.py new file mode 100644 index 0000000..c068368 --- /dev/null +++ b/CNN_DEMO/cnnMnistViz19jan.py @@ -0,0 +1,423 @@ +""" +Creates visualizations for CNN hyperparameter search results. + +@author Benjamin Cowen +@date 17 Jan 2019 +""" + +############################################ +# Imports ---------------------------------- + +# Loading +from utils import ddict, load_dataset +import glob +import torch as th + +# Plotting and Saving +# The following two lines are for the remote branch. +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import os +from shutil import copyfile +import numpy as np +import argparse + + +p = argparse.ArgumentParser(description='Visualize hyperparameter grid search results.') +p.add_argument('--loadPaths', type=str, default=None, metavar='path1/a/, path2/b/', + help='A list of strings/paths to results.') +p.add_argument('--loadPath', type=str, default=None, metavar='path/to/data/', + help='Path to directory containing data to plot.') +p.add_argument('--savePath', type=str, default=None, metavar='path/to/savedIms/', + help='Path to directory containing images saved here.') +p.add_argument('--onlyBest', type=bool, default=False, metavar='True', + help='Set to true if you DONT want all the imgs (only best params).') +p.add_argument('--saveBestHyperParameterPath', type=str, default=None, metavar='path/to/savedHyperParameters/', + help='Path to directory to save the BEST hyperparameters according to the exp.') +p.add_argument('--epoch0-accs', type=str, default=None, metavar='path/to/epoch-0-accuracies/', + help='Path to directory containing the epoch-0 accuracies... (avg result for all methods)') +p.add_argument('--reduced', type=bool, default=False, metavar='T/F', + help='skip methods...') + +args = p.parse_args() + +# Check arguments. +# Assumes loadPaths or loadPath is valid... +if args.loadPaths is None: + # Load and Save Paths + loadPaths = [args.loadPath] +else: + loadPaths = args.loadPaths.strip('[]').split(',') + +# For saving images +if args.savePath is None: + savePath = loadPaths[0] + "ims/" +else: + savePath = args.savePath +if not os.path.exists(savePath): + os.mkdir(savePath) +# For saving the best hyperparameters +if args.saveBestHyperParameterPath is None: + saveHPpath = loadPaths[0] + "bestHypers/" +else: + saveHPpath = args.saveBestHyperParameterPath +if not os.path.exists(saveHPpath): + os.mkdir(saveHPpath) + +print('*********************************************************') +print('Saving images to '+savePath) +print('Saving best hyperparameters to '+saveHPpath) +print('*********************************************************') + +############################################ +# Make a copy of this file in the savePath! +here = os.path.realpath(__file__) +there = savePath + (__file__).replace('../','') +copyfile(here,there) + +############################################ +# Decide tree structure here... (TODO way to automate this?) +# Stands for "all shelves" +allSh = {"sgd":{}, "adam":{}, # Backprop-based methods + "amAdam1_1_me1":{}, + "amAdam5_1_me1":{}, + "amAdam1_5_me1":{}, + "amAdam5_5_me1":{} + } + +# Loop thru given loadpaths. +for k,loadPath in enumerate(loadPaths): + print('Loading files from: ' + loadPath) + for filepath in glob.iglob(loadPath + '*.pt'): + fileName = filepath.replace(loadPath,'') + + # Go ahead and load it. + DD = ddict()._load(filepath) + algBase = DD.args['opt_method'] + + if not hasattr(DD,'perf'): + continue + + # Backprop-type. + if algBase=='sgd': + expID = 'sgd' + elif algBase=='adam': + expID='adam' + # Altmin-type. + elif algBase=='altmin': + nit1 = str(DD.args['n_iter_codes']) + nit2 = str(DD.args['n_iter_weights']) + me = str(DD.args['mini_epochs']) + expID='amAdam' + nit1 + '_' + nit2 + '_me' + me + else: + raise Exception("Unrecognized optimization method "+algBase+" (only 'altmin','sgd','adam')") + + # Now we have to determine the parameter Combination ID. + # Location of the first digit of Combo ID. (comes right after "combo" string.) + numStart = fileName.find('combo')+len('combo') + n = 1 # Number of digits in the Combo ID. + nextChar = fileName[numStart+n] + while not(nextChar=="t"): # Scrolling thru filename until the number ends. + n += 1 + nextChar = fileName[numStart+n] + comboNum = fileName[numStart:(numStart+n)] # this better be an integer combo ID :) + useKey = "combo"+ comboNum +'_'+ str(k) # Combination ID !!! + + # Finally, add it to results. If it's a new key, make it. + # If it's just new trial, append it + if useKey in allSh[expID]: + # Append + allSh[expID][useKey].append(ddict()._load(filepath)) + else: + # Or make new + allSh[expID][useKey] = [ddict()._load(filepath)] + +# Compute the average accuracies for epoch 0 (ie, using initializations only) +epoch0_meanTr = None +if args.epoch0_accs is not None: + epoch0_meanTr = [] + epoch0_meanTe = [] + # Compute means and std devs of the initializations. + for filepath in glob.iglob(args.epoch0_accs + '*.pt'): + DD = ddict()._load(filepath) + epoch0_meanTr += DD.perf['tr'] + epoch0_meanTe += DD.perf['te'] + + # Finally average. + epoch0_stdTr = th.Tensor( epoch0_meanTr ).std() + epoch0_meanTr = th.Tensor( epoch0_meanTr ).mean() + epoch0_stdTe = th.Tensor( epoch0_meanTe ).std() + epoch0_meanTe = th.Tensor( epoch0_meanTe ).mean() + + +############################################ +# Print loss figures ----------------------- +bestRez = {} +allMeanTes={} +for expID, sh in allSh.items(): + bestRez[expID] = {"acc":-1} # Initialize each Exp ID. + allMeanTes[expID] = {} + for comboID, rez in sh.items(): + numTrials = len(rez) + numFailedTrials = 0 + + # (0.) Collect the Training/Testing histories. + # Compute the mean and std dev across trials. + numEpochs = rez[0].args['epochs'] #should be same for all trials. + allTr = None + allTe = None + for trial in range(numTrials): + # Shelf for current trial. + tShelf = rez[trial] + + # CHECK VALIDITY:----------------- + # If no 'perf' field, then performance wasn't yet recorded. + if not hasattr(tShelf,'perf'): + continue + else: + expData = tShelf.perf + + #------------------------------------- + # If the number of recorded epochs << number allowed epochs, + # then add zeros to the end. EG force the length to be numEpochs. + currTrAcc = th.Tensor(expData['tr']) + currTeAcc = th.Tensor(expData['te']) + + # Double check failure to train. +# if currTeAcc.mean()<0.2: +# numFailedTrials+=1 +# continue + if len(expData['tr'])<=2: + continue + # Otherwise you can still visualize incomplete results. + if (len(expData['tr'])< numEpochs): + currTrAcc = th.cat([currTrAcc, th.zeros(numEpochs-len(expData['tr']))]) + currTeAcc = th.cat([currTeAcc, th.zeros(numEpochs-len(expData['te']))]) + + # If it checks out, extract the data. + currTrAcc = currTrAcc.view(numEpochs, 1) + currTeAcc = currTeAcc.view(numEpochs, 1) + + # Append into a numpy array so we can get mean/std later. + if allTr is None: + allTr = currTrAcc + allTe = currTeAcc + else: + allTr = th.cat([allTr, currTrAcc], 1) + allTe = th.cat([allTe, currTeAcc], 1) + # End of loop through the trials. + # If allTr is None, NO trials were valid (for various reasons). + # if numFailedTrials > numTrials/2, then more than half + # of the trials failed due to initialization randomness. + if allTr is None or (numFailedTrials > numTrials/2): + continue + # (1.) Else, compute mean and std devs. + # Update the number of trials actually used in this analysis. + numTrials = allTr.size(1) + # Only average the nonzero entries. + meansTr = (allTr.sum(1)/allTr.gt(0).sum(1).float()) + meansTe = (allTe.sum(1)/allTe.gt(0).sum(1).float()) + # Initialize with zero. + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + # If there were actually multiple trials, fill in using NONZERO entries only. + if numTrials>1: + for epoch, tnsr in enumerate(allTr): + # This is my stupid way of doing ONLY NONZERO std dev's... + stdtr = tnsr[ tnsr.gt(0) ].std().numpy() + if np.isnan(stdtr): + stdtr=np.array([0]) + stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() + if np.isnan(stdte): + stdte=np.array([0]) + # Finally... + stdsTr[epoch] = stdtr + stdsTe[epoch] = stdte + meansTr = meansTr.numpy() + meansTe = meansTe.numpy() + + + # (1.5) for plotting, we REMOVE the zeros added during mean-computation. + while np.isnan(meansTr[-1]) or np.isnan(meansTe[-1]) or meansTr[-1]<1e-6 or meansTe[-1]<1e-6: + meansTr = meansTr[:-1] + meansTe = meansTe[:-1] + stdsTr = stdsTr[:-1] + stdsTe = stdsTe[:-1] + + if epoch0_meanTr is not None: + meansTr = np.insert(meansTr, 0, epoch0_meanTr) + meansTe = np.insert(meansTe, 0, epoch0_meanTe) + stdsTr = np.insert(stdsTr, 0, epoch0_stdTr) + stdsTe = np.insert(stdsTe, 0, epoch0_stdTe) + x_axis = range(0, len(meansTr)) + else: + x_axis = range(1, len(meansTr)+1) + + allMeanTes[expID][comboID]=meansTe + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: +# if expID=='amAdam5_me1': + # (2.a) Extract information about this setting for titles etc. + ar = rez[0].args + # Model Size. + + # Alg name. + algName = ar['opt_method'] + modelName=ar['model'] + TITLE =modelName+' trained with '+algName + + # Print other hyperparameters + LR = ar['lr_weights'] + + if algName in ['sgd', 'adam']: + l1_w = ar['lambda_w'] + TITLE2 = '\nWeight LR={}, L1={}; '.format(LR,l1_w) + else: + algName += '-{}_me{}'.format(ar['n_iter_weights'] , ar['mini_epochs']) + codeLR = ar['lr_codes'] + l1_c = ar['lambda_c'] + muMult = ar['mult_mu'] + muMax = ar['max_mu'] + l1wFact = ar['lambda_w_muFact'] + l1cFact = ar['lambda_c_muFact'] + + TITLE2 = '\nWeight LR={},L1={}\mu; '.format(LR,l1wFact) + TITLE2 += 'Code LR={},L1={}\mu; '.format(codeLR,l1cFact) + TITLE2 += '\n\mu factor = {}; '.format(muMult) + TITLE2 += '\mu-max={}'.format(muMax) + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = '.', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '.', label='Testing') + plt.legend() + plt.ylim([.8,1]) + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+TITLE2) + plt.savefig(savePath + expID + comboID + 'allTrials.png') + + ## (3.) Finally, record the best results for each setting. + #(3.a) determine the final (nonzero) accuracy: + if meansTe[-1] > bestRez[expID]['acc']:# and len(meansTe)>9: + # For Plotting + bestRez[expID]['xax'] = x_axis + bestRez[expID]['meansTr'] = meansTr + bestRez[expID]['stdsTr'] = stdsTr + bestRez[expID]['meansTe'] = meansTe + bestRez[expID]['stdsTe'] = stdsTe + bestRez[expID]['numTr'] = numTrials + # For later. + bestRez[expID]['comboID'] = comboID + bestRez[expID]['acc'] = meansTe[-1] + bestRez[expID]['lr_weights'] = rez[0].args['lr_weights'] + bestRez[expID]['bsz'] = rez[0].args['batch_size'] + if expID.startswith('am'): + bestRez[expID]['lr_codes'] = rez[0].args['lr_codes'] + bestRez[expID]['d_mu'] = rez[0].args['d_mu'] + bestRez[expID]['muMult'] = rez[0].args['mult_mu'] + bestRez[expID]['muMax'] = rez[0].args['max_mu'] + + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. + +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', # Backprop-based methods + "amAdam1_1_me1":'r', + "amAdam5_1_me1":'y', + "amAdam1_5_me1":'m', + "amAdam5_5_me1":'c' + } +markers = {"sgd":'o', "adam":'o', + "amAdam1_1_me1":'*', + "amAdam5_1_me1":'d', + "amAdam1_5_me1":'P', + "amAdam5_5_me1":'x' + } +plotNames={"amAdam1_1_me1":'AM-Adam11', + "amAdam5_1_me1":'AM-Adam51', + "amAdam1_5_me1":'AM-Adam15', + "amAdam5_5_me1":'AM-Adam55', + 'sgd':'SGD', 'adam':"Adam" + } +#colors = ['b','g','r','c','m','y','k'] + +markerSize = 4 +modelName = rez[0].args['model'] +resUsableTitle = modelName + +dataName = rez[0].args['dataset'] +trmark='.' +temark='^' +numEpochs=10 +######## ALL PLOTS +for i, xLims in enumerate([[-0.1, numEpochs+0.1]]): + for k,yLims in enumerate([[0., 1], [0.6,0.93], [0.85,1], [0.75,1]]): + plt.figure(i*k + k) + plt.clf() + colorID=0 + if yLims[0]>0 or yLims[1]<1 or xLims[1] numTrials/2, then more than half + # of the trials failed due to initialization randomness. + if allTr is None or (numFailedTrials > numTrials/2): + continue + # (1.) Else, compute mean and std devs. + # Update the number of trials actually used in this analysis. + numTrials = allTr.size(1) + # Only average the nonzero entries. + meansTr = (allTr.sum(1)/allTr.gt(0).sum(1).float()) + meansTe = (allTe.sum(1)/allTe.gt(0).sum(1).float()) + # Initialize with zero. + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + # If there were actually multiple trials, fill in using NONZERO entries only. + if numTrials>1: + for epoch, tnsr in enumerate(allTr): + # This is my stupid way of doing ONLY NONZERO std dev's... + stdtr = tnsr[ tnsr.gt(0) ].std().numpy() + if np.isnan(stdtr): + stdtr=np.array([0]) + stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() + if np.isnan(stdte): + stdte=np.array([0]) + # Finally... + stdsTr[epoch] = stdtr + stdsTe[epoch] = stdte + meansTr = meansTr.numpy() + meansTe = meansTe.numpy() + + + # (1.5) for plotting, we REMOVE the zeros added during mean-computation. + while np.isnan(meansTr[-1]) or np.isnan(meansTe[-1]) or meansTr[-1]<1e-6 or meansTe[-1]<1e-6: + meansTr = meansTr[:-1] + meansTe = meansTe[:-1] + stdsTr = stdsTr[:-1] + stdsTe = stdsTe[:-1] + + if epoch0_meanTr is not None: + meansTr = np.insert(meansTr, 0, epoch0_meanTr) + meansTe = np.insert(meansTe, 0, epoch0_meanTe) + stdsTr = np.insert(stdsTr, 0, epoch0_stdTr) + stdsTe = np.insert(stdsTe, 0, epoch0_stdTe) + x_axis = range(0, len(meansTr)) + else: + x_axis = range(1, len(meansTr)+1) + + allMeanTes[expID][comboID]=meansTe + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: +# if expID=='amAdam5_me1': + # (2.a) Extract information about this setting for titles etc. + ar = rez[0].args + # Model Size. + + # Alg name. + algName = ar['opt_method'] + modelName=ar['model'] + TITLE =modelName+' trained with '+algName + + # Print other hyperparameters + LR = ar['lr_weights'] + + if algName in ['sgd', 'adam']: + l1_w = ar['lambda_w'] + TITLE2 = '\nWeight LR={}, L1={}; '.format(LR,l1_w) + else: + algName += '-{}_me{}'.format(ar['n_iter_weights'] , ar['mini_epochs']) + codeLR = ar['lr_codes'] + l1_c = ar['lambda_c'] + muMult = ar['mult_mu'] + muMax = ar['max_mu'] + l1wFact = ar['lambda_w_muFact'] + l1cFact = ar['lambda_c_muFact'] + + TITLE2 = '\nWeight LR={},L1={}\mu; '.format(LR,l1wFact) + TITLE2 += 'Code LR={},L1={}\mu; '.format(codeLR,l1cFact) + TITLE2 += '\n\mu factor = {}; '.format(muMult) + TITLE2 += '\mu-max={}'.format(muMax) + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = '.', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '.', label='Testing') + plt.legend() + plt.ylim([.05,0.5]) + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+TITLE2) + plt.savefig(savePath + expID + comboID + 'allTrials.png') + + ## (3.) Finally, record the best results for each setting. + #(3.a) determine the final (nonzero) accuracy: + if meansTe[-1] > bestRez[expID]['acc']:# and len(meansTe)>9: + # For Plotting + bestRez[expID]['xax'] = x_axis + bestRez[expID]['meansTr'] = meansTr + bestRez[expID]['stdsTr'] = stdsTr + bestRez[expID]['meansTe'] = meansTe + bestRez[expID]['stdsTe'] = stdsTe + bestRez[expID]['numTr'] = numTrials + # For later. + bestRez[expID]['comboID'] = comboID + bestRez[expID]['acc'] = meansTe[-1] + bestRez[expID]['lr_weights'] = rez[0].args['lr_weights'] + bestRez[expID]['bsz'] = rez[0].args['batch_size'] + if expID.startswith('am'): + bestRez[expID]['lr_codes'] = rez[0].args['lr_codes'] + bestRez[expID]['d_mu'] = rez[0].args['d_mu'] + bestRez[expID]['muMult'] = rez[0].args['mult_mu'] + bestRez[expID]['muMax'] = rez[0].args['max_mu'] + + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. + +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', # Backprop-based methods + "amAdam1_1_me1":'r', + "amAdam5_1_me1":'y', + "amAdam1_5_me1":'m', + "amAdam5_5_me1":'c' + } +markers = {"sgd":'o', "adam":'o', + "amAdam1_1_me1":'*', + "amAdam5_1_me1":'d', + "amAdam1_5_me1":'P', + "amAdam5_5_me1":'x' + } +plotNames={"amAdam1_1_me1":'AM-Adam11', + "amAdam5_1_me1":'AM-Adam51', + "amAdam1_5_me1":'AM-Adam15', + "amAdam5_5_me1":'AM-Adam55', + 'sgd':'SGD', 'adam':"Adam" + } +#colors = ['b','g','r','c','m','y','k'] + +markerSize = 4 +modelName = rez[0].args['model'] +resUsableTitle = modelName + +dataName = rez[0].args['dataset'] +trmark='.' +temark='^' +numEpochs=10 +######## ALL PLOTS +for i, xLims in enumerate([[-0.1, numEpochs+0.1]]): + for k,yLims in enumerate([[0., 1], [0.6,0.93], [0.85,1]]): + plt.figure(i*k + k) + plt.clf() + colorID=0 + if yLims[0]>0 or yLims[1]<1 or xLims[1] numTrials/2, then more than half + # of the trials failed due to initialization randomness. + if allTr is None or (numFailedTrials > numTrials/2): + continue + # (1.) Else, compute mean and std devs. + # Update the number of trials actually used in this analysis. + numTrials = allTr.size(1) + # Only average the nonzero entries. + meansTr = (allTr.sum(1)/allTr.gt(0).sum(1).float()) + meansTe = (allTe.sum(1)/allTe.gt(0).sum(1).float()) + # Initialize with zero. + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + # If there were actually multiple trials, fill in using NONZERO entries only. + if numTrials>1: + for epoch, tnsr in enumerate(allTr): + # This is my stupid way of doing ONLY NONZERO std dev's... + stdtr = tnsr[ tnsr.gt(0) ].std().numpy() + if np.isnan(stdtr): + stdtr=np.array([0]) + stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() + if np.isnan(stdte): + stdte=np.array([0]) + # Finally... + stdsTr[epoch] = stdtr + stdsTe[epoch] = stdte + meansTr = meansTr.numpy() + meansTe = meansTe.numpy() + + + # (1.5) for plotting, we REMOVE the zeros added during mean-computation. + while np.isnan(meansTr[-1]) or np.isnan(meansTe[-1]) or meansTr[-1]<1e-6 or meansTe[-1]<1e-6: + meansTr = meansTr[:-1] + meansTe = meansTe[:-1] + stdsTr = stdsTr[:-1] + stdsTe = stdsTe[:-1] + + if epoch0_meanTr is not None: + meansTr = np.insert(meansTr, 0, epoch0_meanTr) + meansTe = np.insert(meansTe, 0, epoch0_meanTe) + stdsTr = np.insert(stdsTr, 0, epoch0_stdTr) + stdsTe = np.insert(stdsTe, 0, epoch0_stdTe) + x_axis = range(0, len(meansTr)) + else: + x_axis = range(1, len(meansTr)+1) + + allMeanTes[expID][comboID]=meansTe + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: +# if expID=='amAdam5_me1': + # (2.a) Extract information about this setting for titles etc. + ar = rez[0].args + # Model Size. + + # Alg name. + algName = ar['opt_method'] + modelName=ar['model'] + TITLE =modelName+' trained with '+algName + + # Print other hyperparameters + LR = ar['lr_weights'] + + if algName in ['sgd', 'adam']: + l1_w = ar['lambda_w'] + TITLE2 = '\nWeight LR={}, L1={}; '.format(LR,l1_w) + else: + algName += '-{}_me{}'.format(ar['n_iter_weights'] , ar['mini_epochs']) + codeLR = ar['lr_codes'] + l1_c = ar['lambda_c'] + muMult = ar['mult_mu'] + muMax = ar['max_mu'] + l1wFact = ar['lambda_w_muFact'] + l1cFact = ar['lambda_c_muFact'] + + TITLE2 = '\nWeight LR={},L1={}\mu; '.format(LR,l1wFact) + TITLE2 += 'Code LR={},L1={}\mu; '.format(codeLR,l1cFact) + TITLE2 += '\n\mu factor = {}; '.format(muMult) + TITLE2 += '\mu-max={}'.format(muMax) + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = '.', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '.', label='Testing') + plt.legend() + plt.ylim([.8,1]) + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+TITLE2) + plt.savefig(savePath + expID + comboID + 'allTrials.png') + + ## (3.) Finally, record the best results for each setting. + #(3.a) determine the final (nonzero) accuracy: + if meansTe[-1] > bestRez[expID]['acc']:# and len(meansTe)>9: + # For Plotting + bestRez[expID]['xax'] = x_axis + bestRez[expID]['meansTr'] = meansTr + bestRez[expID]['stdsTr'] = stdsTr + bestRez[expID]['meansTe'] = meansTe + bestRez[expID]['stdsTe'] = stdsTe + bestRez[expID]['numTr'] = numTrials + # For later. + bestRez[expID]['comboID'] = comboID + bestRez[expID]['acc'] = meansTe[-1] + bestRez[expID]['lr_weights'] = rez[0].args['lr_weights'] + bestRez[expID]['bsz'] = rez[0].args['batch_size'] + if expID.startswith('am'): + bestRez[expID]['lr_codes'] = rez[0].args['lr_codes'] + bestRez[expID]['d_mu'] = rez[0].args['d_mu'] + bestRez[expID]['muMult'] = rez[0].args['mult_mu'] + bestRez[expID]['muMax'] = rez[0].args['max_mu'] + + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. + +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', # Backprop-based methods + "amAdam1_1_me1":'r', + "amAdam5_1_me1":'y', + "amAdam1_5_me1":'m', + "amAdam5_5_me1":'c' + } +markers = {"sgd":'o', "adam":'o', + "amAdam1_1_me1":'*', + "amAdam5_1_me1":'d', + "amAdam1_5_me1":'P', + "amAdam5_5_me1":'x' + } +plotNames={"amAdam1_1_me1":'AM-Adam11', + "amAdam5_1_me1":'AM-Adam51', + "amAdam1_5_me1":'AM-Adam15', + "amAdam5_5_me1":'AM-Adam55', + 'sgd':'SGD', 'adam':"Adam" + } +#colors = ['b','g','r','c','m','y','k'] + +markerSize = 4 +modelName = rez[0].args['model'] +resUsableTitle = modelName + +dataName = rez[0].args['dataset'] +trmark='.' +temark='^' +numEpochs=10 +######## ALL PLOTS +for i, xLims in enumerate([[-0.1, numEpochs+0.1]]): + for k,yLims in enumerate([[0., 1], [0.6,0.93], [0.85,1], [0.75,1]]): + plt.figure(i*k + k) + plt.clf() + colorID=0 + if yLims[0]>0 or yLims[1]<1 or xLims[1] numTrials/2, then more than half + # of the trials failed due to initialization randomness. + if allTr is None or (numFailedTrials > numTrials/2): + continue + # (1.) Else, compute mean and std devs. + # Update the number of trials actually used in this analysis. + numTrials = allTr.size(1) + # Only average the nonzero entries. + meansTr = (allTr.sum(1)/allTr.gt(0).sum(1).float()) + meansTe = (allTe.sum(1)/allTe.gt(0).sum(1).float()) + # Initialize with zero. + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + # If there were actually multiple trials, fill in using NONZERO entries only. + if numTrials>1: + for epoch, tnsr in enumerate(allTr): + # This is my stupid way of doing ONLY NONZERO std dev's... + stdtr = tnsr[ tnsr.gt(0) ].std().numpy() + if np.isnan(stdtr): + stdtr=np.array([0]) + stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() + if np.isnan(stdte): + stdte=np.array([0]) + # Finally... + stdsTr[epoch] = stdtr + stdsTe[epoch] = stdte + meansTr = meansTr.numpy() + meansTe = meansTe.numpy() + + + # (1.5) for plotting, we REMOVE the zeros added during mean-computation. + while np.isnan(meansTr[-1]) or np.isnan(meansTe[-1]) or meansTr[-1]<1e-6 or meansTe[-1]<1e-6: + meansTr = meansTr[:-1] + meansTe = meansTe[:-1] + stdsTr = stdsTr[:-1] + stdsTe = stdsTe[:-1] + + if epoch0_meanTr is not None: + meansTr = np.insert(meansTr, 0, epoch0_meanTr) + meansTe = np.insert(meansTe, 0, epoch0_meanTe) + stdsTr = np.insert(stdsTr, 0, epoch0_stdTr) + stdsTe = np.insert(stdsTe, 0, epoch0_stdTe) + x_axis = range(0, len(meansTr)) + else: + x_axis = range(1, len(meansTr)+1) + + allMeanTes[expID][comboID]=meansTe + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: +# if expID=='amAdam5_me1': + # (2.a) Extract information about this setting for titles etc. + ar = rez[0].args + # Model Size. + hidSize = str(ar['rnnHidSize']) + + # Alg name. + algName = ar['opt_method'] + TITLE = 'RNN 1-'+hidSize +'; trained with '+algName + + # Print other hyperparameters + LR = ar['lr_weights'] + + if algName in ['sgd', 'adam']: + l1_w = ar['lambda_w'] + TITLE2 = '\nWeight LR={}, L1={}; '.format(LR,l1_w) + else: + algName += '-{}_me{}'.format(ar['n_iter_weights'] , ar['mini_epochs']) + codeLR = ar['lr_codes'] + l1_c = ar['lambda_c'] + muMult = ar['mult_mu'] + muMax = ar['max_mu'] + l1wFact = ar['lambda_w_muFact'] + l1cFact = ar['lambda_c_muFact'] + + TITLE2 = '\nWeight LR={},L1={}\mu; '.format(LR,l1wFact) + TITLE2 += 'Code LR={},L1={}\mu; '.format(codeLR,l1cFact) + TITLE2 += '\n\mu factor = {}; '.format(muMult) + TITLE2 += '\mu-max={}'.format(muMax) + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = '.', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '.', label='Testing') + plt.legend() + plt.ylim([.05,0.5]) + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+TITLE2) + plt.savefig(savePath + expID + comboID + 'allTrials.png') + + ## (3.) Finally, record the best results for each setting. + #(3.a) determine the final (nonzero) accuracy: + if meansTe[-1] > bestRez[expID]['acc']:# and len(meansTe)>9: + # For Plotting + bestRez[expID]['xax'] = x_axis + bestRez[expID]['meansTr'] = meansTr + bestRez[expID]['meansTe'] = meansTe + bestRez[expID]['stdsTr'] = stdsTr + bestRez[expID]['stdsTe'] = stdsTe + bestRez[expID]['numTr'] = numTrials + # For later. + bestRez[expID]['comboID'] = comboID + bestRez[expID]['acc'] = meansTe[-1] + bestRez[expID]['lr_weights'] = rez[0].args['lr_weights'] + if expID.startswith('am'): + bestRez[expID]['lr_codes'] = rez[0].args['lr_codes'] + bestRez[expID]['d_mu'] = rez[0].args['d_mu'] + bestRez[expID]['muMult'] = rez[0].args['mult_mu'] + bestRez[expID]['muMax'] = rez[0].args['max_mu'] + + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. +hidSize = str(100)#str(rez[0].args['rnnHidSize']) + +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', # Backprop-based methods + "amAdam5_me1":'r', # 5 subproblem iter, 1 miniepoch. + "amAdam5_me2":'r', # 5 subproblem iter, 2 miniepoch. + "amAdam5_me3":'r', # 5 subproblem iter, 3 miniepoch. + "amAdam1_me1":'y', # 1 subproblem iter, 1 miniepoch. + "amAdam1_me2":'y', # 1 subproblem iter, 2 miniepoch. + "amAdam1_me5":'y', # 1 subproblem iter, 5 miniepoch. + "amAdam20_me1":'m', # 20 subproblem iter (codes only), 1 miniepoch. + "amAdam15_me2":'m', # 1 subproblem iter (codes only), 2 miniepoch. + "amAdam15_me3":'m', # 15 subproblem iter (codes only), 3 miniepoch. + "amAdam15_me5":'m', # 15 subproblem iter (codes only), 5 miniepoch. + } +markers = {"sgd":'o', "adam":'o', # Backprop-based methods + "amAdam5_me1":'o', # 5 subproblem iter, 1 miniepoch. + "amAdam1_me1":'o', # 1 subproblem iter, 1 miniepoch. + "amAdam20_me1":'o', # 20 subproblem iter (codes only), 1 miniepoch. + "amAdam5_me2":'*', # 5 subproblem iter, 2 miniepoch. + "amAdam1_me2":'*', # 1 subproblem iter, 2 miniepoch. + "amAdam15_me2":'*', # 1 subproblem iter (codes only), 2 miniepoch. + "amAdam5_me3":'^', # 5 subproblem iter, 3 miniepoch. + "amAdam15_me3":'^', # 15 subproblem iter (codes only), 3 miniepoch. + "amAdam1_me5":'_', # 1 subproblem iter, 5 miniepoch. + "amAdam15_me5":'_', # 15 subproblem iter (codes only), 5 miniepoch. + } + +#colors = ['b','g','r','c','m','y','k'] + +markerSize = 4 +resUsableTitle = 'no-skip RNN: 1-'+hidSize +numEpochs=10 +######## ALL PLOTS +for i, xLims in enumerate([[-0.1, numEpochs+0.1]]): + for k,yLims in enumerate([[0.05, 0.5]]): + plt.figure(i*k + k) + plt.clf() + colorID=0 + if yLims[0]>0 or yLims[1]<1 or xLims[1]0 or yLims[1]<1 or xLims[1]0: + stdsR0 = allR0.std(1).numpy() + stdsTr = allTr.std(1).numpy() + stdsTe = allTe.std(1).numpy() + else: + stdsR0 = th.zeros(meansR0.size()).numpy() + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + + + ############### + # For epoch-0 results to be tacked on.. + x_axis = np.array(range(1, len(meansTr)+1)) +# meansTr = np.insert(meansTr, 0, meansR0[0]) +# meansTe = np.insert(meansTe, 0, meansR0[0]) +# stdsTr = np.insert(stdsTr, 0, stdsR0[0]) +# stdsTe = np.insert(stdsTe, 0, stdsR0[0]) +# x_axis = np.array(range(0, len(meansTr))) + + # Finally divide by the sqrt number of samples: + + stdsR0 /= np.sqrt(numTrials) + stdsTr /= np.sqrt(numTrials) + stdsTe /= np.sqrt(numTrials) + + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: + # (2.a) Extract information about this setting for titles etc. + # These should NOT change for rez[i], i=/=0. + LR = str(rez[0].args['lr_weights']) + # Data type (permuted vs sequential mnist) + if 'data_type' in rez[0].args: #This arg was added late so may not exist. + dataType = rez[0].args['data_type'] + else: + dataType = 'seq' + # Alg name. + algName = rez[0].args['opt_method'] + if algName=='altmin': + algName += str(rez[0].args['n_iter_weights']) + + TITLE = rez[0].args['model']+'; trained with '+expID + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = 'o', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '^', label='Testing') + plt.legend() + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE) + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'allEpoch.png') + + x_axis0 = np.array(range(0, len(meansR0))) + plt.figure(2) + plt.clf() + plt.errorbar(x_axis0, meansR0, stdsR0, marker = '.', label='Testing') + plt.xlabel('Minibatches') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+'\n(Epoch 0 Minibatches)') + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'minibatchE0.png') + + ## (3.) Finally, record the avg for a big plot at the end. + # For Plotting + avgRez[expID]['xax'] = x_axis + avgRez[expID]['xax0'] = x_axis0 + avgRez[expID]['meansTr'] = meansTr + avgRez[expID]['meansTe'] = meansTe + avgRez[expID]['meansR0'] = meansR0 + avgRez[expID]['stdsTr'] = stdsTr + avgRez[expID]['stdsTe'] = stdsTe + avgRez[expID]['stdsR0'] = stdsR0 + avgRez[expID]['numTr'] = numTrials + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. + +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', + "amAdam1_me1":'r' + } +markers = {"sgd":'o', "adam":'P', + "amAdam1_me1":'^' + } +names = {"sgd":'SGD', "adam":'Adam', + "amAdam1_me1":'AM-Adam'} +models={'lenet':'LeNet', 'vgg7':'VGG7'} +datasets={'mnist':'MNIST', 'fashion_mnist':'Fashion-MNIST'} +#colors = ['b','g','r','c','m','y','k'] +modelName = models[rez[0].args['model']] +datName=datasets[rez[0].args['dataset']] +resUsableTitle = datName+', '+modelName +xtraAx1 = 0.1 +xtraAx2 = 1 +trmark='.' +temark='^' +xtras={'adam':1.001, 'amAdam1_me1':1, 'sgd':0.999} + #mnistAll #fMnistAll +for k,lims in enumerate([[0.9, 0.99],[0.675,0.91]]): + plt.figure(1) + plt.clf() + # Plot all results. + for expID, rez in avgRez.items(): + if not ('meansTr' in rez): + continue +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals =rez['meansTe'] + plt.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals[-1], xy=(1,yvals[-1]*xtras[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') + # Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') + plt.xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) + plt.ylim(lims) + plt.xlabel('Epochs') + plt.ylabel('Accuracy') + plt.title(resUsableTitle) + plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + plt.savefig(savePath+'bestEpochs_19jan-'+modelName+datName+'lim'+str(k)+'.png', bbox_inches='tight') +########################################### +## Now do epoch-0. +xtras={'adam':1.02, 'amAdam1_me1':1, 'sgd':0.999} + #fmnist0 #mnist0 +for i,xlims in enumerate([None, [1,50]]): + for k,lims in enumerate([[0.1,0.85],[0.1,1]]): + plt.figure(2) + plt.clf() + + if xlims is None: + END=max(rez['xax0']) + else: + END=xlims[1] + # Plot all results. + for expID, rez in avgRez.items(): + if not ('meansR0' in rez): + continue + yvals = rez['meansR0'] + plt.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals[END], xy=(1,yvals[END]*xtras[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') + # Now plot a marker designating when TESTING phase began. + # plt.axvline(x=59.5, linewidth=2, label='Begin testing') + if xlims is None: + plt.xlim([min(rez['xax0'])-xtraAx2, END+xtraAx2]) + else: + plt.xlim(xlims) + plt.ylim(lims) + plt.xlabel('Minibatches') + plt.ylabel('Accuracy') + plt.title('Epoch 0 Test Accuracy\n' + + resUsableTitle) + plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + plt.savefig(savePath+'bestMiniBatches_19jan-'+modelName+datName+'lim'+str(i)+str(k)+'.png', bbox_inches='tight') + + + + + + + +###################################### +## Combined plots. + +xtras={'adam':1.001, 'amAdam1_me1':1, 'sgd':0.999} +if datName=='mnist': + ylim1 = [0.9, 0.99] + ylim2 = [0.1,1] +else: + ylim1 = [0.675,0.91] + ylim2 = [0.1,0.85] + +fig = plt.figure(23) +fig.clf() +ax1 = fig.add_subplot(111) + +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansTr' in rez): + continue +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals =rez['meansTe'] + ax1.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + ax1.annotate('%0.3f' % yvals[-1], xy=(1,yvals[-1]*xtras[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now set axis limits etc. +ax1.set_xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) +ax1.set_ylim(ylim1) +ax1.set_xlabel('Epochs') +ax1.set_ylabel('Accuracy') +ax1.set_title(resUsableTitle) +ax1.legend(loc='lower left')#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + +########################################### +## Now do epoch-0 inside the above axis... +xtras={'adam':1.02, 'amAdam1_me1':1, 'sgd':0.999} + +ax2=plt.axes([.45, .11, .45, .4]) + +for expID, rez in avgRez.items(): + if not ('meansR0' in rez): + continue + yvals = rez['meansR0'] + ax2.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + ax2.annotate('%0.3f' % yvals[END], xy=(1,yvals[END]*xtras[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +ax2.set_xlim([min(rez['xax0'])-xtraAx2, END+xtraAx2]) +ax2.set_xticks([]) +ax2.set_ylim(ylim2) +#ax2.set_xlabel('Minibatches') +ax2.annotate('Epoch 0 Minibatches', xy=(12,0.79)) +#ax2.ylabel('Accuracy') +#ax2.title('Epoch 0 Test Accuracy\n' + resUsableTitle) +#ax2.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + +plt.savefig('./TESTING.png', bbox_inches='tight') +plt.show() + + + + +######################### +# For computing nonzero std. devs. +# # If there were actually multiple trials, compute standard deviations +# # in accuracies using NONZERO entries only. +# if numTrials>1: +# for epoch, tnsr in enumerate(allTr): +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdtr = tnsr[ tnsr.gt(0) ].std().numpy() +# if np.isnan(stdtr): +# stdtr=np.array([0]) +# # Finally... +# stdsTr[epoch] = stdtr +# ############################## +# # Now do Testing. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() +# if np.isnan(stdte): +# stdte=np.array([0]) +# # Finally... +# stdsTe[epoch] = stdte +# ############################## +# # Finally, do Epoch-0 results.. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# for minibatch, tnsr in enumerate(allR0): +# stdE0 = allTe[minibatch][ allTe[minibatch].gt(0.1) ].std().numpy() +# if np.isnan(stdE0): +# stdE0=np.array([0]) +# # Finally... +# stdE0[epoch] = stdte + + + diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial10_20190119-16:23:16.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial10_20190119-16:23:16.pt new file mode 100644 index 0000000..2997332 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial10_20190119-16:23:16.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial11_20190119-16:47:27.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial11_20190119-16:47:27.pt new file mode 100644 index 0000000..e508c53 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial11_20190119-16:47:27.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial12_20190119-17:11:47.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial12_20190119-17:11:47.pt new file mode 100644 index 0000000..3a52f9f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial12_20190119-17:11:47.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial13_20190119-17:35:27.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial13_20190119-17:35:27.pt new file mode 100644 index 0000000..1bce1e0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial13_20190119-17:35:27.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial14_20190119-17:59:45.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial14_20190119-17:59:45.pt new file mode 100644 index 0000000..cfa2adc Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial14_20190119-17:59:45.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial15_20190119-18:23:43.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial15_20190119-18:23:43.pt new file mode 100644 index 0000000..198a320 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial15_20190119-18:23:43.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial16_20190119-18:47:47.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial16_20190119-18:47:47.pt new file mode 100644 index 0000000..363c944 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial16_20190119-18:47:47.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial17_20190119-19:11:56.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial17_20190119-19:11:56.pt new file mode 100644 index 0000000..901a3e3 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial17_20190119-19:11:56.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial18_20190119-19:36:30.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial18_20190119-19:36:30.pt new file mode 100644 index 0000000..189dcd7 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial18_20190119-19:36:30.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial19_20190119-20:00:35.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial19_20190119-20:00:35.pt new file mode 100644 index 0000000..510d15b Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial19_20190119-20:00:35.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:11:28.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:11:28.pt new file mode 100644 index 0000000..4b6cfb5 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:11:28.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:20:34.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:20:34.pt new file mode 100644 index 0000000..693bd76 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:20:34.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:32:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:32:02.pt new file mode 100644 index 0000000..8bfcdf8 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:32:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:45:01.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:45:01.pt new file mode 100644 index 0000000..699b087 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial1_20190119-12:45:01.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial20_20190119-20:25:00.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial20_20190119-20:25:00.pt new file mode 100644 index 0000000..64e5d9f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial20_20190119-20:25:00.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial21_20190119-20:49:29.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial21_20190119-20:49:29.pt new file mode 100644 index 0000000..46c9d47 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial21_20190119-20:49:29.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial22_20190119-21:13:54.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial22_20190119-21:13:54.pt new file mode 100644 index 0000000..5301f22 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial22_20190119-21:13:54.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial23_20190119-21:38:15.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial23_20190119-21:38:15.pt new file mode 100644 index 0000000..8fcdac0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial23_20190119-21:38:15.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial24_20190119-22:02:38.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial24_20190119-22:02:38.pt new file mode 100644 index 0000000..0680183 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial24_20190119-22:02:38.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial25_20190119-22:26:48.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial25_20190119-22:26:48.pt new file mode 100644 index 0000000..74a2895 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial25_20190119-22:26:48.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial26_20190119-22:51:12.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial26_20190119-22:51:12.pt new file mode 100644 index 0000000..986ff14 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial26_20190119-22:51:12.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial27_20190119-23:15:07.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial27_20190119-23:15:07.pt new file mode 100644 index 0000000..59b2b43 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial27_20190119-23:15:07.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial28_20190119-23:39:16.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial28_20190119-23:39:16.pt new file mode 100644 index 0000000..758203c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial28_20190119-23:39:16.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial29_20190120-00:03:34.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial29_20190120-00:03:34.pt new file mode 100644 index 0000000..21d0410 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial29_20190120-00:03:34.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial2_20190119-13:08:55.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial2_20190119-13:08:55.pt new file mode 100644 index 0000000..c7053ec Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial2_20190119-13:08:55.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial30_20190120-00:30:00.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial30_20190120-00:30:00.pt new file mode 100644 index 0000000..25e21f1 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial30_20190120-00:30:00.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial31_20190120-00:55:49.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial31_20190120-00:55:49.pt new file mode 100644 index 0000000..16b21e4 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial31_20190120-00:55:49.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial32_20190120-01:22:26.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial32_20190120-01:22:26.pt new file mode 100644 index 0000000..fa04d73 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial32_20190120-01:22:26.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial33_20190120-01:48:42.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial33_20190120-01:48:42.pt new file mode 100644 index 0000000..ee15a20 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial33_20190120-01:48:42.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial34_20190120-02:18:03.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial34_20190120-02:18:03.pt new file mode 100644 index 0000000..f937462 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial34_20190120-02:18:03.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial35_20190120-02:47:03.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial35_20190120-02:47:03.pt new file mode 100644 index 0000000..4a227cf Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial35_20190120-02:47:03.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial36_20190120-03:15:38.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial36_20190120-03:15:38.pt new file mode 100644 index 0000000..9d6fc96 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial36_20190120-03:15:38.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial37_20190120-03:44:33.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial37_20190120-03:44:33.pt new file mode 100644 index 0000000..b6e7e2c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial37_20190120-03:44:33.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial38_20190120-04:13:47.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial38_20190120-04:13:47.pt new file mode 100644 index 0000000..380b50a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial38_20190120-04:13:47.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial39_20190120-04:42:17.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial39_20190120-04:42:17.pt new file mode 100644 index 0000000..77dc391 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial39_20190120-04:42:17.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial3_20190119-13:33:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial3_20190119-13:33:02.pt new file mode 100644 index 0000000..917cf90 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial3_20190119-13:33:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial40_20190120-05:11:12.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial40_20190120-05:11:12.pt new file mode 100644 index 0000000..15458fc Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial40_20190120-05:11:12.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial41_20190120-05:40:26.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial41_20190120-05:40:26.pt new file mode 100644 index 0000000..dd2026d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial41_20190120-05:40:26.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial42_20190120-06:09:03.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial42_20190120-06:09:03.pt new file mode 100644 index 0000000..fadcb16 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial42_20190120-06:09:03.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial43_20190120-06:38:04.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial43_20190120-06:38:04.pt new file mode 100644 index 0000000..c89a0b5 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial43_20190120-06:38:04.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial44_20190120-07:07:06.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial44_20190120-07:07:06.pt new file mode 100644 index 0000000..8bcd4f1 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial44_20190120-07:07:06.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial45_20190120-07:35:25.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial45_20190120-07:35:25.pt new file mode 100644 index 0000000..bbea0bc Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial45_20190120-07:35:25.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial46_20190120-08:04:01.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial46_20190120-08:04:01.pt new file mode 100644 index 0000000..750d3bd Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial46_20190120-08:04:01.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial47_20190120-08:45:48.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial47_20190120-08:45:48.pt new file mode 100644 index 0000000..356caf3 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial47_20190120-08:45:48.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial48_20190120-09:25:45.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial48_20190120-09:25:45.pt new file mode 100644 index 0000000..22c98c7 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial48_20190120-09:25:45.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial49_20190120-10:05:36.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial49_20190120-10:05:36.pt new file mode 100644 index 0000000..c8376b0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial49_20190120-10:05:36.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial4_20190119-13:58:01.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial4_20190119-13:58:01.pt new file mode 100644 index 0000000..a8e11a3 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial4_20190119-13:58:01.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial5_20190119-14:22:52.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial5_20190119-14:22:52.pt new file mode 100644 index 0000000..525f701 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial5_20190119-14:22:52.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial6_20190119-14:46:54.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial6_20190119-14:46:54.pt new file mode 100644 index 0000000..a637aed Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial6_20190119-14:46:54.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial7_20190119-15:10:24.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial7_20190119-15:10:24.pt new file mode 100644 index 0000000..2635393 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial7_20190119-15:10:24.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial8_20190119-15:34:57.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial8_20190119-15:34:57.pt new file mode 100644 index 0000000..5b812e0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial8_20190119-15:34:57.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial9_20190119-15:58:59.pt b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial9_20190119-15:58:59.pt new file mode 100644 index 0000000..6c170a3 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_fashionMnist/testResults/sgd_trial9_20190119-15:58:59.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/adam.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/adam.pt new file mode 100644 index 0000000..a80b180 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/adam.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/amAdam1_me1.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/amAdam1_me1.pt new file mode 100644 index 0000000..8b8f893 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/amAdam1_me1.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/getTestRez.sh b/CNN_DEMO/postHPexps/lenet/lenet_mnist/getTestRez.sh new file mode 100755 index 0000000..399ebd8 --- /dev/null +++ b/CNN_DEMO/postHPexps/lenet/lenet_mnist/getTestRez.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +savePath='./testResults/' +gpuID=0 + +max_epochs=10 +numTrials=100 + +adamArgs='adam.pt' +sgdArgs='sgd.pt' +am11Args='amAdam1_me1.pt' + +for ((trial=1; trial<=numTrials; trial++)); do + # Get trial ID + trialNum=$trial #$((gpuID*numTrials + trial)) + trialID='trial'$trialNum + + saveName=$savePath'amAdam11_'$trialID + CUDA_VISIBLE_DEVICES=$gpuID python3 ../../train_cnn_14jan.py --seed=$trialNum --epochs=$max_epochs --loadBestHyperparameters=$am11Args --first-epoch-log --save-filename=$saveName + + # SGD + saveName=$savePath'sgd_'$trialID + CUDA_VISIBLE_DEVICES=$gpuID python3 ../../train_cnn_14jan.py --seed=$trialNum --epochs=$max_epochs --loadBestHyperparameters=$sgdArgs --first-epoch-log --save-filename=$saveName + + # Adam + saveName=$savePath'adam_'$trialID + CUDA_VISIBLE_DEVICES=$gpuID python3 ../../train_cnn_14jan.py --seed=$trialNum --epochs=$max_epochs --loadBestHyperparameters=$adamArgs --first-epoch-log --save-filename=$saveName +done + diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/sgd.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/sgd.pt new file mode 100644 index 0000000..3d8991d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/sgd.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial10_20190119-16:31:11.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial10_20190119-16:31:11.pt new file mode 100644 index 0000000..becbb34 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial10_20190119-16:31:11.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial11_20190119-16:55:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial11_20190119-16:55:10.pt new file mode 100644 index 0000000..829554c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial11_20190119-16:55:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial12_20190119-17:19:55.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial12_20190119-17:19:55.pt new file mode 100644 index 0000000..c2d0433 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial12_20190119-17:19:55.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial13_20190119-17:44:06.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial13_20190119-17:44:06.pt new file mode 100644 index 0000000..ce46483 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial13_20190119-17:44:06.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial14_20190119-18:08:40.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial14_20190119-18:08:40.pt new file mode 100644 index 0000000..81b75fb Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial14_20190119-18:08:40.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial15_20190119-18:32:54.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial15_20190119-18:32:54.pt new file mode 100644 index 0000000..18904c8 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial15_20190119-18:32:54.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial16_20190119-18:57:47.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial16_20190119-18:57:47.pt new file mode 100644 index 0000000..2f656e7 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial16_20190119-18:57:47.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial17_20190119-19:22:04.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial17_20190119-19:22:04.pt new file mode 100644 index 0000000..d35e4f6 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial17_20190119-19:22:04.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial18_20190119-19:46:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial18_20190119-19:46:10.pt new file mode 100644 index 0000000..bf8678b Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial18_20190119-19:46:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial19_20190119-20:10:35.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial19_20190119-20:10:35.pt new file mode 100644 index 0000000..094d4e6 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial19_20190119-20:10:35.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial1_20190119-12:52:39.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial1_20190119-12:52:39.pt new file mode 100644 index 0000000..842b0cf Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial1_20190119-12:52:39.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial20_20190119-20:34:43.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial20_20190119-20:34:43.pt new file mode 100644 index 0000000..cfacbe0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial20_20190119-20:34:43.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial21_20190119-20:59:08.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial21_20190119-20:59:08.pt new file mode 100644 index 0000000..13214de Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial21_20190119-20:59:08.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial22_20190119-21:22:55.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial22_20190119-21:22:55.pt new file mode 100644 index 0000000..f906799 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial22_20190119-21:22:55.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial23_20190119-21:47:14.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial23_20190119-21:47:14.pt new file mode 100644 index 0000000..57c16c2 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial23_20190119-21:47:14.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial24_20190119-22:11:46.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial24_20190119-22:11:46.pt new file mode 100644 index 0000000..459a7b0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial24_20190119-22:11:46.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial25_20190119-22:36:23.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial25_20190119-22:36:23.pt new file mode 100644 index 0000000..53ece30 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial25_20190119-22:36:23.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial26_20190119-23:01:09.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial26_20190119-23:01:09.pt new file mode 100644 index 0000000..9fa9132 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial26_20190119-23:01:09.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial27_20190119-23:25:30.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial27_20190119-23:25:30.pt new file mode 100644 index 0000000..1af85dc Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial27_20190119-23:25:30.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial28_20190119-23:49:51.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial28_20190119-23:49:51.pt new file mode 100644 index 0000000..75742b0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial28_20190119-23:49:51.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial29_20190120-00:14:03.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial29_20190120-00:14:03.pt new file mode 100644 index 0000000..7095d7a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial29_20190120-00:14:03.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial2_20190119-13:16:47.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial2_20190119-13:16:47.pt new file mode 100644 index 0000000..5a108be Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial2_20190119-13:16:47.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial30_20190120-00:40:15.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial30_20190120-00:40:15.pt new file mode 100644 index 0000000..cccf7e0 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial30_20190120-00:40:15.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial31_20190120-01:07:57.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial31_20190120-01:07:57.pt new file mode 100644 index 0000000..baea2c8 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial31_20190120-01:07:57.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial32_20190120-01:33:38.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial32_20190120-01:33:38.pt new file mode 100644 index 0000000..996bb8f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial32_20190120-01:33:38.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial33_20190120-01:59:18.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial33_20190120-01:59:18.pt new file mode 100644 index 0000000..3f552c1 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial33_20190120-01:59:18.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial34_20190120-02:31:14.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial34_20190120-02:31:14.pt new file mode 100644 index 0000000..feca757 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial34_20190120-02:31:14.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial35_20190120-03:00:17.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial35_20190120-03:00:17.pt new file mode 100644 index 0000000..afd3e2a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial35_20190120-03:00:17.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial36_20190120-03:29:14.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial36_20190120-03:29:14.pt new file mode 100644 index 0000000..de17164 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial36_20190120-03:29:14.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial37_20190120-03:58:08.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial37_20190120-03:58:08.pt new file mode 100644 index 0000000..7cad868 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial37_20190120-03:58:08.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial38_20190120-04:27:29.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial38_20190120-04:27:29.pt new file mode 100644 index 0000000..1abca93 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial38_20190120-04:27:29.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial39_20190120-04:56:21.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial39_20190120-04:56:21.pt new file mode 100644 index 0000000..e421dfd Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial39_20190120-04:56:21.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial3_20190119-13:41:17.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial3_20190119-13:41:17.pt new file mode 100644 index 0000000..73c8e2f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial3_20190119-13:41:17.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial40_20190120-05:26:04.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial40_20190120-05:26:04.pt new file mode 100644 index 0000000..7e51a29 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial40_20190120-05:26:04.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial41_20190120-05:54:45.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial41_20190120-05:54:45.pt new file mode 100644 index 0000000..0659908 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial41_20190120-05:54:45.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial42_20190120-06:23:55.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial42_20190120-06:23:55.pt new file mode 100644 index 0000000..9dc491f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial42_20190120-06:23:55.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial43_20190120-06:52:58.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial43_20190120-06:52:58.pt new file mode 100644 index 0000000..4af24ba Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial43_20190120-06:52:58.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial44_20190120-07:21:31.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial44_20190120-07:21:31.pt new file mode 100644 index 0000000..115a942 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial44_20190120-07:21:31.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial45_20190120-07:50:22.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial45_20190120-07:50:22.pt new file mode 100644 index 0000000..b855ee9 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial45_20190120-07:50:22.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial46_20190120-08:25:27.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial46_20190120-08:25:27.pt new file mode 100644 index 0000000..4747bac Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial46_20190120-08:25:27.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial47_20190120-09:06:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial47_20190120-09:06:02.pt new file mode 100644 index 0000000..39c5577 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial47_20190120-09:06:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial48_20190120-09:45:29.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial48_20190120-09:45:29.pt new file mode 100644 index 0000000..b364344 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial48_20190120-09:45:29.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial4_20190119-14:05:20.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial4_20190119-14:05:20.pt new file mode 100644 index 0000000..8c1c6ce Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial4_20190119-14:05:20.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial5_20190119-14:29:27.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial5_20190119-14:29:27.pt new file mode 100644 index 0000000..7dc3706 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial5_20190119-14:29:27.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial6_20190119-14:54:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial6_20190119-14:54:10.pt new file mode 100644 index 0000000..3580d00 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial6_20190119-14:54:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial7_20190119-15:18:28.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial7_20190119-15:18:28.pt new file mode 100644 index 0000000..0cbe5e5 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial7_20190119-15:18:28.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial8_20190119-15:42:33.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial8_20190119-15:42:33.pt new file mode 100644 index 0000000..fc8db80 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial8_20190119-15:42:33.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial9_20190119-16:06:53.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial9_20190119-16:06:53.pt new file mode 100644 index 0000000..7ef73e4 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/adam_trial9_20190119-16:06:53.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial10_20190119-16:15:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial10_20190119-16:15:02.pt new file mode 100644 index 0000000..3dcf26f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial10_20190119-16:15:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial11_20190119-16:38:56.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial11_20190119-16:38:56.pt new file mode 100644 index 0000000..cae7b43 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial11_20190119-16:38:56.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial12_20190119-17:03:32.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial12_20190119-17:03:32.pt new file mode 100644 index 0000000..9a8ecfb Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial12_20190119-17:03:32.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial13_20190119-17:27:57.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial13_20190119-17:27:57.pt new file mode 100644 index 0000000..3863579 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial13_20190119-17:27:57.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial14_20190119-17:52:06.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial14_20190119-17:52:06.pt new file mode 100644 index 0000000..7270a02 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial14_20190119-17:52:06.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial15_20190119-18:16:33.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial15_20190119-18:16:33.pt new file mode 100644 index 0000000..765a377 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial15_20190119-18:16:33.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial16_20190119-18:41:06.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial16_20190119-18:41:06.pt new file mode 100644 index 0000000..e7e18e4 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial16_20190119-18:41:06.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial17_20190119-19:05:49.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial17_20190119-19:05:49.pt new file mode 100644 index 0000000..cd446c4 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial17_20190119-19:05:49.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial18_20190119-19:29:50.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial18_20190119-19:29:50.pt new file mode 100644 index 0000000..bff0665 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial18_20190119-19:29:50.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial19_20190119-19:54:03.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial19_20190119-19:54:03.pt new file mode 100644 index 0000000..d050369 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial19_20190119-19:54:03.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial1_20190119-12:35:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial1_20190119-12:35:10.pt new file mode 100644 index 0000000..5113d80 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial1_20190119-12:35:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial1_20190119-12:36:09.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial1_20190119-12:36:09.pt new file mode 100644 index 0000000..f40b977 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial1_20190119-12:36:09.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial20_20190119-20:18:20.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial20_20190119-20:18:20.pt new file mode 100644 index 0000000..008fe15 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial20_20190119-20:18:20.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial21_20190119-20:42:40.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial21_20190119-20:42:40.pt new file mode 100644 index 0000000..57577a6 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial21_20190119-20:42:40.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial22_20190119-21:07:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial22_20190119-21:07:02.pt new file mode 100644 index 0000000..bb15a53 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial22_20190119-21:07:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial23_20190119-21:30:53.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial23_20190119-21:30:53.pt new file mode 100644 index 0000000..1717b50 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial23_20190119-21:30:53.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial24_20190119-21:55:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial24_20190119-21:55:10.pt new file mode 100644 index 0000000..e4e9a45 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial24_20190119-21:55:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial25_20190119-22:19:49.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial25_20190119-22:19:49.pt new file mode 100644 index 0000000..7a84415 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial25_20190119-22:19:49.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial26_20190119-22:44:33.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial26_20190119-22:44:33.pt new file mode 100644 index 0000000..87bf35e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial26_20190119-22:44:33.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial27_20190119-23:09:14.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial27_20190119-23:09:14.pt new file mode 100644 index 0000000..2c78d5f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial27_20190119-23:09:14.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial28_20190119-23:33:36.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial28_20190119-23:33:36.pt new file mode 100644 index 0000000..769def1 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial28_20190119-23:33:36.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial29_20190119-23:57:43.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial29_20190119-23:57:43.pt new file mode 100644 index 0000000..3227f37 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial29_20190119-23:57:43.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial2_20190119-13:00:43.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial2_20190119-13:00:43.pt new file mode 100644 index 0000000..1ded44e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial2_20190119-13:00:43.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial30_20190120-00:23:23.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial30_20190120-00:23:23.pt new file mode 100644 index 0000000..e111275 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial30_20190120-00:23:23.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial31_20190120-00:49:13.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial31_20190120-00:49:13.pt new file mode 100644 index 0000000..1eb921c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial31_20190120-00:49:13.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial32_20190120-01:16:25.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial32_20190120-01:16:25.pt new file mode 100644 index 0000000..79f2639 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial32_20190120-01:16:25.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial33_20190120-01:42:11.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial33_20190120-01:42:11.pt new file mode 100644 index 0000000..2f81f00 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial33_20190120-01:42:11.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial34_20190120-02:11:07.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial34_20190120-02:11:07.pt new file mode 100644 index 0000000..b9d2c26 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial34_20190120-02:11:07.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial35_20190120-02:40:59.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial35_20190120-02:40:59.pt new file mode 100644 index 0000000..5de2c31 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial35_20190120-02:40:59.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial36_20190120-03:09:57.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial36_20190120-03:09:57.pt new file mode 100644 index 0000000..185f769 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial36_20190120-03:09:57.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial37_20190120-03:38:50.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial37_20190120-03:38:50.pt new file mode 100644 index 0000000..841ee90 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial37_20190120-03:38:50.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial38_20190120-04:08:00.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial38_20190120-04:08:00.pt new file mode 100644 index 0000000..a96a060 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial38_20190120-04:08:00.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial39_20190120-04:37:12.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial39_20190120-04:37:12.pt new file mode 100644 index 0000000..e727f3b Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial39_20190120-04:37:12.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial3_20190119-13:25:08.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial3_20190119-13:25:08.pt new file mode 100644 index 0000000..891613e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial3_20190119-13:25:08.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial40_20190120-05:06:00.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial40_20190120-05:06:00.pt new file mode 100644 index 0000000..28c5328 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial40_20190120-05:06:00.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial41_20190120-05:35:25.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial41_20190120-05:35:25.pt new file mode 100644 index 0000000..8463581 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial41_20190120-05:35:25.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial42_20190120-06:04:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial42_20190120-06:04:10.pt new file mode 100644 index 0000000..b7ed56e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial42_20190120-06:04:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial43_20190120-06:33:43.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial43_20190120-06:33:43.pt new file mode 100644 index 0000000..8d451ca Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial43_20190120-06:33:43.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial44_20190120-07:02:28.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial44_20190120-07:02:28.pt new file mode 100644 index 0000000..8f249de Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial44_20190120-07:02:28.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial45_20190120-07:31:10.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial45_20190120-07:31:10.pt new file mode 100644 index 0000000..8aa9b12 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial45_20190120-07:31:10.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial46_20190120-08:00:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial46_20190120-08:00:02.pt new file mode 100644 index 0000000..5717b8d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial46_20190120-08:00:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial47_20190120-08:40:07.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial47_20190120-08:40:07.pt new file mode 100644 index 0000000..a201546 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial47_20190120-08:40:07.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial48_20190120-09:19:32.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial48_20190120-09:19:32.pt new file mode 100644 index 0000000..381b095 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial48_20190120-09:19:32.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial49_20190120-09:58:56.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial49_20190120-09:58:56.pt new file mode 100644 index 0000000..178ea48 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial49_20190120-09:58:56.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial4_20190119-13:49:26.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial4_20190119-13:49:26.pt new file mode 100644 index 0000000..8de4b54 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial4_20190119-13:49:26.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial5_20190119-14:13:13.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial5_20190119-14:13:13.pt new file mode 100644 index 0000000..519a24e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial5_20190119-14:13:13.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial6_20190119-14:37:32.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial6_20190119-14:37:32.pt new file mode 100644 index 0000000..a0f4c34 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial6_20190119-14:37:32.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial7_20190119-15:02:05.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial7_20190119-15:02:05.pt new file mode 100644 index 0000000..4d0a681 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial7_20190119-15:02:05.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial8_20190119-15:26:11.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial8_20190119-15:26:11.pt new file mode 100644 index 0000000..49edff5 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial8_20190119-15:26:11.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial9_20190119-15:50:29.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial9_20190119-15:50:29.pt new file mode 100644 index 0000000..7439ffc Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/amAdam11_trial9_20190119-15:50:29.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/adamallEpoch.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/adamallEpoch.png new file mode 100644 index 0000000..911de96 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/adamallEpoch.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/adamminibatchE0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/adamminibatchE0.png new file mode 100644 index 0000000..3631e5c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/adamminibatchE0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/amAdam1_me1allEpoch.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/amAdam1_me1allEpoch.png new file mode 100644 index 0000000..5d13bc1 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/amAdam1_me1allEpoch.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/amAdam1_me1minibatchE0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/amAdam1_me1minibatchE0.png new file mode 100644 index 0000000..83685aa Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/amAdam1_me1minibatchE0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_19jan-LeNetMNISTlim0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_19jan-LeNetMNISTlim0.png new file mode 100644 index 0000000..9cf768d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_19jan-LeNetMNISTlim0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_19jan-LeNetMNISTlim1.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_19jan-LeNetMNISTlim1.png new file mode 100644 index 0000000..7db2733 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_19jan-LeNetMNISTlim1.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_22jan-LeNetMNIST.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_22jan-LeNetMNIST.png new file mode 100644 index 0000000..a840faf Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestEpochs_22jan-LeNetMNIST.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim00.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim00.png new file mode 100644 index 0000000..7430893 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim00.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim01.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim01.png new file mode 100644 index 0000000..6272592 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim01.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim10.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim10.png new file mode 100644 index 0000000..eed36ae Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim10.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim11.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim11.png new file mode 100644 index 0000000..e430aac Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_19jan-LeNetMNISTlim11.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_22jan-LeNetMNIST.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_22jan-LeNetMNIST.png new file mode 100644 index 0000000..ca16d89 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/bestMiniBatches_22jan-LeNetMNIST.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/combinedMNIST_LeNet.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/combinedMNIST_LeNet.png new file mode 100644 index 0000000..18b1418 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/combinedMNIST_LeNet.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/sgdallEpoch.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/sgdallEpoch.png new file mode 100644 index 0000000..abdc95c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/sgdallEpoch.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/sgdminibatchE0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/sgdminibatchE0.png new file mode 100644 index 0000000..b117c95 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/sgdminibatchE0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/adamallEpoch.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/adamallEpoch.png new file mode 100644 index 0000000..ce03986 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/adamallEpoch.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/adamminibatchE0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/adamminibatchE0.png new file mode 100644 index 0000000..7d6007f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/adamminibatchE0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/amAdam1_me1allEpoch.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/amAdam1_me1allEpoch.png new file mode 100644 index 0000000..c1fab2b Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/amAdam1_me1allEpoch.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/amAdam1_me1minibatchE0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/amAdam1_me1minibatchE0.png new file mode 100644 index 0000000..98748ef Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/amAdam1_me1minibatchE0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestEpochs_19jan-LeNetMNISTlim0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestEpochs_19jan-LeNetMNISTlim0.png new file mode 100644 index 0000000..5625456 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestEpochs_19jan-LeNetMNISTlim0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestEpochs_19jan-LeNetMNISTlim1.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestEpochs_19jan-LeNetMNISTlim1.png new file mode 100644 index 0000000..7d80cce Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestEpochs_19jan-LeNetMNISTlim1.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestMiniBatches_19jan-LeNetMNISTlim0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestMiniBatches_19jan-LeNetMNISTlim0.png new file mode 100644 index 0000000..7ecbf99 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestMiniBatches_19jan-LeNetMNISTlim0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestMiniBatches_19jan-LeNetMNISTlim1.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestMiniBatches_19jan-LeNetMNISTlim1.png new file mode 100644 index 0000000..7250a37 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/bestMiniBatches_19jan-LeNetMNISTlim1.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/sgdallEpoch.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/sgdallEpoch.png new file mode 100644 index 0000000..98e3432 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/sgdallEpoch.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/sgdminibatchE0.png b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/sgdminibatchE0.png new file mode 100644 index 0000000..e0900fc Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/sgdminibatchE0.png differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/testRezViz_19jan.py b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/testRezViz_19jan.py new file mode 100644 index 0000000..3b7da4c --- /dev/null +++ b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/stdDevErrBar/testRezViz_19jan.py @@ -0,0 +1,391 @@ +""" +Creates visualizations for RNN test results. + +@author Benjamin Cowen +@date 19 Jan 2019 +""" + +############################################ +# Imports ---------------------------------- + +# Loading +from utils import ddict, load_dataset +import glob +import torch as th + +# Plotting and Saving +# The following two lines are for the remote branch. +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import os +from shutil import copyfile +import numpy as np +import argparse + + +p = argparse.ArgumentParser(description='Visualize hyperparameter grid search results.') +p.add_argument('--loadPaths', type=str, default=None, metavar='path1/a/, path2/b/', + help='A list of strings/paths to results.') +p.add_argument('--loadPath', type=str, default=None, metavar='path/to/data/', + help='Path to directory containing data to plot.') +p.add_argument('--savePath', type=str, default=None, metavar='path/to/savedIms/', + help='Path to directory containing images saved here.') +p.add_argument('--onlyBest', type=bool, default=False, metavar='True', + help='Set to true if you DONT want all the imgs (only best params).') +p.add_argument('--reduced', type=bool, default=False, metavar='True', + help='Set to true if you only want SGD, Adam, and AltMin-Adam.') +p.add_argument('--epoch0-accs', type=str, default=None, metavar='path/to/epoch-0-accuracies/', + help='Path to directory containing the epoch-0 accuracies... (avg result for all methods)') +args = p.parse_args() + +# Check arguments. +# Assumes loadPaths or loadPath is valid... +if args.loadPaths is None: + # Load and Save Paths + loadPaths = [args.loadPath] +else: + loadPaths = args.loadPaths.strip('[]').split(',') + +# For saving images +if args.savePath is None: + savePath = loadPaths[0] + "ims/" + if not os.path.exists(savePath): + os.mkdir(savePath) +print('*********************************************************') +print('Saving images to '+savePath) +print('*********************************************************') + +############################################ +# Make a copy of this file in the savePath! +here = os.path.realpath(__file__) +there = savePath + (__file__).replace('../','') +copyfile(here,there) + +############################################ +# Decide tree structure here... (TODO way to automate this?) +# Stands for "all shelves" +allSh = {"amAdam1_me1":[], + "sgd":[], "adam":[] + } + + +# Loop thru given loadpaths. +for k,loadPath in enumerate(loadPaths): + print('Loading files from: ' + loadPath) + for filepath in glob.iglob(loadPath + '*.pt'): + fileName = filepath.replace(loadPath,'') + + # Go ahead and load it. + DD = ddict()._load(filepath) + algBase = DD.args['opt_method'] + + # If SGD, check whether it's Nesterov. + if algBase=='sgd': + expID = 'sgd' + elif algBase=='adam': + expID='adam' + elif algBase=='altmin': + nit = str(DD.args['n_iter_codes']) + me = str(DD.args['mini_epochs']) + expID='amAdam' + nit + '_me' + me + else: + raise Exception("Unrecognized optimization method "+algBase+" (only 'altmin','sgd','adam')") + + # Append results. + if args.reduced: + if not (expID=='sgd' or expID=='adam' or expID=='amAdam5_me1'): + continue + allSh[expID].append(ddict()._load(filepath)) + + +# Compute the average accuracies for epoch 0 (ie, using initializations only) +epoch0_meanTr = None +if args.epoch0_accs is not None: + epoch0_meanTr = [] + epoch0_meanTe = [] + # Compute means and std devs of the initializations. + for filepath in glob.iglob(args.epoch0_accs + '*.pt'): + DD = ddict()._load(filepath) + epoch0_meanTr += DD.perf['tr'] + epoch0_meanTe += DD.perf['te'] + + # Finally average. + epoch0_stdTr = th.Tensor( epoch0_meanTr ).std() + epoch0_meanTr = th.Tensor( epoch0_meanTr ).mean() + epoch0_stdTe = th.Tensor( epoch0_meanTe ).std() + epoch0_meanTe = th.Tensor( epoch0_meanTe ).mean() + + +# This better be the same for all DD anyway :) +############################################ +# Print all single-exp. figures ----------------------- +avgRez = {} +numE0btch = 469 # Number of elements in 'first_epoch' +numEpochs = 10 # Number of elements in 'te' and 'tr' + +for expID, rez in allSh.items(): + if len(rez)==0: + continue + # Initialize each Exp ID. + avgRez[expID] = {'tr':[], 'te':[]} + numTrials = len(rez) + + # (0.) Collect the Training/Testing histories. + # Compute the mean and std dev across trials. + allR0 = None + for trial in range(numTrials): + # Shelf for current trial. + tShelf = rez[trial] + + # CHECK VALIDITY:----------------- + # If no 'perf' field, then performance wasn't yet recorded. + if hasattr(tShelf,'perf'): + expData = tShelf.perf + r0 = expData['first_epoch'] + rTr = expData['tr'] + rTe = expData['te'] + else: + continue + + # If the number of recorded epochs << number allowed epochs, it's in-progress. + # If the mean training accurac < 14%, consider this a FAILED attempt at training. + if len(rTe)0: + stdsR0 = allR0.std(1).numpy() + stdsTr = allTr.std(1).numpy() + stdsTe = allTe.std(1).numpy() + else: + stdsR0 = th.zeros(meansR0.size()).numpy() + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + + + ############### + # For epoch-0 results to be tacked on.. + x_axis = np.array(range(1, len(meansTr)+1)) +# meansTr = np.insert(meansTr, 0, meansR0[0]) +# meansTe = np.insert(meansTe, 0, meansR0[0]) +# stdsTr = np.insert(stdsTr, 0, stdsR0[0]) +# stdsTe = np.insert(stdsTe, 0, stdsR0[0]) +# x_axis = np.array(range(0, len(meansTr))) + + # Finally divide by the sqrt number of samples: + +# stdsR0 /= np.sqrt(numTrials) +# stdsTr /= np.sqrt(numTrials) +# stdsTe /= np.sqrt(numTrials) + + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: + # (2.a) Extract information about this setting for titles etc. + # These should NOT change for rez[i], i=/=0. + LR = str(rez[0].args['lr_weights']) + # Data type (permuted vs sequential mnist) + if 'data_type' in rez[0].args: #This arg was added late so may not exist. + dataType = rez[0].args['data_type'] + else: + dataType = 'seq' + # Alg name. + algName = rez[0].args['opt_method'] + if algName=='altmin': + algName += str(rez[0].args['n_iter_weights']) + + TITLE = rez[0].args['model']+'; trained with '+expID + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = 'o', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '^', label='Testing') + plt.legend() + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE) + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'allEpoch.png') + + x_axis0 = np.array(range(0, len(meansR0))) + plt.figure(2) + plt.clf() + plt.errorbar(x_axis0, meansR0, stdsR0, marker = '.', label='Testing') + plt.xlabel('Minibatches') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+'\n(Epoch 0 Minibatches)') + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'minibatchE0.png') + + ## (3.) Finally, record the avg for a big plot at the end. + # For Plotting + avgRez[expID]['xax'] = x_axis + avgRez[expID]['xax0'] = x_axis0 + avgRez[expID]['meansTr'] = meansTr + avgRez[expID]['meansTe'] = meansTe + avgRez[expID]['meansR0'] = meansR0 + avgRez[expID]['stdsTr'] = stdsTr + avgRez[expID]['stdsTe'] = stdsTe + avgRez[expID]['stdsR0'] = stdsR0 + avgRez[expID]['numTr'] = numTrials + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. + +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', + "amAdam1_me1":'r' + } +markers = {"sgd":'o', "adam":'P', + "amAdam1_me1":'^' + } +names = {"sgd":'SGD', "adam":'Adam', + "amAdam1_me1":'AM-Adam'} +models={'lenet':'LeNet', 'vgg7':'VGG7'} +datasets={'mnist':'MNIST', 'fashion_mnist':'Fashion-MNIST'} +#colors = ['b','g','r','c','m','y','k'] +modelName = models[rez[0].args['model']] +datName=datasets[rez[0].args['dataset']] +resUsableTitle = modelName + ' trained on '+datName +xtraAx1 = 0.1 +xtraAx2 = 1 +trmark='.' +temark='^' +xtras={'adam':1.001, 'amAdam1_me1':1, 'sgd':0.999} + #mnistAll #fMnistAll +for k,lims in enumerate([[0.9, 0.99],[0.675,0.91]]): + plt.figure(1) + plt.clf() + # Plot all results. + for expID, rez in avgRez.items(): + if not ('meansTr' in rez): + continue + if lims[0]>.7: + mmm=' (zoomed)' + else: + mmm='' +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals =rez['meansTe'] + plt.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals.max(), xy=(1,yvals.max()*xtras[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') + # Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') + plt.xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) + plt.ylim(lims) + plt.xlabel('Epochs') + plt.ylabel('Accuracy') + plt.title('Test Accuracy \n' + + resUsableTitle+mmm) + plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + plt.savefig(savePath+'bestEpochs_19jan-'+modelName+datName+'lim'+str(k)+'.png', bbox_inches='tight') +########################################### +## Now do epoch-0. +xtras={'adam':1.02, 'amAdam1_me1':1, 'sgd':0.999} + #fmnist0 #mnist0 +for k,lims in enumerate([[0.1,0.85],[0.1,1]]): + plt.figure(2) + plt.clf() + # Plot all results. + for expID, rez in avgRez.items(): + if not ('meansR0' in rez): + continue + yvals = rez['meansR0'] + plt.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals.max(), xy=(1,yvals.max()*xtras[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') + # Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') + plt.xlim([min(rez['xax0'])-xtraAx2, max(rez['xax0'])+xtraAx2]) + plt.ylim(lims) + plt.xlabel('Minibatches') + plt.ylabel('Accuracy') + plt.title('Epoch 0 Test Accuracy\n' + + resUsableTitle) + plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + plt.savefig(savePath+'bestMiniBatches_19jan-'+modelName+datName+'lim'+str(k)+'.png', bbox_inches='tight') + + + + + + + +######################### +# For computing nonzero std. devs. +# # If there were actually multiple trials, compute standard deviations +# # in accuracies using NONZERO entries only. +# if numTrials>1: +# for epoch, tnsr in enumerate(allTr): +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdtr = tnsr[ tnsr.gt(0) ].std().numpy() +# if np.isnan(stdtr): +# stdtr=np.array([0]) +# # Finally... +# stdsTr[epoch] = stdtr +# ############################## +# # Now do Testing. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() +# if np.isnan(stdte): +# stdte=np.array([0]) +# # Finally... +# stdsTe[epoch] = stdte +# ############################## +# # Finally, do Epoch-0 results.. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# for minibatch, tnsr in enumerate(allR0): +# stdE0 = allTe[minibatch][ allTe[minibatch].gt(0.1) ].std().numpy() +# if np.isnan(stdE0): +# stdE0=np.array([0]) +# # Finally... +# stdE0[epoch] = stdte + + + diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/testRezViz_19jan.py b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/testRezViz_19jan.py new file mode 100644 index 0000000..3b5cda0 --- /dev/null +++ b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/ims/testRezViz_19jan.py @@ -0,0 +1,507 @@ +""" +Creates visualizations for RNN test results. + +@author Benjamin Cowen +@date 19 Jan 2019 +""" + +############################################ +# Imports ---------------------------------- + +# Loading +from utils import ddict, load_dataset +import glob +import torch as th + +# Plotting and Saving +# The following two lines are for the remote branch. +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import os +from shutil import copyfile +import numpy as np +import argparse + + +p = argparse.ArgumentParser(description='Visualize hyperparameter grid search results.') +p.add_argument('--loadPaths', type=str, default=None, metavar='path1/a/, path2/b/', + help='A list of strings/paths to results.') +p.add_argument('--loadPath', type=str, default=None, metavar='path/to/data/', + help='Path to directory containing data to plot.') +p.add_argument('--savePath', type=str, default=None, metavar='path/to/savedIms/', + help='Path to directory containing images saved here.') +p.add_argument('--onlyBest', type=bool, default=False, metavar='True', + help='Set to true if you DONT want all the imgs (only best params).') +p.add_argument('--reduced', type=bool, default=False, metavar='True', + help='Set to true if you only want SGD, Adam, and AltMin-Adam.') +p.add_argument('--epoch0-accs', type=str, default=None, metavar='path/to/epoch-0-accuracies/', + help='Path to directory containing the epoch-0 accuracies... (avg result for all methods)') +args = p.parse_args() + +# Check arguments. +# Assumes loadPaths or loadPath is valid... +if args.loadPaths is None: + # Load and Save Paths + loadPaths = [args.loadPath] +else: + loadPaths = args.loadPaths.strip('[]').split(',') + +# For saving images +if args.savePath is None: + savePath = loadPaths[0] + "ims/" + if not os.path.exists(savePath): + os.mkdir(savePath) +print('*********************************************************') +print('Saving images to '+savePath) +print('*********************************************************') + +############################################ +# Make a copy of this file in the savePath! +here = os.path.realpath(__file__) +there = savePath + (__file__).replace('../','') +copyfile(here,there) + +############################################ +# Decide tree structure here... (TODO way to automate this?) +# Stands for "all shelves" +allSh = {"amAdam1_me1":[], + "amAdam5_me1":[], + "sgd":[], "adam":[] + } + + +# Loop thru given loadpaths. +for k,loadPath in enumerate(loadPaths): + print('Loading files from: ' + loadPath) + for filepath in glob.iglob(loadPath + '*.pt'): + fileName = filepath.replace(loadPath,'') + + # Go ahead and load it. + DD = ddict()._load(filepath) + algBase = DD.args['opt_method'] + + # If SGD, check whether it's Nesterov. + if algBase=='sgd': + expID = 'sgd' + elif algBase=='adam': + expID='adam' + elif algBase=='altmin': + nit = str(DD.args['n_iter_codes']) + me = str(DD.args['mini_epochs']) + expID='amAdam' + nit + '_me' + me + else: + raise Exception("Unrecognized optimization method "+algBase+" (only 'altmin','sgd','adam')") + + # Append results. + if args.reduced: + if not (expID=='sgd' or expID=='adam' or expID=='amAdam5_me1'): + continue + allSh[expID].append(ddict()._load(filepath)) + + +# Compute the average accuracies for epoch 0 (ie, using initializations only) +epoch0_meanTr = None +if args.epoch0_accs is not None: + epoch0_meanTr = [] + epoch0_meanTe = [] + # Compute means and std devs of the initializations. + for filepath in glob.iglob(args.epoch0_accs + '*.pt'): + DD = ddict()._load(filepath) + epoch0_meanTr += DD.perf['tr'] + epoch0_meanTe += DD.perf['te'] + + # Finally average. + epoch0_stdTr = th.Tensor( epoch0_meanTr ).std() + epoch0_meanTr = th.Tensor( epoch0_meanTr ).mean() + epoch0_stdTe = th.Tensor( epoch0_meanTe ).std() + epoch0_meanTe = th.Tensor( epoch0_meanTe ).mean() + + +# This better be the same for all DD anyway :) +############################################ +# Print all single-exp. figures ----------------------- +avgRez = {} +numE0btch = 469 #59#300 # Number of elements in 'first_epoch' +numEpochs = 10 # Number of elements in 'te' and 'tr' + +for expID, rez in allSh.items(): + if len(rez)==0: + continue + # Initialize each Exp ID. + avgRez[expID] = {'tr':[], 'te':[]} + numTrials = len(rez) + + # (0.) Collect the Training/Testing histories. + # Compute the mean and std dev across trials. + allR0 = None + for trial in range(numTrials): + # Shelf for current trial. + tShelf = rez[trial] + + # CHECK VALIDITY:----------------- + # If no 'perf' field, then performance wasn't yet recorded. + if hasattr(tShelf,'perf'): + expData = tShelf.perf + r0 = expData['first_epoch'] + rTr = expData['tr'] + rTe = expData['te'] + else: + continue + + # If the number of recorded epochs << number allowed epochs, it's in-progress. + # If the mean training accurac < 14%, consider this a FAILED attempt at training. + if len(rTe)0: + stdsR0 = allR0.std(1).numpy() + stdsTr = allTr.std(1).numpy() + stdsTe = allTe.std(1).numpy() + else: + stdsR0 = th.zeros(meansR0.size()).numpy() + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + + + ############### + # For epoch-0 results to be tacked on.. + x_axis = np.array(range(1, len(meansTr)+1)) +# meansTr = np.insert(meansTr, 0, meansR0[0]) +# meansTe = np.insert(meansTe, 0, meansR0[0]) +# stdsTr = np.insert(stdsTr, 0, stdsR0[0]) +# stdsTe = np.insert(stdsTe, 0, stdsR0[0]) +# x_axis = np.array(range(0, len(meansTr))) + + # Finally divide by the sqrt number of samples: + + stdsR0 /= np.sqrt(numTrials) + stdsTr /= np.sqrt(numTrials) + stdsTe /= np.sqrt(numTrials) + + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: + # (2.a) Extract information about this setting for titles etc. + # These should NOT change for rez[i], i=/=0. + LR = str(rez[0].args['lr_weights']) + # Data type (permuted vs sequential mnist) + if 'data_type' in rez[0].args: #This arg was added late so may not exist. + dataType = rez[0].args['data_type'] + else: + dataType = 'seq' + # Alg name. + algName = rez[0].args['opt_method'] + if algName=='altmin': + algName += str(rez[0].args['n_iter_weights']) + if 'rnnHidSize' in rez[0].args: + modelName='RNN 1-{}'.format(rez[0].args['rnnHidSize']) + else: + modelName=rez[0].args['model'] + TITLE = modelName +'; trained with '+expID + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = 'o', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '^', label='Testing') + plt.legend() + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE) + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'allEpoch.png') + + x_axis0 = np.array(range(0, len(meansR0))) + plt.figure(2) + plt.clf() + plt.errorbar(x_axis0, meansR0, stdsR0, marker = '.', label='Testing') + plt.xlabel('Minibatches') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+'\n(Epoch 0 Minibatches)') + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'minibatchE0.png') + + ## (3.) Finally, record the avg for a big plot at the end. + # For Plotting + avgRez[expID]['xax'] = x_axis + avgRez[expID]['xax0'] = x_axis0 + avgRez[expID]['meansTr'] = meansTr + avgRez[expID]['meansTe'] = meansTe + avgRez[expID]['meansR0'] = meansR0 + avgRez[expID]['stdsTr'] = stdsTr + avgRez[expID]['stdsTe'] = stdsTe + avgRez[expID]['stdsR0'] = stdsR0 + avgRez[expID]['numTr'] = numTrials + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', + "amAdam1_me1":'r', + "amAdam5_me1":'r' + } +markers = {"sgd":'o', "adam":'P', + "amAdam1_me1":'^', + "amAdam5_me1":'^' + } +names = {"sgd":'SGD', "adam":'Adam', + "amAdam1_me1":'AM-Adam', + "amAdam5_me1":'AM-Adam'} +models={'lenet':'LeNet', 'vgg7':'VGG7'} +datasets={'mnist':'MNIST', 'fashion_mnist':'Fashion-MNIST'} +#colors = ['b','g','r','c','m','y','k'] +if 'rnnHidSize' in rez[0].args: + modelName='RNN 1-{}'.format(rez[0].args['rnnHidSize']) + datName='Sequential MNIST' +else: + modelName=models[rez[0].args['model']] + datName=datasets[rez[0].args['dataset']] +resUsableTitle = datName+', '+modelName + + +if datName=='MNIST': + ylim1 = [0.96, 0.99] + ylim2 = [0.1,1] + forget='amAdam5_me1' + numMini=None +elif datName=='Fashion-MNIST': + ylim1 = [0.78,0.9] + ylim2 = [0.1,0.8] + forget='amAdam1_me1' + numMini=150 +elif datName=='Sequential MNIST': + if modelName[-2:]=='50': + ylim1 = [0.3,0.965] + ylim2 = [0.1,1] + elif modelName[-2:]=='15': + ylim1 = [0.775,0.93] + ylim2 = [0.1,0.96] + if numEpochs==50: + ylim1=[0.775, 0.96] + forget='' + numMini=None + +epoch0Display=0.9 +if datName=='Sequential MNIST': + xtras1={'adam':1.001, 'amAdam1_me1':1, 'amAdam5_me1':1, 'sgd':0.99} + xtras2={'adam':1.03, 'amAdam1_me1':1,'amAdam5_me1':1, 'sgd':0.999} +else: + if rez[0].args['batch_size']==200: + if datName=='Fashion-MNIST': + zzz=0.999 + xtras1={'adam':1.005, 'amAdam1_me1':zzz, 'amAdam5_me1':zzz, 'sgd':1} + xtras2={'adam':1.06, 'amAdam1_me1':zzz,'amAdam5_me1':zzz, 'sgd':1} + epoch0Display = 0.8 + else: + zzz=0.999 + xtras1={'adam':1.001, 'amAdam1_me1':zzz, 'amAdam5_me1':zzz, 'sgd':1} + xtras2={'adam':1.06, 'amAdam1_me1':0.95,'amAdam5_me1':0.95, 'sgd':1} + epoch0Display = 1 + else: + xtras1={'adam':1.0001, 'amAdam1_me1':1, 'amAdam5_me1':1, 'sgd':0.99} + xtras2={'adam':1.06, 'amAdam1_me1':1,'amAdam5_me1':1, 'sgd':0.999} + +xtraAx1 = 0.1 +xtraAx2 = 1 +trmark='.' +temark='^' + #mnistAll #fMnistAll +plt.figure(1) +plt.clf() +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansTr' in rez) or expID==forget: + continue +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals =rez['meansTe'] + plt.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals[-1], xy=(1,yvals[-1]*xtras1[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') +plt.xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) +plt.ylim(ylim1) +plt.xlabel('Epochs') +plt.ylabel('Accuracy') +plt.title(resUsableTitle) +plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) +plt.savefig(savePath+'bestEpochs_22jan-'+modelName+datName+'.png', bbox_inches='tight') +########################################### +## Now do epoch-0. +plt.figure(2) +plt.clf() + +END=max(rez['xax0']) +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansR0' in rez) or expID==forget: + continue + yvals = rez['meansR0'] + plt.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals[END], xy=(1,yvals[END]*xtras2[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') +plt.xlim([min(rez['xax0'])-xtraAx2, END+xtraAx2]) +plt.ylim(ylim2) +plt.xlabel('Minibatches') +plt.ylabel('Accuracy') +plt.title(resUsableTitle) +plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) +plt.savefig(savePath+'bestMiniBatches_22jan-'+modelName+datName+'.png', bbox_inches='tight') + + + + + +###################################### +## Combined plots. + +fig = plt.figure(23) +fig.clf() +ax1 = fig.add_subplot(111) + +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansTr' in rez) or expID==forget: + continue +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals = rez['meansTe'] + ax1.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + ax1.annotate('%0.3f' % yvals[-1], xy=(1,yvals[-1]*xtras1[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now set axis limits etc. +ax1.set_xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) +ax1.set_ylim(ylim1) +ax1.set_xlabel('Epochs') +ax1.set_ylabel('Accuracy') +ax1.set_title(resUsableTitle) +ax1.legend(loc=(.05,.0),fontsize='small') +#loc=(0.5,0.65)) +#'best')#'lower left') +#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + +########################################### +## Now do epoch-0 inside the above axis... +ax2=plt.axes([.45, .2, .45, .4]) + +if numMini is not None: + END=numMini +else: + END=len(rez['meansR0'])-1 +for expID, rez in avgRez.items(): + if not ('meansR0' in rez) or expID==forget: + continue + yvals = rez['meansR0'] + ax2.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + ax2.annotate('%0.3f' % yvals[END], xy=(1,yvals[END]*xtras2[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +ax2.set_xlim([min(rez['xax0'])-xtraAx2, END+xtraAx2]) +#ax2.set_xticks([]) +ax2.set_ylim(ylim2) +ax2.set_xlabel('Minibatches') +ax2.annotate('1st Epoch', xy=(12,epoch0Display), fontweight='bold') +#ax2.ylabel('Accuracy') +#ax2.title('Epoch 0 Test Accuracy\n' + resUsableTitle) +#ax2.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + +plt.savefig(savePath+'combined'+datName+'_'+modelName+'.png', bbox_inches='tight') +plt.show() + + + + +######################### +# For computing nonzero std. devs. +# # If there were actually multiple trials, compute standard deviations +# # in accuracies using NONZERO entries only. +# if numTrials>1: +# for epoch, tnsr in enumerate(allTr): +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdtr = tnsr[ tnsr.gt(0) ].std().numpy() +# if np.isnan(stdtr): +# stdtr=np.array([0]) +# # Finally... +# stdsTr[epoch] = stdtr +# ############################## +# # Now do Testing. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() +# if np.isnan(stdte): +# stdte=np.array([0]) +# # Finally... +# stdsTe[epoch] = stdte +# ############################## +# # Finally, do Epoch-0 results.. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# for minibatch, tnsr in enumerate(allR0): +# stdE0 = allTe[minibatch][ allTe[minibatch].gt(0.1) ].std().numpy() +# if np.isnan(stdE0): +# stdE0=np.array([0]) +# # Finally... +# stdE0[epoch] = stdte + + + diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial10_20190119-16:23:19.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial10_20190119-16:23:19.pt new file mode 100644 index 0000000..54705d1 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial10_20190119-16:23:19.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial11_20190119-16:47:21.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial11_20190119-16:47:21.pt new file mode 100644 index 0000000..33dd97d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial11_20190119-16:47:21.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial12_20190119-17:11:52.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial12_20190119-17:11:52.pt new file mode 100644 index 0000000..4ae5cf5 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial12_20190119-17:11:52.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial13_20190119-17:36:12.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial13_20190119-17:36:12.pt new file mode 100644 index 0000000..8c74537 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial13_20190119-17:36:12.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial14_20190119-18:00:20.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial14_20190119-18:00:20.pt new file mode 100644 index 0000000..a3687a8 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial14_20190119-18:00:20.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial15_20190119-18:25:00.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial15_20190119-18:25:00.pt new file mode 100644 index 0000000..67f18a8 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial15_20190119-18:25:00.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial16_20190119-18:49:39.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial16_20190119-18:49:39.pt new file mode 100644 index 0000000..6db5d2c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial16_20190119-18:49:39.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial17_20190119-19:14:08.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial17_20190119-19:14:08.pt new file mode 100644 index 0000000..5101b0a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial17_20190119-19:14:08.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial18_20190119-19:38:05.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial18_20190119-19:38:05.pt new file mode 100644 index 0000000..b52bbf7 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial18_20190119-19:38:05.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial19_20190119-20:02:13.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial19_20190119-20:02:13.pt new file mode 100644 index 0000000..4336dca Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial19_20190119-20:02:13.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial1_20190119-12:44:30.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial1_20190119-12:44:30.pt new file mode 100644 index 0000000..61d1627 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial1_20190119-12:44:30.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial20_20190119-20:26:34.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial20_20190119-20:26:34.pt new file mode 100644 index 0000000..7134f2c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial20_20190119-20:26:34.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial21_20190119-20:51:05.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial21_20190119-20:51:05.pt new file mode 100644 index 0000000..2bcf606 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial21_20190119-20:51:05.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial22_20190119-21:15:07.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial22_20190119-21:15:07.pt new file mode 100644 index 0000000..a3c3a36 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial22_20190119-21:15:07.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial23_20190119-21:39:21.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial23_20190119-21:39:21.pt new file mode 100644 index 0000000..3704627 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial23_20190119-21:39:21.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial24_20190119-22:03:46.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial24_20190119-22:03:46.pt new file mode 100644 index 0000000..ec7b58d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial24_20190119-22:03:46.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial25_20190119-22:28:17.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial25_20190119-22:28:17.pt new file mode 100644 index 0000000..29083b2 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial25_20190119-22:28:17.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial26_20190119-22:53:01.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial26_20190119-22:53:01.pt new file mode 100644 index 0000000..5388021 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial26_20190119-22:53:01.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial27_20190119-23:17:15.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial27_20190119-23:17:15.pt new file mode 100644 index 0000000..b751260 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial27_20190119-23:17:15.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial28_20190119-23:41:56.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial28_20190119-23:41:56.pt new file mode 100644 index 0000000..451068b Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial28_20190119-23:41:56.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial29_20190120-00:05:59.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial29_20190120-00:05:59.pt new file mode 100644 index 0000000..c2b3ee3 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial29_20190120-00:05:59.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial2_20190119-13:09:01.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial2_20190119-13:09:01.pt new file mode 100644 index 0000000..613c739 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial2_20190119-13:09:01.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial30_20190120-00:32:15.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial30_20190120-00:32:15.pt new file mode 100644 index 0000000..6734479 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial30_20190120-00:32:15.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial31_20190120-00:58:39.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial31_20190120-00:58:39.pt new file mode 100644 index 0000000..46df233 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial31_20190120-00:58:39.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial32_20190120-01:25:25.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial32_20190120-01:25:25.pt new file mode 100644 index 0000000..e554a98 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial32_20190120-01:25:25.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial33_20190120-01:50:49.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial33_20190120-01:50:49.pt new file mode 100644 index 0000000..6dd4049 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial33_20190120-01:50:49.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial34_20190120-02:21:39.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial34_20190120-02:21:39.pt new file mode 100644 index 0000000..ef06c1a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial34_20190120-02:21:39.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial35_20190120-02:50:43.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial35_20190120-02:50:43.pt new file mode 100644 index 0000000..411909e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial35_20190120-02:50:43.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial36_20190120-03:19:44.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial36_20190120-03:19:44.pt new file mode 100644 index 0000000..6282c24 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial36_20190120-03:19:44.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial37_20190120-03:48:42.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial37_20190120-03:48:42.pt new file mode 100644 index 0000000..53a7f06 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial37_20190120-03:48:42.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial38_20190120-04:17:48.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial38_20190120-04:17:48.pt new file mode 100644 index 0000000..d969fd3 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial38_20190120-04:17:48.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial39_20190120-04:46:50.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial39_20190120-04:46:50.pt new file mode 100644 index 0000000..81685de Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial39_20190120-04:46:50.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial3_20190119-13:33:18.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial3_20190119-13:33:18.pt new file mode 100644 index 0000000..54e9acb Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial3_20190119-13:33:18.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial40_20190120-05:15:56.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial40_20190120-05:15:56.pt new file mode 100644 index 0000000..2e6f2d6 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial40_20190120-05:15:56.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial41_20190120-05:45:18.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial41_20190120-05:45:18.pt new file mode 100644 index 0000000..4705a8d Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial41_20190120-05:45:18.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial42_20190120-06:14:02.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial42_20190120-06:14:02.pt new file mode 100644 index 0000000..0a6469a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial42_20190120-06:14:02.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial43_20190120-06:43:44.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial43_20190120-06:43:44.pt new file mode 100644 index 0000000..e022e9e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial43_20190120-06:43:44.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial44_20190120-07:11:59.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial44_20190120-07:11:59.pt new file mode 100644 index 0000000..be1f39a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial44_20190120-07:11:59.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial45_20190120-07:40:58.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial45_20190120-07:40:58.pt new file mode 100644 index 0000000..bfed476 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial45_20190120-07:40:58.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial46_20190120-08:10:47.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial46_20190120-08:10:47.pt new file mode 100644 index 0000000..8378f31 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial46_20190120-08:10:47.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial47_20190120-08:53:23.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial47_20190120-08:53:23.pt new file mode 100644 index 0000000..3000c56 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial47_20190120-08:53:23.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial48_20190120-09:32:50.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial48_20190120-09:32:50.pt new file mode 100644 index 0000000..5353b4a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial48_20190120-09:32:50.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial4_20190119-13:57:26.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial4_20190119-13:57:26.pt new file mode 100644 index 0000000..af3ae7e Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial4_20190119-13:57:26.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial5_20190119-14:21:27.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial5_20190119-14:21:27.pt new file mode 100644 index 0000000..4a6b4b7 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial5_20190119-14:21:27.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial6_20190119-14:45:50.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial6_20190119-14:45:50.pt new file mode 100644 index 0000000..017e06c Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial6_20190119-14:45:50.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial7_20190119-15:10:17.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial7_20190119-15:10:17.pt new file mode 100644 index 0000000..6a44421 Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial7_20190119-15:10:17.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial8_20190119-15:34:44.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial8_20190119-15:34:44.pt new file mode 100644 index 0000000..0fc0a0a Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial8_20190119-15:34:44.pt differ diff --git a/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial9_20190119-15:58:42.pt b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial9_20190119-15:58:42.pt new file mode 100644 index 0000000..4797b2f Binary files /dev/null and b/CNN_DEMO/postHPexps/lenet/lenet_mnist/testResults/sgd_trial9_20190119-15:58:42.pt differ diff --git a/CNN_DEMO/postHPexps/paperFigs/epochAll_lenet_FashionMN.png b/CNN_DEMO/postHPexps/paperFigs/epochAll_lenet_FashionMN.png new file mode 100644 index 0000000..dbfda38 Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/epochAll_lenet_FashionMN.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/epochAll_lenet_MNIST.png b/CNN_DEMO/postHPexps/paperFigs/epochAll_lenet_MNIST.png new file mode 100644 index 0000000..9cf768d Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/epochAll_lenet_MNIST.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/jan21/LeNet_FashMN_bsz200.png b/CNN_DEMO/postHPexps/paperFigs/jan21/LeNet_FashMN_bsz200.png new file mode 100644 index 0000000..f1578b8 Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/jan21/LeNet_FashMN_bsz200.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/jan21/LeNet_MNIST_bsz200.png b/CNN_DEMO/postHPexps/paperFigs/jan21/LeNet_MNIST_bsz200.png new file mode 100644 index 0000000..ad61340 Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/jan21/LeNet_MNIST_bsz200.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-15_seqMN.png b/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-15_seqMN.png new file mode 100644 index 0000000..d0c774e Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-15_seqMN.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-50_seqMN.png b/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-50_seqMN.png new file mode 100644 index 0000000..2481b8b Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-50_seqMN.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-50_seqMN_23jan.png b/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-50_seqMN_23jan.png new file mode 100644 index 0000000..845028e Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/jan21/RNN1-50_seqMN_23jan.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/miniBatches50_lenet_FashionMN.png b/CNN_DEMO/postHPexps/paperFigs/miniBatches50_lenet_FashionMN.png new file mode 100644 index 0000000..b5ab74a Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/miniBatches50_lenet_FashionMN.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/miniBatches50_lenet_MNIST.png b/CNN_DEMO/postHPexps/paperFigs/miniBatches50_lenet_MNIST.png new file mode 100644 index 0000000..e430aac Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/miniBatches50_lenet_MNIST.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/miniBatchesAll_lenet_FashionMN.png b/CNN_DEMO/postHPexps/paperFigs/miniBatchesAll_lenet_FashionMN.png new file mode 100644 index 0000000..801c326 Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/miniBatchesAll_lenet_FashionMN.png differ diff --git a/CNN_DEMO/postHPexps/paperFigs/miniBatchesAll_lenet_MNIST.png b/CNN_DEMO/postHPexps/paperFigs/miniBatchesAll_lenet_MNIST.png new file mode 100644 index 0000000..6272592 Binary files /dev/null and b/CNN_DEMO/postHPexps/paperFigs/miniBatchesAll_lenet_MNIST.png differ diff --git a/CNN_DEMO/printHPOrez_cnn.py b/CNN_DEMO/printHPOrez_cnn.py new file mode 100644 index 0000000..2a8530a --- /dev/null +++ b/CNN_DEMO/printHPOrez_cnn.py @@ -0,0 +1,49 @@ +""" +Loads saved .pt's and prints best hyperparameters. +@author Benjamin Cowen +@date 19 January 2019 +""" +from utils import ddict +import numpy as np +import glob +import torch +import argparse +parser = argparse.ArgumentParser(description='Load and print hyperparameters.') +parser.add_argument('--loadPath', default='./', + help='Path to directory w/data.') + +args = parser.parse_args() + +argList2Print={'all':['opt_method', + 'batch_size', + 'dataset', + 'model', + 'lr_weights' + ], + 'altmin':[ + # Algorithm + 'mini_epochs', + 'n_iter_codes', + 'n_iter_weights', + 'lr_codes', + # Model Parameters + 'lambda_w_muFact', + 'lambda_c_muFact', + 'min_mu', + 'max_mu', + 'd_mu', + 'mu_update_freq', + 'mult_mu' + ], + 'sgd':['lambda_w'], + 'adam':['lambda_w'] + } +for filepath in glob.iglob(args.loadPath + '*.pt'): + D = ddict()._load(filepath).args + + print('============= {} ================='.format(D['algorithm'])) + for argName in argList2Print['all']: + print('{} = {}'.format(argName, D[argName])) + for argName in argList2Print[D['opt_method']]: + print('{} = {}'.format(argName, D[argName])) + diff --git a/CNN_DEMO/testRezViz_19jan.py b/CNN_DEMO/testRezViz_19jan.py new file mode 100644 index 0000000..3b5cda0 --- /dev/null +++ b/CNN_DEMO/testRezViz_19jan.py @@ -0,0 +1,507 @@ +""" +Creates visualizations for RNN test results. + +@author Benjamin Cowen +@date 19 Jan 2019 +""" + +############################################ +# Imports ---------------------------------- + +# Loading +from utils import ddict, load_dataset +import glob +import torch as th + +# Plotting and Saving +# The following two lines are for the remote branch. +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt +import os +from shutil import copyfile +import numpy as np +import argparse + + +p = argparse.ArgumentParser(description='Visualize hyperparameter grid search results.') +p.add_argument('--loadPaths', type=str, default=None, metavar='path1/a/, path2/b/', + help='A list of strings/paths to results.') +p.add_argument('--loadPath', type=str, default=None, metavar='path/to/data/', + help='Path to directory containing data to plot.') +p.add_argument('--savePath', type=str, default=None, metavar='path/to/savedIms/', + help='Path to directory containing images saved here.') +p.add_argument('--onlyBest', type=bool, default=False, metavar='True', + help='Set to true if you DONT want all the imgs (only best params).') +p.add_argument('--reduced', type=bool, default=False, metavar='True', + help='Set to true if you only want SGD, Adam, and AltMin-Adam.') +p.add_argument('--epoch0-accs', type=str, default=None, metavar='path/to/epoch-0-accuracies/', + help='Path to directory containing the epoch-0 accuracies... (avg result for all methods)') +args = p.parse_args() + +# Check arguments. +# Assumes loadPaths or loadPath is valid... +if args.loadPaths is None: + # Load and Save Paths + loadPaths = [args.loadPath] +else: + loadPaths = args.loadPaths.strip('[]').split(',') + +# For saving images +if args.savePath is None: + savePath = loadPaths[0] + "ims/" + if not os.path.exists(savePath): + os.mkdir(savePath) +print('*********************************************************') +print('Saving images to '+savePath) +print('*********************************************************') + +############################################ +# Make a copy of this file in the savePath! +here = os.path.realpath(__file__) +there = savePath + (__file__).replace('../','') +copyfile(here,there) + +############################################ +# Decide tree structure here... (TODO way to automate this?) +# Stands for "all shelves" +allSh = {"amAdam1_me1":[], + "amAdam5_me1":[], + "sgd":[], "adam":[] + } + + +# Loop thru given loadpaths. +for k,loadPath in enumerate(loadPaths): + print('Loading files from: ' + loadPath) + for filepath in glob.iglob(loadPath + '*.pt'): + fileName = filepath.replace(loadPath,'') + + # Go ahead and load it. + DD = ddict()._load(filepath) + algBase = DD.args['opt_method'] + + # If SGD, check whether it's Nesterov. + if algBase=='sgd': + expID = 'sgd' + elif algBase=='adam': + expID='adam' + elif algBase=='altmin': + nit = str(DD.args['n_iter_codes']) + me = str(DD.args['mini_epochs']) + expID='amAdam' + nit + '_me' + me + else: + raise Exception("Unrecognized optimization method "+algBase+" (only 'altmin','sgd','adam')") + + # Append results. + if args.reduced: + if not (expID=='sgd' or expID=='adam' or expID=='amAdam5_me1'): + continue + allSh[expID].append(ddict()._load(filepath)) + + +# Compute the average accuracies for epoch 0 (ie, using initializations only) +epoch0_meanTr = None +if args.epoch0_accs is not None: + epoch0_meanTr = [] + epoch0_meanTe = [] + # Compute means and std devs of the initializations. + for filepath in glob.iglob(args.epoch0_accs + '*.pt'): + DD = ddict()._load(filepath) + epoch0_meanTr += DD.perf['tr'] + epoch0_meanTe += DD.perf['te'] + + # Finally average. + epoch0_stdTr = th.Tensor( epoch0_meanTr ).std() + epoch0_meanTr = th.Tensor( epoch0_meanTr ).mean() + epoch0_stdTe = th.Tensor( epoch0_meanTe ).std() + epoch0_meanTe = th.Tensor( epoch0_meanTe ).mean() + + +# This better be the same for all DD anyway :) +############################################ +# Print all single-exp. figures ----------------------- +avgRez = {} +numE0btch = 469 #59#300 # Number of elements in 'first_epoch' +numEpochs = 10 # Number of elements in 'te' and 'tr' + +for expID, rez in allSh.items(): + if len(rez)==0: + continue + # Initialize each Exp ID. + avgRez[expID] = {'tr':[], 'te':[]} + numTrials = len(rez) + + # (0.) Collect the Training/Testing histories. + # Compute the mean and std dev across trials. + allR0 = None + for trial in range(numTrials): + # Shelf for current trial. + tShelf = rez[trial] + + # CHECK VALIDITY:----------------- + # If no 'perf' field, then performance wasn't yet recorded. + if hasattr(tShelf,'perf'): + expData = tShelf.perf + r0 = expData['first_epoch'] + rTr = expData['tr'] + rTe = expData['te'] + else: + continue + + # If the number of recorded epochs << number allowed epochs, it's in-progress. + # If the mean training accurac < 14%, consider this a FAILED attempt at training. + if len(rTe)0: + stdsR0 = allR0.std(1).numpy() + stdsTr = allTr.std(1).numpy() + stdsTe = allTe.std(1).numpy() + else: + stdsR0 = th.zeros(meansR0.size()).numpy() + stdsTr = th.zeros(meansTr.size()).numpy() + stdsTe = th.zeros(meansTe.size()).numpy() + + + ############### + # For epoch-0 results to be tacked on.. + x_axis = np.array(range(1, len(meansTr)+1)) +# meansTr = np.insert(meansTr, 0, meansR0[0]) +# meansTe = np.insert(meansTe, 0, meansR0[0]) +# stdsTr = np.insert(stdsTr, 0, stdsR0[0]) +# stdsTe = np.insert(stdsTe, 0, stdsR0[0]) +# x_axis = np.array(range(0, len(meansTr))) + + # Finally divide by the sqrt number of samples: + + stdsR0 /= np.sqrt(numTrials) + stdsTr /= np.sqrt(numTrials) + stdsTe /= np.sqrt(numTrials) + + ## (2.) If requested, plot the results for each parameter combination. + if not args.onlyBest: + # (2.a) Extract information about this setting for titles etc. + # These should NOT change for rez[i], i=/=0. + LR = str(rez[0].args['lr_weights']) + # Data type (permuted vs sequential mnist) + if 'data_type' in rez[0].args: #This arg was added late so may not exist. + dataType = rez[0].args['data_type'] + else: + dataType = 'seq' + # Alg name. + algName = rez[0].args['opt_method'] + if algName=='altmin': + algName += str(rez[0].args['n_iter_weights']) + if 'rnnHidSize' in rez[0].args: + modelName='RNN 1-{}'.format(rez[0].args['rnnHidSize']) + else: + modelName=rez[0].args['model'] + TITLE = modelName +'; trained with '+expID + + plt.figure(1) + plt.clf() + plt.errorbar(x_axis, meansTr, stdsTr, marker = 'o', label='Training') + plt.errorbar(x_axis, meansTe, stdsTe, marker = '^', label='Testing') + plt.legend() + plt.xlabel('Epochs') + plt.ylabel('Classification Accuracy') + plt.title(TITLE) + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'allEpoch.png') + + x_axis0 = np.array(range(0, len(meansR0))) + plt.figure(2) + plt.clf() + plt.errorbar(x_axis0, meansR0, stdsR0, marker = '.', label='Testing') + plt.xlabel('Minibatches') + plt.ylabel('Classification Accuracy') + plt.title(TITLE+'\n(Epoch 0 Minibatches)') + # plt.ylim([0.875, 1.05]) + plt.savefig(savePath + expID + 'minibatchE0.png') + + ## (3.) Finally, record the avg for a big plot at the end. + # For Plotting + avgRez[expID]['xax'] = x_axis + avgRez[expID]['xax0'] = x_axis0 + avgRez[expID]['meansTr'] = meansTr + avgRez[expID]['meansTe'] = meansTe + avgRez[expID]['meansR0'] = meansR0 + avgRez[expID]['stdsTr'] = stdsTr + avgRez[expID]['stdsTe'] = stdsTe + avgRez[expID]['stdsR0'] = stdsR0 + avgRez[expID]['numTr'] = numTrials + + +###################################################### +# Now we combine results into a plost that compare the +# methods using their best performing parameters. +# (0.) Each method gets a unique color. +colors = {"sgd":'b', "adam":'k', + "amAdam1_me1":'r', + "amAdam5_me1":'r' + } +markers = {"sgd":'o', "adam":'P', + "amAdam1_me1":'^', + "amAdam5_me1":'^' + } +names = {"sgd":'SGD', "adam":'Adam', + "amAdam1_me1":'AM-Adam', + "amAdam5_me1":'AM-Adam'} +models={'lenet':'LeNet', 'vgg7':'VGG7'} +datasets={'mnist':'MNIST', 'fashion_mnist':'Fashion-MNIST'} +#colors = ['b','g','r','c','m','y','k'] +if 'rnnHidSize' in rez[0].args: + modelName='RNN 1-{}'.format(rez[0].args['rnnHidSize']) + datName='Sequential MNIST' +else: + modelName=models[rez[0].args['model']] + datName=datasets[rez[0].args['dataset']] +resUsableTitle = datName+', '+modelName + + +if datName=='MNIST': + ylim1 = [0.96, 0.99] + ylim2 = [0.1,1] + forget='amAdam5_me1' + numMini=None +elif datName=='Fashion-MNIST': + ylim1 = [0.78,0.9] + ylim2 = [0.1,0.8] + forget='amAdam1_me1' + numMini=150 +elif datName=='Sequential MNIST': + if modelName[-2:]=='50': + ylim1 = [0.3,0.965] + ylim2 = [0.1,1] + elif modelName[-2:]=='15': + ylim1 = [0.775,0.93] + ylim2 = [0.1,0.96] + if numEpochs==50: + ylim1=[0.775, 0.96] + forget='' + numMini=None + +epoch0Display=0.9 +if datName=='Sequential MNIST': + xtras1={'adam':1.001, 'amAdam1_me1':1, 'amAdam5_me1':1, 'sgd':0.99} + xtras2={'adam':1.03, 'amAdam1_me1':1,'amAdam5_me1':1, 'sgd':0.999} +else: + if rez[0].args['batch_size']==200: + if datName=='Fashion-MNIST': + zzz=0.999 + xtras1={'adam':1.005, 'amAdam1_me1':zzz, 'amAdam5_me1':zzz, 'sgd':1} + xtras2={'adam':1.06, 'amAdam1_me1':zzz,'amAdam5_me1':zzz, 'sgd':1} + epoch0Display = 0.8 + else: + zzz=0.999 + xtras1={'adam':1.001, 'amAdam1_me1':zzz, 'amAdam5_me1':zzz, 'sgd':1} + xtras2={'adam':1.06, 'amAdam1_me1':0.95,'amAdam5_me1':0.95, 'sgd':1} + epoch0Display = 1 + else: + xtras1={'adam':1.0001, 'amAdam1_me1':1, 'amAdam5_me1':1, 'sgd':0.99} + xtras2={'adam':1.06, 'amAdam1_me1':1,'amAdam5_me1':1, 'sgd':0.999} + +xtraAx1 = 0.1 +xtraAx2 = 1 +trmark='.' +temark='^' + #mnistAll #fMnistAll +plt.figure(1) +plt.clf() +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansTr' in rez) or expID==forget: + continue +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals =rez['meansTe'] + plt.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals[-1], xy=(1,yvals[-1]*xtras1[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') +plt.xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) +plt.ylim(ylim1) +plt.xlabel('Epochs') +plt.ylabel('Accuracy') +plt.title(resUsableTitle) +plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) +plt.savefig(savePath+'bestEpochs_22jan-'+modelName+datName+'.png', bbox_inches='tight') +########################################### +## Now do epoch-0. +plt.figure(2) +plt.clf() + +END=max(rez['xax0']) +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansR0' in rez) or expID==forget: + continue + yvals = rez['meansR0'] + plt.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + plt.annotate('%0.3f' % yvals[END], xy=(1,yvals[END]*xtras2[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now plot a marker designating when TESTING phase began. +# plt.axvline(x=59.5, linewidth=2, label='Begin testing') +plt.xlim([min(rez['xax0'])-xtraAx2, END+xtraAx2]) +plt.ylim(ylim2) +plt.xlabel('Minibatches') +plt.ylabel('Accuracy') +plt.title(resUsableTitle) +plt.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) +plt.savefig(savePath+'bestMiniBatches_22jan-'+modelName+datName+'.png', bbox_inches='tight') + + + + + +###################################### +## Combined plots. + +fig = plt.figure(23) +fig.clf() +ax1 = fig.add_subplot(111) + +# Plot all results. +for expID, rez in avgRez.items(): + if not ('meansTr' in rez) or expID==forget: + continue +# plt.errorbar(rez['xax'], rez['meansTr'], rez['stdsTr'], +# marker = trmark, color=colors[expID], +# markersize = 4, linewidth = 1, +# label=names[expID]+' TR') + yvals = rez['meansTe'] + ax1.errorbar(rez['xax'], yvals, rez['stdsTe'], + marker = temark, color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + ax1.annotate('%0.3f' % yvals[-1], xy=(1,yvals[-1]*xtras1[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +# Now set axis limits etc. +ax1.set_xlim([min(rez['xax'])-xtraAx1, max(rez['xax'])+xtraAx1]) +ax1.set_ylim(ylim1) +ax1.set_xlabel('Epochs') +ax1.set_ylabel('Accuracy') +ax1.set_title(resUsableTitle) +ax1.legend(loc=(.05,.0),fontsize='small') +#loc=(0.5,0.65)) +#'best')#'lower left') +#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + +########################################### +## Now do epoch-0 inside the above axis... +ax2=plt.axes([.45, .2, .45, .4]) + +if numMini is not None: + END=numMini +else: + END=len(rez['meansR0'])-1 +for expID, rez in avgRez.items(): + if not ('meansR0' in rez) or expID==forget: + continue + yvals = rez['meansR0'] + ax2.errorbar(rez['xax0'], yvals, rez['stdsR0'], + marker = markers[expID], color=colors[expID], + markersize = 4, linewidth = 1, + label=names[expID]+' (N={})'.format(rez['numTr'])) + # Plot the final accuracy to the right of the axes. + ax2.annotate('%0.3f' % yvals[END], xy=(1,yvals[END]*xtras2[expID]), xytext=(8,0), + xycoords=('axes fraction', 'data'), textcoords='offset points') +ax2.set_xlim([min(rez['xax0'])-xtraAx2, END+xtraAx2]) +#ax2.set_xticks([]) +ax2.set_ylim(ylim2) +ax2.set_xlabel('Minibatches') +ax2.annotate('1st Epoch', xy=(12,epoch0Display), fontweight='bold') +#ax2.ylabel('Accuracy') +#ax2.title('Epoch 0 Test Accuracy\n' + resUsableTitle) +#ax2.legend()#loc=9, bbox_to_anchor=(0.5, -0.1), ncol=3) + +plt.savefig(savePath+'combined'+datName+'_'+modelName+'.png', bbox_inches='tight') +plt.show() + + + + +######################### +# For computing nonzero std. devs. +# # If there were actually multiple trials, compute standard deviations +# # in accuracies using NONZERO entries only. +# if numTrials>1: +# for epoch, tnsr in enumerate(allTr): +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdtr = tnsr[ tnsr.gt(0) ].std().numpy() +# if np.isnan(stdtr): +# stdtr=np.array([0]) +# # Finally... +# stdsTr[epoch] = stdtr +# ############################## +# # Now do Testing. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# stdte = allTe[epoch][ allTe[epoch].gt(0.1) ].std().numpy() +# if np.isnan(stdte): +# stdte=np.array([0]) +# # Finally... +# stdsTe[epoch] = stdte +# ############################## +# # Finally, do Epoch-0 results.. +# # This is my stupid way of doing ONLY NONZERO std dev's... +# for minibatch, tnsr in enumerate(allR0): +# stdE0 = allTe[minibatch][ allTe[minibatch].gt(0.1) ].std().numpy() +# if np.isnan(stdE0): +# stdE0=np.array([0]) +# # Finally... +# stdE0[epoch] = stdte + + + diff --git a/CNN_DEMO/train_cnn_14jan.py b/CNN_DEMO/train_cnn_14jan.py new file mode 100644 index 0000000..cc75d35 --- /dev/null +++ b/CNN_DEMO/train_cnn_14jan.py @@ -0,0 +1,365 @@ +""" +@author Benjamin Cowen and Mattia Rigotti +@date 23 Jan 2019 +""" +from __future__ import print_function, division +import argparse +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +import random as rand # needed for validation shuffle +from altmin14jan import get_mods, get_codes +from altmin14jan import update_codes, update_last_layer_, update_hidden_weights_adam_, set_mod_optimizers_ +from altmin14jan import post_processing_step + +from utils import ddict, load_dataset + +########################################################################################## +########################################################################################## + +parser = argparse.ArgumentParser(description='AltMin on mnist') +#............................................................. +# New experimental option is to load best hyperparameters from saved thing. +# Will use the Bunch class to make a "namespace" from this loaded dict, if requested. +class Bunch(object): + def __init__(self, adict): + self.__dict__.update(adict) +parser.add_argument('--loadBestHyperparameters', type=str, default=None, + help='Path to file containing ddict() of desired arguments.') +#............................................................. + +# Training settings +parser.add_argument('--delta-batch-size', type=int, default=0, metavar='N', + help='batch size increas after each epoch') +parser.add_argument('--epochs', type=int, default=200, metavar='N', + help='number of epochs to train (default: 10)') + +parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') +parser.add_argument('--postprocessing-steps', type=int, default=0, metavar='N', + help='number of Carreira-Perpinan post-processing steps after training') +parser.add_argument('--dataset', default='mnist', metavar='D', + help='name of dataset') +parser.add_argument('--use-validation-size', type=int, default=-1, metavar='N', + help='If <=0, uses test data. Otherwise, uses validation set of size N from training set to "test".') +parser.add_argument('--data-augmentation', action='store_true', default=False, + help='enables data augmentation') +parser.add_argument('--model', default='LeNet', metavar='M', + help='name of model') +parser.add_argument('--log-frequency', type=int, default=4, metavar='N', + help='how many times per epoch to log training status') +parser.add_argument('--save-interval', type=int, default=1000, metavar='N', + help='how many batches to wait before saving test performance (only relevant if one epoch is larger than that)') +parser.add_argument('--first-epoch-log', action='store_true', default=False, + help='whether or not it should test and log after every mini-batch in first epoch') +# WE ARE NOT ALLOWING INDEPENDENT PARAMETER HERE: +#parser.add_argument('--lr-out', type=float, default=0.001, metavar='LR', +# help='learning rate for last layer weights updates') + +# Logistics: +parser.add_argument('--seed', type=int, default=1, metavar='S', + help='random seed (default: 1)') +parser.add_argument('--save-filename', default='', metavar='F', + help='name of file where results are saved') +parser.add_argument('--opt-method', type=str, default='altmin', + help='"altmin" or "adam" or "sgd"') + +# Things we loop over: +parser.add_argument('--batch-size', type=int, default=128, metavar='N', + help='input batch size for training') +# CODE LEARNING PARAMS +parser.add_argument('--lr-codes', type=float, default=0.5, metavar='LR', + help='learning rate for codes updates') +parser.add_argument('--n-iter-codes', type=int, default=1, metavar='N', + help='number of internal iterations for codes optimization') +# Sparsity: +parser.add_argument('--lambda_c', type=float, default=0.0, metavar='L', + help='codes sparsity') +# WEIGHT LEARNING PARAMS +parser.add_argument('--n-iter-weights', type=int, default=1, metavar='N', + help='number of internal iterations in learning weights') +parser.add_argument('--lr-weights', type=float, default=0.002, metavar='LR', + help='learning rate for hidden weights updates') +# Sparsity: +parser.add_argument('--lambda_w', type=float, default=0.0, metavar='L', + help='weight sparsity') + +# Newly added: +######## MU: +parser.add_argument('--min-mu', type=float, default=0.01, metavar='M', + help='starting point for mu') +parser.add_argument('--d-mu', type=float, default=0.001/600, metavar='M', + help='increase in mu after every mini-batch') +parser.add_argument('--mult-mu', type=float, default=1, metavar='M', + help='multiplies into mu after every mini-batch') +parser.add_argument('--mu-update-freq', type=int, default=1, metavar='M', + help='Number of minibatches between \mu updates (or just at end of epoch if larger than number of minibatches/epoch).') +parser.add_argument('--max-mu', type=float, default=10000, metavar='M', + help='cap for mu (can\'t go higher)') + +######### MU-DEPENDENT SPARSITY: +parser.add_argument('--lambda_c_muFact', type=float, default=0.0, metavar='L', + help='Takes precedent over regular lambda_c. Sets lambda_c = L*mu.') +parser.add_argument('--lambda_w_muFact', type=float, default=0.0, metavar='L', + help='Takes precedent over regular lambda_w. Sets lambda_w = L*mu.') +####### MINI-EPOCHS +parser.add_argument('--mini-epochs', type=int, default=1, metavar='N', + help='Alternate between code and weight subproblem N times per minibatch') +######### LAYER-WISE DROPOUT!! + +parser.add_argument('--LW-dropout-perc', type=float, default=0, + help='Percent chance of dropping out.') +parser.add_argument('--LW-dropout-delay', type=int, default=1, + help='Number of epochs to delay application of layer-wise dropout.') +########################################################################################## +########################################################################################## + +# Continuing... +args_og = parser.parse_args() +if args_og.loadBestHyperparameters is None: + args = args_og +else: + # OR~~ load args from previously saved ddict(). SOME ARGS (below) + # WILL BE OVERWRITTEN (ie to load "best hyperparameters" then + # run diff number of epochs etc) + args = Bunch(ddict()._load(args_og.loadBestHyperparameters).args) + # Certain args will be changed from original: + # all the "Logistics" arguments + args.save_filename = args_og.save_filename + args.seed = args_og.seed + args.no_cuda = args_og.no_cuda + args.log_frequency = args_og.log_frequency + args.save_interval = args_og.save_interval + args.first_epoch_log = args_og.first_epoch_log + + # epochs + args.epochs = args_og.epochs + + # dataset (noisy test set) + args.use_validation_size = args_og.use_validation_size + + # TODO Maybe change these as well at some point?... + # L1 ? + # n-iter-codes? + + + + + + +algName=args.opt_method +args.algorithm =algName + '_CNN' +# Train all layers with the same learning rate for controllability! +args.lr_out = args.lr_weights + +# Check cuda +device = torch.device("cuda:0" if not args.no_cuda and torch.cuda.is_available() else "cpu") +torch.manual_seed(args.seed) +if device.type != 'cpu': + print('\033[93m'+'Using CUDA'+'\033[0m') + torch.cuda.manual_seed(args.seed) + +# Do not use multi-gpu!! +multi_gpu, num_workers = False, 1 +if False: + if device.type != 'cpu' and torch.cuda.device_count() > 1: + print('\033[93m'+'Using ', torch.cuda.device_count(), 'GPUs'+'\033[0m') + multi_gpu = True + num_workers = torch.cuda.device_count() + +# Load dataset +print('\n* Loading dataset {}'.format(args.dataset)) +if args.use_validation_size>0: + dataName = 'valid_' + data_seed=23 + rand.seed(data_seed) +else: + dataName = '' +if args.data_augmentation: + print(' data augmentation') +train_loader, test_loader, n_inputs = load_dataset(dataName+args.dataset, batch_size=args.batch_size, conv_net=True, data_augmentation=args.data_augmentation, num_workers=num_workers, valid_size=args.use_validation_size) + +window_size = train_loader.dataset.train_data[0].shape[0] +if len(train_loader.dataset.train_data[0].shape) == 3: + num_input_channels = train_loader.dataset.train_data[0].shape[2] +else: + num_input_channels = 1 + +if hasattr(train_loader, 'numSamples'): + numTrData = train_loader.numSamples + numTeData = test_loader.numSamples +else: + numTrData = len(train_loader.dataset) + numTeData = len(test_loader.dataset) + + +# Load model +print('* Loading model {}'.format(args.model)) +if args.model.lower() == 'lenet': + from models import test, LeNet + model = LeNet(num_input_channels=num_input_channels, window_size=window_size, bias=True).to(device) +elif args.model.lower() == 'vgg7': + from models import test, VGG7 + model = VGG7(num_input_channels=num_input_channels, window_size=window_size, bias=True).to(device) +elif args.model.lower() in ['resnet', 'resnet18']: + from models import test, resnet18 + model = resnet18(num_classes=10).to(device) +criterion = nn.CrossEntropyLoss() + + +# Main +if __name__ == "__main__": + # Save everything in SHELF + print('********************') + print('Saving shelf to:') + print(args.save_filename) + print('********************') + SH = ddict(args=args.__dict__) + if args.save_filename: + SH._save(args.save_filename, date=True) + + # Store training and test performance after each training epoch + SH.perf = ddict(tr=[], te=[]) + + # Store test performance after each iteration in first epoch + SH.perf.first_epoch = [] + + # Store test performance after each args.save_interval iterations + SH.perf.te_vs_iterations = [] + + # Create model, and expose modules that has_codes + if algName=='altmin': + model = get_mods(model) + set_mod_optimizers_(model, optimizer = 'Adam', + optimizer_params = {'lr':args.lr_weights}, data_parallel=multi_gpu) + model[-1].optimizer.param_groups[0]['lr'] = args.lr_weights + + elif algName == 'adam': + optimizer = optim.Adam(model.parameters(), lr=args.lr_weights) + elif algName == 'sgd': + optimizer = optim.SGD(model.parameters(), lr=args.lr_weights) + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9) + + # Initial mu and increment after every mini-batch + mu = args.min_mu + mu_max = args.max_mu + + for epoch in range(1, args.epochs+1): + + # Configure dropout parameters. + if args.LW_dropout_perc>0 and args.LW_dropout_delay0: + epoch_lam_c = args.lambda_c_muFact * mu + else: + epoch_lam_c = args.lambda_c + if args.lambda_w_muFact>0: + epoch_lam_w = args.lambda_w_muFact * mu + else: + epoch_lam_w = args.lambda_w + + # Now run alt-min algorithm. + for mini_epoch in range(args.mini_epochs): + ######## Forward: compute codes. + model.train() + with torch.no_grad(): + outputs, codes = get_codes(model, data) + + # Update codes w/weights fixed. + codes = update_codes(codes, model, targets, criterion, mu, lambda_c=epoch_lam_c, n_iter=args.n_iter_codes, lr=args.lr_codes) + + # Update weights w/codes fixed. + # Manually apply dropout for last layer: + if (torch.rand(1).item() > useDropout) or useDropout<=0.00001: + update_last_layer_(model[-1], codes[-1], targets, criterion, n_iter=args.n_iter_weights) + + update_hidden_weights_adam_(model, data, codes, lambda_w=epoch_lam_w, n_iter=args.n_iter_weights, dropout_perc=useDropout) + + # Increment mu. (batch_idx init to 0) + if ((batch_idx+1) % args.mu_update_freq == 0) and (mu < mu_max): + mu += args.d_mu + mu *= args.mult_mu + #---------------------------------------------------------- + elif algName in ['sgd','adam']: + optimizer.zero_grad() + outputs = model(data) + loss = criterion(outputs, targets) + loss.backward() + optimizer.step() + #---------------------------------------------------------- + + # Store all iterations of first epoch + if epoch == 1 and args.first_epoch_log: + SH.perf.first_epoch += [test(model, data_loader=test_loader, label=" - Test")] + + # Outputs to terminal + if batch_idx % int(len(train_loader)/args.log_frequency) == 0: + loss = criterion(outputs, targets) + print(' Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), numTrData, + 100. * batch_idx / len(train_loader), loss.item())) + + # After every args.save_interval iterations compute and save test error + if batch_idx % args.save_interval == 0 and batch_idx > 0: + SH.perf.te_vs_iterations += [test(model, data_loader=test_loader, label=" - Test")] + if args.save_filename: + SH._save() + + # If the mu-increment-frequency is larger than the number of batches, + # here is a separate update for epoch-wise mu updating. + if (args.mu_update_freq > len(train_loader)) and (mu < mu_max): + mu += args.d_mu + mu *= args.mult_mu + + # Print performances + SH.perf.tr += [test(model, data_loader=train_loader, label="Training")] + SH.perf.te += [test(model, data_loader=test_loader, label="Test")] + + # Save data after every epoch + if args.save_filename: + SH._save() + + # ---------------------------------------------------------------- + # Post-processing step from Carreira-Perpinan (fit last layer): + # ---------------------------------------------------------------- + if args.postprocessing_steps > 0: + + print('\nPost-processing step:\n') + + for epoch in range(1, args.postprocessing_steps+1): + for batch_idx, (data, targets) in enumerate(train_loader): + data, targets = data.to(device), targets.to(device) + post_processing_step(model, data, targets, criterion, args.lambda_w) + + # Outputs to terminal + if batch_idx % args.log_interval == 0: + print(' Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, batch_idx * len(data), len(train_loader.dataset), + 100. * batch_idx / len(train_loader), loss.item())) + + # Print performances + SH.perf.tr_final = test(model, data_loader=train_loader, label=" Training set after post-processing") + SH.perf.te_final = test(model, data_loader=test_loader, label=" Test set after post-processing ") + + # Save data after every epoch + if args.save_filename: + SH._save() + + if args.save_model: + SH.perf.model = model diff --git a/CNN_DEMO/utils/__init__.py b/CNN_DEMO/utils/__init__.py new file mode 100644 index 0000000..cb26718 --- /dev/null +++ b/CNN_DEMO/utils/__init__.py @@ -0,0 +1,13 @@ +try: + from .utils import ddict + from .datasets import load_dataset + from .mnist_multitask import load_multitask_dataset + +except ImportError as e: + import sys + print('''Could not import submodules (exact error was: %s).''' % e, file=sys.stderr) + + +__all__ = [ + 'ddict', 'load_dataset', 'load_multitask_dataset', 'Flatten' +] diff --git a/CNN_DEMO/utils/__pycache__/__init__.cpython-36.pyc b/CNN_DEMO/utils/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..ee58269 Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/__init__.cpython-36.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/__init__.cpython-37.pyc b/CNN_DEMO/utils/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000..a3bc7bd Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/__init__.cpython-37.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/datasets.cpython-36.pyc b/CNN_DEMO/utils/__pycache__/datasets.cpython-36.pyc new file mode 100644 index 0000000..c419d1c Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/datasets.cpython-36.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/datasets.cpython-37.pyc b/CNN_DEMO/utils/__pycache__/datasets.cpython-37.pyc new file mode 100644 index 0000000..e108292 Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/datasets.cpython-37.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/mnist_multitask.cpython-36.pyc b/CNN_DEMO/utils/__pycache__/mnist_multitask.cpython-36.pyc new file mode 100644 index 0000000..097653c Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/mnist_multitask.cpython-36.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/mnist_multitask.cpython-37.pyc b/CNN_DEMO/utils/__pycache__/mnist_multitask.cpython-37.pyc new file mode 100644 index 0000000..d2c2563 Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/mnist_multitask.cpython-37.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/nn.cpython-36.pyc b/CNN_DEMO/utils/__pycache__/nn.cpython-36.pyc new file mode 100644 index 0000000..040aedc Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/nn.cpython-36.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/nn.cpython-37.pyc b/CNN_DEMO/utils/__pycache__/nn.cpython-37.pyc new file mode 100644 index 0000000..57b688f Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/nn.cpython-37.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/utils.cpython-36.pyc b/CNN_DEMO/utils/__pycache__/utils.cpython-36.pyc new file mode 100644 index 0000000..ea7afc4 Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/utils.cpython-36.pyc differ diff --git a/CNN_DEMO/utils/__pycache__/utils.cpython-37.pyc b/CNN_DEMO/utils/__pycache__/utils.cpython-37.pyc new file mode 100644 index 0000000..c9d8379 Binary files /dev/null and b/CNN_DEMO/utils/__pycache__/utils.cpython-37.pyc differ diff --git a/CNN_DEMO/utils/datasets.py b/CNN_DEMO/utils/datasets.py new file mode 100644 index 0000000..e61ea36 --- /dev/null +++ b/CNN_DEMO/utils/datasets.py @@ -0,0 +1,347 @@ +""" +dataloaders + +@author Benjamin Cowen and Mattia Rigotti +@date 20 December 2018 +""" +import torch +from torchvision import datasets, transforms +from torch.utils.data.sampler import SubsetRandomSampler +from random import shuffle + +def load_dataset(namedataset='mnist', batch_size=200, data_augmentation=False, conv_net=False, num_workers=1, noise_sigma=0.0, valid_size=-1): + '''data_augmentation: use data augmentation, if it is available for dataset + conv_net: set to `True` if the dataset is being used with a conv net (i.e. the inputs have to be 3d tensors and not flattened) + ''' + + # Load mnist dataset + if namedataset == 'mnist': + + DIR_DATASET = '~/data/mnist' + + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,))] + + if not conv_net: + transform_list.append(transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2)))) + + transform = transforms.Compose(transform_list) + + trainset = datasets.MNIST(DIR_DATASET, train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + testset = datasets.MNIST(DIR_DATASET, train=False, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + classes = tuple(range(10)) + n_inputs = 784 + elif namedataset == 'valid_mnist': + + if valid_size < 1: + raise ValueError('Validation requested with validation size = {}'.format(valid_size)) + + DIR_DATASET = '~/data/mnist' + + # Define the transforms applied to every sample. + # Desired mean and Std Dev of data: + MEAN = 0.1307 + STD = 0.3081 + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((MEAN,), (STD,))] + transform = transforms.Compose(transform_list) + + # Now load training data and make two dataloaders (train/validation) for it. + allTrainData = datasets.MNIST(DIR_DATASET, train=True, download=True, transform=transform) + num_train = len(allTrainData) + indices = list(range(num_train)) + # First shuffle the indices: + shuffle(indices) + # Assign sampled before split to train; after split to validation. + split = num_train - valid_size + print('nTrain='+str(num_train)) + print('validsize='+str(valid_size)) + print('split='+str(split)) + train_idx, valid_idx = indices[:split], indices[split:] + # Random samplers for the relevant indices only. + train_sampler = SubsetRandomSampler(train_idx) + valid_sampler = SubsetRandomSampler(valid_idx) + + # Finally, instantiate the data loaders. + train_loader = torch.utils.data.DataLoader(allTrainData, + sampler = train_sampler, + batch_size = batch_size, + num_workers = num_workers) + train_loader.numSamples=len(train_idx) + # THIS IS THE VALIDATION SET: + test_loader = torch.utils.data.DataLoader(allTrainData, + sampler = valid_sampler, + batch_size = batch_size, + num_workers = num_workers) + test_loader.numSamples=len(valid_idx) + + print('train size = '+str(train_loader.numSamples)) + print('valid size = '+str(test_loader.numSamples)) + classes = tuple(range(10)) + n_inputs = 784 + + # Load SEQUENTIAL mnist dataloader + # That is, vectorized and grayscaled to a 784 length vector. + elif namedataset == 'seq_mnist': + + DIR_DATASET = '~/data/mnist' + + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,))] + + # Vectorize + transform_list.append(transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2)))) + + transform = transforms.Compose(transform_list) + + trainset = datasets.MNIST(DIR_DATASET, train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + testset = datasets.MNIST(DIR_DATASET, train=False, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + classes = tuple(range(10)) + n_inputs = 784 + + elif namedataset == 'noisy_seq_mnist': + + DIR_DATASET = '~/data/mnist' + + # Desired mean and Std Dev of data: + MEAN = 0.1307 + STD = 0.3081 + + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((MEAN,), (STD,))] + + # Vectorize + transform_list.append(transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2)))) + + transform = transforms.Compose(transform_list) + + trainset = datasets.MNIST(DIR_DATASET, train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + # Add Gaussian noise to the test set. + transform_list.append(transforms.Lambda(lambda x: x + MEAN + + noise_sigma*STD*torch.randn(x.size()))) + + testset = datasets.MNIST(DIR_DATASET, train=False, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + classes = tuple(range(10)) + n_inputs = 784 + + elif namedataset == 'valid_seq_mnist': + + if valid_size < 1: + raise ValueError('Validation requested with validation size = {}'.format(valid_size)) + + DIR_DATASET = '~/data/mnist' + + # Define the transforms applied to every sample. + # Desired mean and Std Dev of data: + MEAN = 0.1307 + STD = 0.3081 + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((MEAN,), (STD,))] + # Vectorize + transform_list.append(transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2)))) + transform = transforms.Compose(transform_list) + + # Now load training data and make two dataloaders (train/validation) for it. + allTrainData = datasets.MNIST(DIR_DATASET, train=True, download=True, transform=transform) + num_train = len(allTrainData) + indices = list(range(num_train)) + # First shuffle the indices: + shuffle(indices) + # Assign sampled before split to train; after split to validation. + split = num_train - valid_size + print('nTrain='+str(num_train)) + print('validsize='+str(valid_size)) + print('split='+str(split)) + train_idx, valid_idx = indices[:split], indices[split:] + # Random samplers for the relevant indices only. + train_sampler = SubsetRandomSampler(train_idx) + valid_sampler = SubsetRandomSampler(valid_idx) + + # Finally, instantiate the data loaders. + train_loader = torch.utils.data.DataLoader(allTrainData, + sampler = train_sampler, + batch_size = batch_size, + num_workers = num_workers) + train_loader.numSamples=len(train_idx) + # THIS IS THE VALIDATION SET: + test_loader = torch.utils.data.DataLoader(allTrainData, + sampler = valid_sampler, + batch_size = batch_size, + num_workers = num_workers) + test_loader.numSamples=len(valid_idx) + + print('train size = '+str(train_loader.numSamples)) + print('valid size = '+str(test_loader.numSamples)) + classes = tuple(range(10)) + n_inputs = 784 + + # Load mnist dataset + elif namedataset == 'fashion_mnist': + + DIR_DATASET = '~/data/fashion_mnist' + + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,))] + + if not conv_net: + transform_list.append(transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2)))) + + transform = transforms.Compose(transform_list) + + trainset = datasets.FashionMNIST(DIR_DATASET, train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + testset = datasets.FashionMNIST(DIR_DATASET, train=False, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + classes = tuple(range(10)) + n_inputs = 784 + + elif namedataset == 'valid_fashion_mnist': + + if valid_size < 1: + raise ValueError('Validation requested with validation size = {}'.format(valid_size)) + + DIR_DATASET = '~/data/fashion_mnist' + + # Define the transforms applied to every sample. + # Desired mean and Std Dev of data: + MEAN = 0.1307 + STD = 0.3081 + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((MEAN,), (STD,))] + + if not conv_net: + transform_list.append(transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2)))) + + transform = transforms.Compose(transform_list) + + # Now load training data and make two dataloaders (train/validation) for it. + allTrainData = datasets.FashionMNIST(DIR_DATASET, train=True, download=True, transform=transform) + num_train = len(allTrainData) + indices = list(range(num_train)) + # First shuffle the indices: + shuffle(indices) + # Assign sampled before split to train; after split to validation. + split = num_train - valid_size + print('nTrain='+str(num_train)) + print('validsize='+str(valid_size)) + print('split='+str(split)) + train_idx, valid_idx = indices[:split], indices[split:] + # Random samplers for the relevant indices only. + train_sampler = SubsetRandomSampler(train_idx) + valid_sampler = SubsetRandomSampler(valid_idx) + + # Finally, instantiate the data loaders. + train_loader = torch.utils.data.DataLoader(allTrainData, + sampler = train_sampler, + batch_size = batch_size, + num_workers = num_workers) + train_loader.numSamples=len(train_idx) + # THIS IS THE VALIDATION SET: + test_loader = torch.utils.data.DataLoader(allTrainData, + sampler = valid_sampler, + batch_size = batch_size, + num_workers = num_workers) + test_loader.numSamples=len(valid_idx) + + print('train size = '+str(train_loader.numSamples)) + print('valid size = '+str(test_loader.numSamples)) + classes = tuple(range(10)) + n_inputs = 784 + + # Load mnist_tf dataset (mnist with tensorflow validation split, i.e. remove + # first 5000 samples from training set for validation split) + elif namedataset == 'mnist_tf': + + from .mnist_tf import MNIST_TF + DIR_DATASET = '~/data/mnist' + + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2))), + ]) + + trainset = MNIST_TF(DIR_DATASET, train=True, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + testset = MNIST_TF(DIR_DATASET, train=False, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=True, num_workers=num_workers) + + classes = tuple(range(10)) + n_inputs = 784 + + # Load cifar10 (preprocessing from https://github.com/kuangliu/pytorch-cifar) + elif namedataset == 'cifar10': + + DIR_DATASET = '~/data/cifar10' + + transform_list = [ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))] + + if not conv_net: + transform_list.append( + transforms.Lambda(lambda x: x.view(x.size(0)*x.size(1)*x.size(2)))) + + transform_test = transforms.Compose(transform_list) + + if data_augmentation: + transform_train_list = [ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))] + + if not conv_net: + transform_train_list.append( + transforms.Lambda(lambda x: x.view(x.size(0)*x.size(1)*x.size(2)))) + + transform_train = transforms.Compose(transform_train_list) + + else: + transform_train = transform_test + + trainset = datasets.CIFAR10(DIR_DATASET, train=True, download=True, transform=transform_train) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + testset = datasets.CIFAR10(DIR_DATASET, train=False, download=True, transform=transform_test) + test_loader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=num_workers) + + classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') + n_inputs = 3*32*32 + + # Load Higgs (first 10'000 samples of dataset are test, the rest are training) + elif namedataset == 'higgs': + + from .higgs import HIGGS_LOADER + DIR_DATASET = '~/data/higgs' + + train_loader = HIGGS_LOADER(DIR_DATASET, train=True, download=True, batch_size=batch_size, num_workers=num_workers) + test_loader = HIGGS_LOADER(DIR_DATASET, train=False, download=True, batch_size=batch_size, num_workers=num_workers) + n_inputs = train_loader.n_inputs + + else: + raise ValueError('Dataset {} not recognized'.format(namedataset)) + + return train_loader, test_loader, n_inputs diff --git a/CNN_DEMO/utils/higgs.py b/CNN_DEMO/utils/higgs.py new file mode 100644 index 0000000..aaabd4a --- /dev/null +++ b/CNN_DEMO/utils/higgs.py @@ -0,0 +1,184 @@ +from __future__ import print_function +import torch.utils.data as data +import sys +import os +import os.path +import errno +import torch +import codecs +from utils.utils import download_progress +import itertools +import pandas as pd + + +class HIGGS_LOADER(object): + """HIGGS dataset + Args: + root (string): Root directory of dataset where ``processed/training.pt`` + and ``processed/test.pt`` exist. + train (bool, optional): If True, creates dataset from ``training.pt``, + otherwise from ``test.pt``. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in root directory. If dataset is already downloaded, it is not + downloaded again. + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + """ + url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz' + + n_test = 500000 # (as in paper use last 500k samples for test) + test_file = 'test.csv' + n_train = 11000000-n_test + chunk_size = 100000 # chuck_size must be a divisor of n_test and n_train + n_inputs = 28 + classes = [0.0, 1.0] + + def __init__(self, root, train=True, transform=None, target_transform=None, download=False, batch_size=100): + self.root = os.path.expanduser(root) + self.filename = os.path.basename(self.url) + self.filepath = os.path.join(self.root, self.filename) + self.testfilepath = os.path.join(self.root, self.test_file) + self.transform = transform + self.target_transform = target_transform + self.train = train # training set or test set + self.batch_size = batch_size + + self.columns = ['y'] + list(range(self.n_inputs)) + + if download: + self.download() + + if not (self._check_exists_raw() and self._check_exists_test()): + raise RuntimeError('Dataset not found.' + + ' You can use download=True to download it') + # Quick and dirty, so that len(dataloader.dataset) works + class dl: + def __init__(self, n): + self.n = n + def __len__(self): + return self.n + if self.train: + self.dataset = dl(self.n_train) + else: + self.dataset = dl(self.n_test) + + def __len__(self): + if self.train: + return int(self.n_train/self.batch_size) + else: + return int(self.n_test/self.batch_size) + + def _check_exists_raw(self): + return os.path.exists(self.filepath) + + def _check_exists_test(self): + # Check test set and raw data for training + return os.path.exists(os.path.join(self.root, self.test_file)) + + def download(self): + """Download the MNIST data if it doesn't exist in processed_folder already.""" + if not self._check_exists_raw(): + + # download files + try: + os.makedirs(self.root) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + + download_progress(self.url, self.root) + print('Done!') + + # Make the test set if file does not exist + if not self._check_exists_test(): + print('\nCreating HIGGS test dataset (this will take some time):') + iter_start = 0 + iter_end = int((self.n_train+self.n_test)/self.chunk_size) + test_start = int(self.n_train/self.chunk_size) + + test_data = pd.DataFrame() + for i, data_chunk in enumerate(itertools.islice( + pd.read_csv(self.filepath, compression='gzip', sep=',', chunksize=self.chunk_size, names=self.columns), + iter_start, iter_end)): + + sys.stdout.write(" Processes %d of %d data chunks (%0.2f%%)\r" % + (i+1, iter_end, (i+1)*100/iter_end)) + if i >= iter_end-1: + sys.stdout.write('\n') + + if i >= test_start: + test_data = pd.concat([test_data, data_chunk]) + + print('Writing test file'+self.testfilepath) + test_data.to_csv(self.testfilepath) + print('Done!') + + def __iter__(self): + if self.train: + iter_start = 0 + iter_end = int(self.n_train/self.chunk_size) + csv_file = self.filepath + else: + iter_start = 0 + iter_end = int(self.n_test/self.chunk_size) + csv_file = self.testfilepath + + self.data_iter = itertools.islice( + pd.read_csv(self.filepath, compression='gzip', sep=',', chunksize=self.chunk_size, names=self.columns), + iter_start, iter_end) + + return _DataLoaderIter(self) + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + tmp = 'train' if self.train is True else 'test' + fmt_str += ' Split: {}\n'.format(tmp) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + +class _DataLoaderIter(object): + r"""Iterates once over the DataLoader's dataset""" + + def __init__(self, loader): + self.data_iter = loader.data_iter + self.batch_size = loader.batch_size + self.n_inputs = loader.n_inputs + self.n_batches = len(loader) + + self.data_chunk = next(self.data_iter) + + def __next__(self): + if len(self.data_chunk) < self.batch_size: + try: + new_data_chunk = next(self.data_iter) + self.data_chunk = pd.concat([self.data_chunk, new_data_chunk]) + except StopIteration: + raise StopIteration + + while True: + inputs = torch.from_numpy( + self.data_chunk.head(self.batch_size)[list(range(self.n_inputs))].values).float() + targets = torch.from_numpy( + self.data_chunk.head(self.batch_size)['y'].values).long() + + self.data_chunk = self.data_chunk.iloc[self.batch_size:] + + return inputs, targets + + next = __next__ # Python 2 compatibility + + def __len__(self): + return self.n_batches + + def __iter__(self): + return self diff --git a/CNN_DEMO/utils/mnist_multitask.py b/CNN_DEMO/utils/mnist_multitask.py new file mode 100644 index 0000000..2513106 --- /dev/null +++ b/CNN_DEMO/utils/mnist_multitask.py @@ -0,0 +1,146 @@ +import torch +from torchvision import datasets, transforms +from PIL import Image + + +def load_multitask_dataset(namedataset='mnist_rotate', n_samples=1000, rotation=90, batch_size=100): + + if namedataset == 'mnist_rotate': + + DIR_DATASET = '~/data/mnist' + + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2))), + ]) + + trainset = MNIST_ROTATE(DIR_DATASET, train=True, rotation=rotation, n_samples=n_samples, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True) + + testset = MNIST_ROTATE(DIR_DATASET, train=False, rotation=rotation, n_samples=n_samples, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True) + + classes = tuple(range(10)) + n_inputs = 784 + + elif namedataset == 'mnist_permute': + + DIR_DATASET = '~/data/mnist' + + transform = transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)), + transforms.Lambda(lambda x: x.view(x.size(1)*x.size(2))), + ]) + + trainset = MNIST_PERMUTE(DIR_DATASET, train=True, n_samples=n_samples, download=True, transform=transform) + train_loader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True) + + # Make sure to use same permutation for traning and test set... + testset = MNIST_PERMUTE(DIR_DATASET, permutation=trainset.permutation, train=False, n_samples=n_samples, transform=transform) + test_loader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True) + + classes = tuple(range(10)) + n_inputs = 784 + + else: + raise ValueError('Dataset {} not recognized'.format(namedataset)) + + return train_loader, test_loader, n_inputs + + +class MNIST_ROTATE(datasets.MNIST): + + def __init__(self, root, train=True, rotation=90, n_samples=1000, transform=None, target_transform=None, download=False): + super(MNIST_ROTATE, self).__init__(root=root, train=train, transform=transform, + target_transform=target_transform, download=download) + self.rotation = rotation + + '''New torchvision convention:''' + if not hasattr(self, 'data'): + if self.train: + self.data = self.train_data + self.targets = self.train_labels + else: + self.data = self.test_data + self.targets = self.test_labels + + self.data = self.data[:n_samples] + self.targets = self.targets[:n_samples] + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is index of the target class. + """ + img, target = self.data[index], self.targets[index] + + # doing this so that it is consistent with all other datasets + # to return a PIL Image + img = Image.fromarray(img.numpy(), mode='L') + img = img.rotate(self.rotation) + + if self.transform is not None: + img = self.transform(img) + + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target + + def __len__(self): + return len(self.data) + + +class MNIST_PERMUTE(datasets.MNIST): + + def __init__(self, root, permutation=None, train=True, n_samples=1000, transform=None, target_transform=None, download=False): + super(MNIST_PERMUTE, self).__init__(root=root, train=train, transform=transform, + target_transform=target_transform, download=download) + '''New torchvision convention:''' + if not hasattr(self, 'data'): + if self.train: + self.data = self.train_data + self.targets = self.train_labels + else: + self.data = self.test_data + self.targets = self.test_labels + + if permutation is None: + self.permutation = torch.randperm(self.data.numel()//len(self.data)).long().view(-1) + else: + self.permutation = permutation + + size = self.data.size() + self.data = self.data.view(size[0], -1).index_select(1, self.permutation).view(size) + + self.data = self.data[:n_samples] + self.targets = self.targets[:n_samples] + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is index of the target class. + """ + img, target = self.data[index], self.targets[index] + + + # doing this so that it is consistent with all other datasets + # to return a PIL Image + img = Image.fromarray(img.numpy(), mode='L') + + if self.transform is not None: + img = self.transform(img) + + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target + + def __len__(self): + return len(self.data) diff --git a/CNN_DEMO/utils/mnist_tf.py b/CNN_DEMO/utils/mnist_tf.py new file mode 100644 index 0000000..879e0d8 --- /dev/null +++ b/CNN_DEMO/utils/mnist_tf.py @@ -0,0 +1,187 @@ +from __future__ import print_function +import torch.utils.data as data +from PIL import Image +import os +import os.path +import errno +import numpy as np +import torch +import codecs +from torchvision.datasets.utils import download_url + + +class MNIST_TF(data.Dataset): + """`MNIST `_ Dataset, with training and validation split as tensorflow + (i.e., the first 5000 samples of the standard training set are used as validation split) + Args: + root (string): Root directory of dataset where ``processed/training.pt`` + and ``processed/test.pt`` exist. + train (bool, optional): If True, creates dataset from ``training.pt``, + otherwise from ``test.pt``. + download (bool, optional): If true, downloads the dataset from the internet and + puts it in root directory. If dataset is already downloaded, it is not + downloaded again. + transform (callable, optional): A function/transform that takes in an PIL image + and returns a transformed version. E.g, ``transforms.RandomCrop`` + target_transform (callable, optional): A function/transform that takes in the + target and transforms it. + """ + urls = [ + 'http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz', + 'http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz', + 'http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz', + 'http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz', + ] + raw_folder = 'raw' + processed_folder = 'processed' + training_file = 'mnist_tf_training.pt' + test_file = 'mnist_tf_test.pt' + valid_file = 'mnist_tf_valid.pt' + classes = ['0 - zero', '1 - one', '2 - two', '3 - three', '4 - four', + '5 - five', '6 - six', '7 - seven', '8 - eight', '9 - nine'] + class_to_idx = {_class: i for i, _class in enumerate(classes)} + + def __init__(self, root, train=True, transform=None, target_transform=None, download=False): + self.root = os.path.expanduser(root) + self.transform = transform + self.target_transform = target_transform + self.train = train # training set or test set + + if download: + self.download() + + if not self._check_exists(): + raise RuntimeError('Dataset not found.' + + ' You can use download=True to download it') + + if self.train == True: + data_file = self.training_file + elif self.train == False: + data_file = self.test_file + else: + data_file = self.valid_file + self.data, self.targets = torch.load(os.path.join(self.root, self.processed_folder, data_file)) + + def __getitem__(self, index): + """ + Args: + index (int): Index + Returns: + tuple: (image, target) where target is index of the target class. + """ + img, target = self.data[index], self.targets[index] + + # doing this so that it is consistent with all other datasets + # to return a PIL Image + img = Image.fromarray(img.numpy(), mode='L') + + if self.transform is not None: + img = self.transform(img) + + if self.target_transform is not None: + target = self.target_transform(target) + + return img, target + + def __len__(self): + return len(self.data) + + def _check_exists(self): + return os.path.exists(os.path.join(self.root, self.processed_folder, self.training_file)) and \ + os.path.exists(os.path.join(self.root, self.processed_folder, self.test_file)) + + def download(self): + """Download the MNIST data if it doesn't exist in processed_folder already.""" + import gzip + + if self._check_exists(): + return + + # download files + try: + os.makedirs(os.path.join(self.root, self.raw_folder)) + os.makedirs(os.path.join(self.root, self.processed_folder)) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise + + for url in self.urls: + filename = url.rpartition('/')[2] + file_path = os.path.join(self.root, self.raw_folder, filename) + download_url(url, root=os.path.join(self.root, self.raw_folder), + filename=filename, md5=None) + with open(file_path.replace('.gz', ''), 'wb') as out_f, \ + gzip.GzipFile(file_path) as zip_f: + out_f.write(zip_f.read()) + os.unlink(file_path) + + # process and save as torch files + print('Processing...') + + training_set = ( + read_image_file(os.path.join(self.root, self.raw_folder, 'train-images-idx3-ubyte')), + read_label_file(os.path.join(self.root, self.raw_folder, 'train-labels-idx1-ubyte')) + ) + test_set = ( + read_image_file(os.path.join(self.root, self.raw_folder, 't10k-images-idx3-ubyte')), + read_label_file(os.path.join(self.root, self.raw_folder, 't10k-labels-idx1-ubyte')) + ) + + # Split training set in validation and training set: + valid_set = ( + training_set[0][:5000], + training_set[1][:5000] + ) + training_set = ( + training_set[0][5000:], + training_set[1][5000:] + ) + + with open(os.path.join(self.root, self.processed_folder, self.training_file), 'wb') as f: + torch.save(training_set, f) + with open(os.path.join(self.root, self.processed_folder, self.test_file), 'wb') as f: + torch.save(test_set, f) + with open(os.path.join(self.root, self.processed_folder, self.valid_file), 'wb') as f: + torch.save(valid_set, f) + + print('Done!') + + def __repr__(self): + fmt_str = 'Dataset ' + self.__class__.__name__ + '\n' + fmt_str += ' Number of datapoints: {}\n'.format(self.__len__()) + tmp = 'train' if self.train is True else 'test' + fmt_str += ' Split: {}\n'.format(tmp) + fmt_str += ' Root Location: {}\n'.format(self.root) + tmp = ' Transforms (if any): ' + fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + tmp = ' Target Transforms (if any): ' + fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp))) + return fmt_str + + + +def get_int(b): + return int(codecs.encode(b, 'hex'), 16) + + +def read_label_file(path): + with open(path, 'rb') as f: + data = f.read() + assert get_int(data[:4]) == 2049 + length = get_int(data[4:8]) + parsed = np.frombuffer(data, dtype=np.uint8, offset=8) + return torch.from_numpy(parsed).view(length).long() + + +def read_image_file(path): + with open(path, 'rb') as f: + data = f.read() + assert get_int(data[:4]) == 2051 + length = get_int(data[4:8]) + num_rows = get_int(data[8:12]) + num_cols = get_int(data[12:16]) + images = [] + parsed = np.frombuffer(data, dtype=np.uint8, offset=16) + return torch.from_numpy(parsed).view(length, num_rows, num_cols) diff --git a/CNN_DEMO/utils/nn.py b/CNN_DEMO/utils/nn.py new file mode 100644 index 0000000..2383765 --- /dev/null +++ b/CNN_DEMO/utils/nn.py @@ -0,0 +1,69 @@ +import torch +from torch import nn + +class LpNorm(nn.Module): + '''Lp-Normalization + Normalizes inputs by dividing by norm(inputs, p) + ''' + def __init__(self, p=2, eps=1e-9): + super(LpNorm, self).__init__() + self.eps = eps + self.p = p + + def forward(self, inputs): + norm = inputs.norm(self.p, -1, keepdim=True).add(self.eps) + return inputs.div(norm) + + def extra_repr(self): + return '{p}'.format(**self.__dict__) + +class BatchCenter(nn.Module): + '''Mini-batch centering + computes the component-wise average across minibatch and substracts it + ''' + def __init__(self, num_features, track_running_stats=True, use_running_stats=False): + super(BatchCenter, self).__init__() + self.num_features = num_features + + self.track_running_stats = track_running_stats + self.use_running_stats = use_running_stats + if self.track_running_stats: + self.register_buffer('running_mean', torch.zeros(num_features)) + self.register_buffer('num_batches_tracked', torch.tensor(0, dtype=torch.long)) + else: + self.register_buffer('running_mean', None) + self.register_buffer('num_batches_tracked', None) + + def reset_running_stats(self): + if self.track_running_stats: + self.running_mean.zero_() + self.num_batches_tracked.zero_() + + def forward(self, inputs): + mean = inputs.mean(0) + if self.training and self.track_running_stats: + self.num_batches_tracked += 1 + exp_avg_factor = 1/self.num_batches_tracked.float() + self.running_mean.mul_(1 - exp_avg_factor) + self.running_mean.add_(exp_avg_factor*mean) + if not self.use_running_stats: + return inputs - mean + return inputs - self.running_mean + + def extra_repr(self): + return '{num_features}, track_running_stats={track_running_stats}'.format(**self.__dict__) + +class Flatten(nn.Module): + r"""Reshapes the input tensor as a 2d tensor, where the size of the first (batch) dimension is preserved. + + Inputs: input + - **input** (batch, num_dim1, num_dim1,...): tensor containing input features + + Outputs: output + - **output'**: (batch, num_dim1*num_dim2*...): tensor containing the output + """ + def __init__(self): + super(Flatten, self).__init__() + + def forward(self, x): + return x.view(x.size(0), -1) diff --git a/CNN_DEMO/utils/utils.py b/CNN_DEMO/utils/utils.py new file mode 100644 index 0000000..e132084 --- /dev/null +++ b/CNN_DEMO/utils/utils.py @@ -0,0 +1,234 @@ +from itertools import product +from collections import namedtuple, MutableMapping +from random import randint +import os +import time +import torch +import copy +import sys +try: + from urllib.request import urlopen # For Python 3.0 and later +except ImportError: + from urllib2 import urlopen # Fall back to Python 2's urllib + + +class ddict(object): + ''' + dd = ddict(lr=[0.1, 0.2], n_hiddens=[100, 500, 1000], n_layers=2) + # Save to shelf: + dd._save('my_file', date=False) + # Load ddict from file: + new_dd = ddict()._load('my_file') + ''' + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __add__(self, other): + if isinstance(other, type(self)): + sum_dct = copy.copy(self.__dict__) + for k,v in other.__dict__.items(): + if k not in sum_dct: + sum_dct[k] = v + else: + if type(v) is list and type(sum_dct[k]) is list: + sum_dct[k] = sum_dct[k] + v + elif type(v) is not list and type(sum_dct[k]) is list: + sum_dct[k] = sum_dct[k] + [v] + elif type(v) is list and type(sum_dct[k]) is not list: + sum_dct[k] = [sum_dct[k]] + v + else: + sum_dct[k] = [sum_dct[k]] + [v] + return ddict(**sum_dct) + + elif isinstance(other, dict): + return self.__add__(ddict(**other)) + else: + raise ValueError("ddict or dict is required") + + def __radd__(self, other): + return self.__add__(other) + + def __repr__(self): + items = ("{}={!r}".format(k, self.__dict__[k]) for k in self._keys()) + return "{}({})".format(type(self).__name__, ", ".join(items)) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __iter__(self): + return self.__dict__.__iter__() + + def __len__(self): + return len(self.__dict__) + + def __setitem__(self, key, value): + self.__dict__[key] = value + + def __getitem__(self, key): + return self.__dict__[key] + + def _keys(self): + return tuple(sorted([k for k in self.__dict__ if not k.startswith('_')])) + + def _values(self): + return tuple([self.__dict__[k] for k in self._keys()]) + + def _items(self): + return tuple(zip(self._keys(), self._values())) + + def _save(self, filename=None, date=True): + if filename is None: + if not hasattr(self, '_filename'): # First save + raise ValueError("filename must be provided the first time you call _save()") + else: # Already saved + torch.save(self, self._filename + '.pt') + else: # New filename + if date: + filename += '_'+time.strftime("%Y%m%d-%H:%M:%S") + # Check if filename does not already exist. If it does, change name. + while os.path.exists(filename + '.pt') and len(filename) < 100: + filename += str(randint(0,9)) + self._filename = filename + torch.save(self, self._filename + '.pt') + return self + + def _load(self, filename): + try: + self = torch.load(filename) + except FileNotFoundError: + self = torch.load(filename + '.pt') + return self + + def _to_dict(self): + "Returns a dict (it's recursive)" + return_dict = {} + for k,v in self.__dict__.items(): + if isinstance(v, type(self)): + return_dict[k] = v._to_dict() + else: + return_dict[k] = v + return return_dict + + @staticmethod + def _flatten_dict(d, parent_key='', sep='_'): + "Recursively flattens nested dicts" + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, MutableMapping): + items.extend(ddict._flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + def _flatten(self, parent_key='', sep='_'): + "Recursively flattens nested ddicts" + d = self._to_dict() + return ddict._flatten_dict(d) + + +class Gridder(object): + ''' + g = Gridder(**{ + 'lr': 0.1, + 'n_hiddens': [100, 500, 1000], + 'repetitions': list(range(5))}) + h = Gridder(lr=[0.1, 0.2], n_hiddens=[100, 500, 1000], n_layers=2) + ''' + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.__grid__ = None + self.__tuple__ = None + self.__normalize__() + + def __normalize__(self): + "Checks that all values are lists (and not for instance individual instances)" + for k in self.__keys__(): + if type(self.__dict__[k]) is not list: + self.__dict__[k] = [self.__dict__[k]] + + def __add__(self, other): + if isinstance(other, type(self)): + uniq_keys = set(self.__keys__() + other.__keys__()) + sum_dct = {} + for k in uniq_keys: + vals = [] + try: + vals += self.__dict__[k] + except KeyError: + pass + try: + vals += other.__dict__[k] + except KeyError: + pass + sum_dct[k] = list(set(vals)) + return Gridder(**sum_dct) + + elif isinstance(other, dict): + return self.__add__(Gridder(**other)) + else: + raise ValueError("dict or Gridder is required") + + def __radd__(self, other): + return self.__add__(other) + + def __keys__(self): + keys = sorted([k for k in self.__dict__ + if not k.startswith('__') and not k.endswith('__')]) + return keys + + def __values__(self): + return [self.__dict__[k] for k in self.__keys__()] + + def __repr__(self): + keys = self.__keys__() + items = ("{}={!r}".format(k, self.__dict__[k]) for k in keys) + return "{}({})".format(type(self).__name__, ", ".join(items)) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __next__(self): + if self.__grid__ is None: + self.__grid__ = product(*self.__values__()) + self.__tuple__ = namedtuple('params', self.__keys__()) + return self.__tuple__(*next(self.__grid__)) + + def __iter__(self): + return self + + def __len__(self): + return len(self.__keys__()) + + +def download_progress(url, dest_folder): + + print('Downloading ' + url) + + with open(os.path.join(dest_folder, os.path.basename(url)), "wb") as local_file: + + def chunk_report(bytes_so_far, chunk_size, total_size): + percent = float(bytes_so_far) / total_size + percent = round(percent*100, 2) + sys.stdout.write(" Downloaded %d of %d kB (%0.2f%%)\r" % + (bytes_so_far/1024, total_size/1024, percent)) + if bytes_so_far >= total_size: + sys.stdout.write('\n') + + def chunk_read(response, chunk_size=102400, report_hook=None): + total_size = response.info().get('Content-Length').strip() + total_size = int(total_size) + bytes_so_far = 0 + while 1: + chunk = response.read(chunk_size) + local_file.write(chunk) + + bytes_so_far += len(chunk) + if not chunk: + break + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size) + return bytes_so_far + + response = urlopen(url) + chunk_read(response, report_hook=chunk_report) diff --git a/README.md b/README.md index 037c9e6..84d6cc9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ -Beyond Backprop: Online Alternating Minimization with Auxiliary Variables -Source code for the paper + +# [Beyond Backprop: Online Alternating Minimization with Auxiliary Variables](http://proceedings.mlr.press/v97/choromanska19a/choromanska19a.pdf) +This repository hosts source code for the CNN experiments published in this ICML paper. @inproceedings{BeyondBackpropAM,