change text_processing to text_helper

models.py

@@ -42,16 +42,18 @@ def __init__(self, qst_vocab_size, word_embed_size, embed_size, num_layers, hidd
         super(QstEncoder, self).__init__()
         self.word2vec = nn.Embedding(qst_vocab_size, word_embed_size)
         self.tanh = nn.Tanh()
-        self.lstm = nn.LSTM(word_embed_size, hidden_size, num_layers)
+        self.lstm = nn.LSTM(word_embed_size, hidden_size, num_layers, bidirectional=True)
         self.fc = nn.Linear(2*num_layers*hidden_size, embed_size)     # 2 for hidden and cell states
 
     def forward(self, question):
 
         qst_vec = self.word2vec(question)                             # [batch_size, max_qst_length=30, word_embed_size=300]
         qst_vec = self.tanh(qst_vec)
         qst_vec = qst_vec.transpose(0, 1)                             # [max_qst_length=30, batch_size, word_embed_size=300]
+        self.lstm.flatten_parameters()
         _, (hidden, cell) = self.lstm(qst_vec)                        # [num_layers=2, batch_size, hidden_size=512]
-        qst_feature = torch.cat((hidden, cell), 2)                    # [num_layers=2, batch_size, 2*hidden_size=1024]
+#         qst_feature = torch.cat((hidden, cell), 2)                    # [num_layers=2, batch_size, 2*hidden_size=1024]
+        qst_feature = hidden
         qst_feature = qst_feature.transpose(0, 1)                     # [batch_size, num_layers=2, 2*hidden_size=1024]
         qst_feature = qst_feature.reshape(qst_feature.size()[0], -1)  # [batch_size, 2*num_layers*hidden_size=2048]
         qst_feature = self.tanh(qst_feature)
@@ -67,7 +69,9 @@ def __init__(self, embed_size, qst_vocab_size, ans_vocab_size, word_embed_size,
         super(VqaModel, self).__init__()
         self.img_encoder = ImgEncoder(embed_size)
         self.qst_encoder = QstEncoder(qst_vocab_size, word_embed_size, embed_size, num_layers, hidden_size)
-        self.tanh = nn.Tanh()
+#         self.tanh = nn.Tanh()
+        self.ReLU = nn.ReLU()
+        self.batch_norm = nn.BatchNorm1d(embed_size)
         self.dropout = nn.Dropout(0.5)
         self.fc1 = nn.Linear(embed_size, ans_vocab_size)
         self.fc2 = nn.Linear(ans_vocab_size, ans_vocab_size)
@@ -77,10 +81,11 @@ def forward(self, img, qst):
         img_feature = self.img_encoder(img)                     # [batch_size, embed_size]
         qst_feature = self.qst_encoder(qst)                     # [batch_size, embed_size]
         combined_feature = torch.mul(img_feature, qst_feature)  # [batch_size, embed_size]
-        combined_feature = self.tanh(combined_feature)
+        combined_feature = self.batch_norm(combined_feature)
+        combined_feature = self.ReLU(combined_feature)
         combined_feature = self.dropout(combined_feature)
         combined_feature = self.fc1(combined_feature)           # [batch_size, ans_vocab_size=1000]
-        combined_feature = self.tanh(combined_feature)
+        combined_feature = self.ReLU(combined_feature)
         combined_feature = self.dropout(combined_feature)
         combined_feature = self.fc2(combined_feature)           # [batch_size, ans_vocab_size=1000]
 

train.py

@@ -1,14 +1,19 @@
 import os
 import argparse
 import numpy as np
+os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
+os.environ['CUDA_VISIBLE_DEVICES'] = "0,1,2,3"
 import torch
 import torch.nn as nn
 import torch.optim as optim
 from torch.optim import lr_scheduler
 from data_loader import get_loader
 from models import VqaModel
 
-
+if torch.cuda.is_available() :
+    print("cuda is True")
+else :
+    print("cuda is False")
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 
 
@@ -28,22 +33,27 @@ def main(args):
 
     qst_vocab_size = data_loader['train'].dataset.qst_vocab.vocab_size
     ans_vocab_size = data_loader['train'].dataset.ans_vocab.vocab_size
-    ans_unk_idx = data_loader['train'].dataset.ans_vocab.unk2idx
+    ans_unk_idx    = data_loader['train'].dataset.ans_vocab.unk2idx
 
     model = VqaModel(
         embed_size=args.embed_size,
         qst_vocab_size=qst_vocab_size,
         ans_vocab_size=ans_vocab_size,
         word_embed_size=args.word_embed_size,
         num_layers=args.num_layers,
-        hidden_size=args.hidden_size).to(device)
+        hidden_size=args.hidden_size)
+
+    if torch.cuda.device_count() > 1:
+        model = nn.DataParallel(model)
+    model.to(device)
+
 
     criterion = nn.CrossEntropyLoss()
 
-    params = list(model.img_encoder.fc.parameters()) \
-        + list(model.qst_encoder.parameters()) \
-        + list(model.fc1.parameters()) \
-        + list(model.fc2.parameters())
+    params = list(model.module.img_encoder.fc.parameters()) \
+        + list(model.module.qst_encoder.parameters()) \
+        + list(model.module.fc1.parameters()) \
+        + list(model.module.fc2.parameters())
 
     optimizer = optim.Adam(params, lr=args.learning_rate)
     scheduler = lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma)

tutorials/check_vggnet.ipynb

@@ -156,7 +156,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.5.6"
+   "version": "3.6.9"
   }
  },
  "nbformat": 4,

utils/build_vqa_inputs.py

@@ -37,7 +37,7 @@ def vqa_processing(image_dir, annotation_file, question_file, valid_answer_set,
         image_name = image_name_template % image_id
         image_path = os.path.join(abs_image_dir, image_name+'.jpg')
         question_str = q['question']
-        question_tokens = text_processing.tokenize(question_str)
+        question_tokens = text_helper.tokenize(question_str)
 
         iminfo = dict(image_name=image_name,
                       image_path=image_path,
@@ -66,7 +66,7 @@ def main(args):
     question_file = args.input_dir+'/Questions/v2_OpenEnded_mscoco_%s_questions.json'
 
     vocab_answer_file = args.output_dir+'/vocab_answers.txt'
-    answer_dict = text_processing.VocabDict(vocab_answer_file)
+    answer_dict = text_helper.VocabDict(vocab_answer_file)
     valid_answer_set = set(answer_dict.word_list)    
 
     train = vqa_processing(image_dir, annotation_file, question_file, valid_answer_set, 'train2014')