From 2de30932d1524a7521ddb942ad9584c897402d95 Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Tue, 23 Apr 2019 17:16:16 +0800 Subject: [PATCH 01/12] scikit-learn version change --- requirements.txt | Bin 1414 -> 1410 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 41578722595c23bede689d011519e3083cd7c7a7..64041ffba91767eed62c0de6e06d9156f08d8914 100644 GIT binary patch delta 19 acmZqUZsOh`z{+mKV8EcqU_4ozH5&jZ{{zSX delta 24 fcmZqTZsXn{z{+mOV9B7zU^rQvHJgQ(fr|kEJe33K From b31ccd2775b12290e8535ac4afad734f0311d02d Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Tue, 23 Apr 2019 17:30:12 +0800 Subject: [PATCH 02/12] change repository link --- ec2deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2deploy.py b/ec2deploy.py index cec3311..f5e65b2 100755 --- a/ec2deploy.py +++ b/ec2deploy.py @@ -28,7 +28,7 @@ apt update apt install -y awscli docker.io zip cd /home/ubuntu -git clone https://github.com/Aishwarya26l/lambda-nbconvert.git +git clone https://github.com/NUS-ALSET/lambda-nbconvert.git cd lambda-nbconvert ./scripts/deploy.sh '{bucket_name}' '{stack_name}' '{region}' '{enable_cors}' cat<<'END' From 87bd958927a5bb1dd3c55d0871e4998c7575dce9 Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Fri, 21 Jun 2019 11:32:53 +0800 Subject: [PATCH 03/12] Added tensorflow-hub and nltk --- ec2deploy.py | 14 ++++++-------- requirements.txt | Bin 1410 -> 1482 bytes 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ec2deploy.py b/ec2deploy.py index f5e65b2..262e0dc 100755 --- a/ec2deploy.py +++ b/ec2deploy.py @@ -10,8 +10,8 @@ parser.add_argument('enable_cors') args = parser.parse_args() -#comment occurences of AWS_SESSION_TOKEN and session_token -#if you are not using an AWS Educate Account +# comment occurences of AWS_SESSION_TOKEN and session_token +# if you are not using an AWS Educate Account session = boto3.Session() access_key = session.get_credentials().access_key @@ -19,7 +19,7 @@ session_token = session.get_credentials().token ec2 = session.resource('ec2') -start_up_script="""#!/usr/bin/env bash +start_up_script = """#!/usr/bin/env bash export AWS_ACCESS_KEY_ID='{access_key}' export AWS_SECRET_ACCESS_KEY='{secret_key}' export AWS_SESSION_TOKEN='{session_token}' @@ -54,7 +54,7 @@ print('Instance {} was started. Waiting for output ...'.format(i.instance_id)) -prev_output='' +prev_output = '' while True: time.sleep(10) o = i.console_output() @@ -63,16 +63,14 @@ if output != prev_output: print('...\n'*3) print(output) - prev_output=output + prev_output = output if i.state['Name'] == 'terminated': break -success_string='Stack {} was successfully deployed'.format(args.stack_name) +success_string = 'Stack {} was successfully deployed'.format(args.stack_name) print(('#'*60+'\n')*3) if success_string in prev_output: print(success_string) else: print('There were some problems. See above output for more details') - - diff --git a/requirements.txt b/requirements.txt index 64041ffba91767eed62c0de6e06d9156f08d8914..73834925cae6df3c74cccaeba043c72d78993c87 100644 GIT binary patch delta 80 zcmZqTKE=JEiPeLbfr}xJA%~%aA)CP#2#p!^7)*fJhyf&1!jKA-EoR7PC}Kzhs>)|5 UXNYIWU?^os0;@KFsy1K%0AA(|lK=n! delta 7 OcmX@b-Ne12i4_0~o&uTx From a6f1d69acbde859ccca10cc1d7acb357f553fcc9 Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Fri, 21 Jun 2019 17:02:34 +0800 Subject: [PATCH 04/12] Updating tensorflow version --- ec2deploy.py | 2 +- requirements.txt | Bin 1482 -> 1482 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/ec2deploy.py b/ec2deploy.py index 262e0dc..1c9f7b6 100755 --- a/ec2deploy.py +++ b/ec2deploy.py @@ -28,7 +28,7 @@ apt update apt install -y awscli docker.io zip cd /home/ubuntu -git clone https://github.com/NUS-ALSET/lambda-nbconvert.git +git clone https://github.com/Aishwarya26l/lambda-nbconvert.git cd lambda-nbconvert ./scripts/deploy.sh '{bucket_name}' '{stack_name}' '{region}' '{enable_cors}' cat<<'END' diff --git a/requirements.txt b/requirements.txt index 73834925cae6df3c74cccaeba043c72d78993c87..726b8fe8c43423e3b7130616c293492958e9c85f 100644 GIT binary patch delta 13 UcmX@beTsWS4J)I` Date: Fri, 21 Jun 2019 17:14:15 +0800 Subject: [PATCH 05/12] Adding tensflow requirements --- requirements.txt | Bin 1482 -> 1482 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 726b8fe8c43423e3b7130616c293492958e9c85f..678c67b53e0e697d7e7a5c9bc359809dc60febfe 100644 GIT binary patch delta 13 VcmX@beTsX-D;7r6$*)=N0{|$F1>gVx delta 15 XcmX@beTsX-E0)O(tZb7%u-pOwHg5*V From 7e772ac8236abf0d962a19b718f4e4463819c790 Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Thu, 27 Jun 2019 14:31:30 +0800 Subject: [PATCH 06/12] Adding results as a json in post response --- index.html | 2 +- main.py | 54 +++++++++++++++++++++++++++++------------------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/index.html b/index.html index 6b65aaa..5205b76 100644 --- a/index.html +++ b/index.html @@ -149,7 +149,7 @@

Test Results console.log(userInfo.duration, "seconds of execution time.") console.log(userInfo,"userInfo") document.getElementById("responseJSON").value = JSON.stringify(userInfo.ipynb); - document.getElementById("resultJSON").value = JSON.stringify(userInfo.result); + document.getElementById("resultJSON").value = JSON.stringify(userInfo.results,null,'\t'); // pass data to component via problem global object window.problem = userInfo.ipynb; diff --git a/main.py b/main.py index 413f546..5bebc81 100644 --- a/main.py +++ b/main.py @@ -1,3 +1,10 @@ +from timeit import default_timer as timer +import base64 +import json +from io import StringIO +from nbconvert.preprocessors import ExecutePreprocessor +import nbformat +import logging import os import sys import ast @@ -18,35 +25,27 @@ sys.path.append(BUILD_DIR) sys.path.append(OVERLAY_DIR) -os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + ":" + CURRENT_DIR + ":" + BUILD_DIR + ":" + OVERLAY_DIR +os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + \ + ":" + CURRENT_DIR + ":" + BUILD_DIR + ":" + OVERLAY_DIR - - -import logging -import nbformat -from nbconvert.preprocessors import ExecutePreprocessor -from io import StringIO -import json -import base64 - -from timeit import default_timer as timer logger = logging.getLogger(__name__) def return_result_json(): os.chdir('/tmp') if(os.path.isfile('results.json')): - result_file_reader = open('results.json','r') + result_file_reader = open('results.json', 'r') result_file = result_file_reader.read() return result_file + def base64_decode_and_persist(filename, contents): os.chdir('/tmp') decoded = base64.b64decode(contents) with open(filename, 'wb') as outfile: outfile.write(decoded) - + def save_files_to_temp_dir(files): for key in files: @@ -66,7 +65,8 @@ def execute_notebook(source): nb = nbformat.read(in_memory_source, as_version=4) logger.debug("Launching kernels") - ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=True) + ep = ExecutePreprocessor( + timeout=600, kernel_name='python3', allow_errors=True) ep.preprocess(nb, {'metadata': {'path': '/tmp/'}}) ex = StringIO() @@ -75,14 +75,19 @@ def execute_notebook(source): logger.debug("Returning results") return ex.getvalue() + homepage = "" with open('index.html') as f: - homepage=f.read() + homepage = f.read() + def populate_cors(response): - response['headers']['Access-Control-Allow-Origin'] = os.environ.get('CORS_DOMAIN', '*') - response['headers']['Access-Control-Allow-Headers'] = os.environ.get('CORS_HEADERS', 'Content-Type, Authorization, X-Amz-Date, X-Api-Key, X-Amz-Security-Token') - response['headers']['Access-Control-Allow-Methods'] = os.environ.get('CORS_METHODS', 'DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT') + response['headers']['Access-Control-Allow-Origin'] = os.environ.get( + 'CORS_DOMAIN', '*') + response['headers']['Access-Control-Allow-Headers'] = os.environ.get( + 'CORS_HEADERS', 'Content-Type, Authorization, X-Amz-Date, X-Api-Key, X-Amz-Security-Token') + response['headers']['Access-Control-Allow-Methods'] = os.environ.get( + 'CORS_METHODS', 'DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT') def handler(event, context): @@ -106,18 +111,17 @@ def handler(event, context): 'Content-Type': 'application/json', } } - populate_cors(response) - return response + populate_cors(response) + return response else: request_body = json.loads(event["body"]) notebook_source = request_body['notebook'] - attached_files = request_body['files'] if 'files' in request_body else {} + attached_files = request_body['files'] if 'files' in request_body else { + } save_files_to_temp_dir(attached_files) - - - start = timer() + start = timer() result = execute_notebook(json.dumps(notebook_source)) result = json.loads(result) @@ -127,7 +131,7 @@ def handler(event, context): duration = end - start response = { "statusCode": 200, - "body": json.dumps({"duration": duration,"ipynb": result, "result": exec_result}), + "body": json.dumps({"duration": duration, "ipynb": result, "result": exec_result, "results": json.loads(exec_result)}), "headers": { 'Content-Type': 'application/json', } From a52c7ca75e6efceaa6cb079fcba27c662f8e202b Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Thu, 27 Jun 2019 15:03:51 +0800 Subject: [PATCH 07/12] Reorder imports --- main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 5bebc81..1a661f6 100644 --- a/main.py +++ b/main.py @@ -1,14 +1,15 @@ -from timeit import default_timer as timer -import base64 -import json -from io import StringIO -from nbconvert.preprocessors import ExecutePreprocessor -import nbformat -import logging import os import sys import ast import tempfile +import logging +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor +from io import StringIO +import json +import base64 + +from timeit import default_timer as timer CURRENT_DIR = os.getcwd() BUILD_DIR = os.path.join(os.getcwd(), "build", "code") @@ -55,7 +56,6 @@ def save_files_to_temp_dir(files): def execute_notebook(source): """ - :param source: Jupyter Notebook :return: Result of the notebook invocation """ @@ -131,7 +131,7 @@ def handler(event, context): duration = end - start response = { "statusCode": 200, - "body": json.dumps({"duration": duration, "ipynb": result, "result": exec_result, "results": json.loads(exec_result)}), + "body": json.dumps({"duration": duration, "ipynb": result, "result": exec_result}), "headers": { 'Content-Type': 'application/json', } From 87256cf3d7bb7b4cd80db2b6f4b36be8957df3ff Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Thu, 27 Jun 2019 15:16:47 +0800 Subject: [PATCH 08/12] Reordered imports --- main.py | 50 ++++++++++++++++++++------------------------------ 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/main.py b/main.py index 1a661f6..acf2c2a 100644 --- a/main.py +++ b/main.py @@ -2,14 +2,6 @@ import sys import ast import tempfile -import logging -import nbformat -from nbconvert.preprocessors import ExecutePreprocessor -from io import StringIO -import json -import base64 - -from timeit import default_timer as timer CURRENT_DIR = os.getcwd() BUILD_DIR = os.path.join(os.getcwd(), "build", "code") @@ -26,27 +18,32 @@ sys.path.append(BUILD_DIR) sys.path.append(OVERLAY_DIR) -os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + \ - ":" + CURRENT_DIR + ":" + BUILD_DIR + ":" + OVERLAY_DIR +os.environ['PYTHONPATH'] = os.environ['PYTHONPATH'] + ":" + CURRENT_DIR + ":" + BUILD_DIR + ":" + OVERLAY_DIR +import logging +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor +from io import StringIO +import json +import base64 +from timeit import default_timer as timer logger = logging.getLogger(__name__) def return_result_json(): os.chdir('/tmp') if(os.path.isfile('results.json')): - result_file_reader = open('results.json', 'r') + result_file_reader = open('results.json','r') result_file = result_file_reader.read() return result_file - def base64_decode_and_persist(filename, contents): os.chdir('/tmp') decoded = base64.b64decode(contents) with open(filename, 'wb') as outfile: outfile.write(decoded) - + def save_files_to_temp_dir(files): for key in files: @@ -65,8 +62,7 @@ def execute_notebook(source): nb = nbformat.read(in_memory_source, as_version=4) logger.debug("Launching kernels") - ep = ExecutePreprocessor( - timeout=600, kernel_name='python3', allow_errors=True) + ep = ExecutePreprocessor(timeout=600, kernel_name='python3', allow_errors=True) ep.preprocess(nb, {'metadata': {'path': '/tmp/'}}) ex = StringIO() @@ -75,19 +71,14 @@ def execute_notebook(source): logger.debug("Returning results") return ex.getvalue() - homepage = "" with open('index.html') as f: - homepage = f.read() - + homepage=f.read() def populate_cors(response): - response['headers']['Access-Control-Allow-Origin'] = os.environ.get( - 'CORS_DOMAIN', '*') - response['headers']['Access-Control-Allow-Headers'] = os.environ.get( - 'CORS_HEADERS', 'Content-Type, Authorization, X-Amz-Date, X-Api-Key, X-Amz-Security-Token') - response['headers']['Access-Control-Allow-Methods'] = os.environ.get( - 'CORS_METHODS', 'DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT') + response['headers']['Access-Control-Allow-Origin'] = os.environ.get('CORS_DOMAIN', '*') + response['headers']['Access-Control-Allow-Headers'] = os.environ.get('CORS_HEADERS', 'Content-Type, Authorization, X-Amz-Date, X-Api-Key, X-Amz-Security-Token') + response['headers']['Access-Control-Allow-Methods'] = os.environ.get('CORS_METHODS', 'DELETE,GET,HEAD,OPTIONS,PATCH,POST,PUT') def handler(event, context): @@ -111,17 +102,16 @@ def handler(event, context): 'Content-Type': 'application/json', } } - populate_cors(response) - return response + populate_cors(response) + return response else: request_body = json.loads(event["body"]) notebook_source = request_body['notebook'] - attached_files = request_body['files'] if 'files' in request_body else { - } + attached_files = request_body['files'] if 'files' in request_body else {} save_files_to_temp_dir(attached_files) - start = timer() + start = timer() result = execute_notebook(json.dumps(notebook_source)) result = json.loads(result) @@ -131,7 +121,7 @@ def handler(event, context): duration = end - start response = { "statusCode": 200, - "body": json.dumps({"duration": duration, "ipynb": result, "result": exec_result}), + "body": json.dumps({"duration": duration,"ipynb": result, "result": exec_result, "results":json.loads(exec_result)}), "headers": { 'Content-Type': 'application/json', } From 5a90353eaed34fd7da60c6be576ce882046e17d9 Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Thu, 27 Jun 2019 15:35:12 +0800 Subject: [PATCH 09/12] Error handling added --- main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index acf2c2a..055d49f 100644 --- a/main.py +++ b/main.py @@ -116,12 +116,17 @@ def handler(event, context): result = json.loads(result) exec_result = return_result_json() - + print(exec_result) + print(type(exec_result)) + if(exec_result and isinstance(exec_result, str)): + exec_result_loaded = json.loads(exec_result) + else: + exec_result_loaded = exec_result end = timer() duration = end - start response = { "statusCode": 200, - "body": json.dumps({"duration": duration,"ipynb": result, "result": exec_result, "results":json.loads(exec_result)}), + "body": json.dumps({"duration": duration,"ipynb": result, "result": exec_result, "results":exec_result_loaded}), "headers": { 'Content-Type': 'application/json', } From 3a78b5bb8a14c4e1b29b6ecee95a1071deb9134e Mon Sep 17 00:00:00 2001 From: Aishwarya Lakshminarasimhan Date: Thu, 27 Jun 2019 16:43:53 +0800 Subject: [PATCH 10/12] Added networkx package --- requirements.txt | Bin 1482 -> 1516 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 678c67b53e0e697d7e7a5c9bc359809dc60febfe..59bf9a77aa50aee9d565466e39b056cbfb2f5da3 100644 GIT binary patch delta 42 ucmX@b{f2wPDOM$31}=s?hE#?UhH{2{h9ZV+h6)B-AT(moV=x9{0|o%)d Date: Wed, 3 Jul 2019 14:14:53 +0800 Subject: [PATCH 11/12] Added sentencepiece manager package --- requirements.txt | Bin 1516 -> 1562 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 59bf9a77aa50aee9d565466e39b056cbfb2f5da3..6a6c8c7f9acb595b95a29c86418162c59f9ab04d 100644 GIT binary patch delta 54 ycmaFEJ&R|<8&*AD1}=tThE#?;h7vHD3?vH}GJzzBWebD`40;TPKy1Na!~g(e Date: Mon, 23 Sep 2019 14:21:42 +0800 Subject: [PATCH 12/12] Added requests package --- requirements.txt | Bin 1562 -> 1598 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6a6c8c7f9acb595b95a29c86418162c59f9ab04d..740b9f60ca2e563621530ce3a6a105bbbf60e5b9 100644 GIT binary patch delta 44 rcmbQmvyW%P6lVD%hE#?^hEgC|%uoWvwm@japa;Yt(qMBxb2k$J_2&t} delta 12 TcmdnTGmB@#6z0u4nA?~DAp8Wb