From 9c41e363e531eb94a67d4c926d800624e1cd710d Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 10:43:31 +0000 Subject: [PATCH 01/11] Done --- q01_load_data/build.py | 7 ++++++- q01_load_data/tests/test_sol.pkl | Bin 0 -> 79 bytes q01_load_data/tests/user_sol.pkl | Bin 0 -> 67 bytes 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 q01_load_data/tests/test_sol.pkl create mode 100644 q01_load_data/tests/user_sol.pkl diff --git a/q01_load_data/build.py b/q01_load_data/build.py index a29c139..ee38216 100644 --- a/q01_load_data/build.py +++ b/q01_load_data/build.py @@ -1,7 +1,12 @@ +# %load q01_load_data/build.py import pandas as pd import numpy as np from sklearn.model_selection import train_test_split +def q01_load_data(path): + df = pd.read_csv(path) + df['Datetime'] = pd.to_datetime(df['Datetime']) + return df.shape,df +q01_load_data('data/elecdemand.csv') - diff --git a/q01_load_data/tests/test_sol.pkl b/q01_load_data/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7912fb668f4a4bff9f60d47546462bb183207435 GIT binary patch literal 79 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{S&&f|r W0g9I->LrzC=A>|;h;bD%=m7w&xE!N{S&&f|r0g9I->LrzC=A>|; Kh;bD%=m7vu+!yBn literal 0 HcmV?d00001 From caf00b136967d25a75defcde01b76b2f86d8d406 Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 10:57:15 +0000 Subject: [PATCH 02/11] Done --- q02_data_splitter/build.py | 11 ++++++++++- q02_data_splitter/tests/test_sol.pkl | Bin 0 -> 87 bytes q02_data_splitter/tests/user_sol.pkl | Bin 0 -> 75 bytes 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 q02_data_splitter/tests/test_sol.pkl create mode 100644 q02_data_splitter/tests/user_sol.pkl diff --git a/q02_data_splitter/build.py b/q02_data_splitter/build.py index b6c715f..07fe25e 100644 --- a/q02_data_splitter/build.py +++ b/q02_data_splitter/build.py @@ -1,7 +1,16 @@ +# %load q02_data_splitter/build.py import pandas as pd import numpy as np from sklearn.model_selection import TimeSeriesSplit from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data +def q02_data_splitter(path): + np.random.seed(9) + df_shape,df = q01_load_data(path) + time_series_Split = TimeSeriesSplit(n_splits=2) + train = time_series_Split.split(X=df) + return list(train) + +q02_data_splitter('data/elecdemand.csv') + - diff --git a/q02_data_splitter/tests/test_sol.pkl b/q02_data_splitter/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a3e9cc57e21382b4149f86a10e44fd859a5d4213 GIT binary patch literal 87 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D+IDr5=#=} ZiwklxOG;9U^pZ+5b5gi4#JLI?^Z>TKAY=dl literal 0 HcmV?d00001 diff --git a/q02_data_splitter/tests/user_sol.pkl b/q02_data_splitter/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..70c23336a7d13cc99ae68a1dcdbdc3e6646e261a GIT binary patch literal 75 zcmZo*PA Date: Thu, 3 Jan 2019 11:05:51 +0000 Subject: [PATCH 03/11] Done --- q03_time_plot/build.py | 7 +++++++ test_sol.pkl | Bin 0 -> 79 bytes user_sol.pkl | Bin 0 -> 67 bytes 3 files changed, 7 insertions(+) create mode 100644 test_sol.pkl create mode 100644 user_sol.pkl diff --git a/q03_time_plot/build.py b/q03_time_plot/build.py index bf18743..430cb62 100644 --- a/q03_time_plot/build.py +++ b/q03_time_plot/build.py @@ -1,7 +1,14 @@ +# %load q03_time_plot/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +import datetime + +def q03_time_plot(path): + df_shape,df = q01_load_data(path) + plt.plot(df['Datetime'],df['Demand']) +q03_time_plot('data/elecdemand.csv') diff --git a/test_sol.pkl b/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..cec104b8f2d3d8445aa4256f364f0ca31036739d GIT binary patch literal 79 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{U2TK>^ UN{U2TK>^ Date: Thu, 3 Jan 2019 11:26:06 +0000 Subject: [PATCH 04/11] Done --- q04_boxplot/build.py | 10 +++++++++- q04_boxplot/tests/test_sol.pkl | Bin 0 -> 75 bytes q04_boxplot/tests/user_sol.pkl | Bin 0 -> 63 bytes 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 q04_boxplot/tests/test_sol.pkl create mode 100644 q04_boxplot/tests/user_sol.pkl diff --git a/q04_boxplot/build.py b/q04_boxplot/build.py index c69f931..1a2a143 100644 --- a/q04_boxplot/build.py +++ b/q04_boxplot/build.py @@ -1,7 +1,15 @@ +# %load q04_boxplot/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +import seaborn as sns + +def q04_boxplot(path): + df_shape,df = q01_load_data(path) + sns.boxplot(x='WorkDay',y='Demand',data=df) + +q04_boxplot('data/elecdemand.csv') + - diff --git a/q04_boxplot/tests/test_sol.pkl b/q04_boxplot/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f863f64a8c994210960bcdfa9c645b392115b34a GIT binary patch literal 75 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{PPs*<- V$jL9!ODfIGN#Q~g;VNX%0{}mk8<+q9 literal 0 HcmV?d00001 diff --git a/q04_boxplot/tests/user_sol.pkl b/q04_boxplot/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..44dda795407bb9cf236e1b00c6328005eb47a0ac GIT binary patch literal 63 zcmZo*PAN{PPs*<-$jL9!ODfIGN#Q~g J;VNX%0|4{i7Rvwt literal 0 HcmV?d00001 From d89afb7d525659bc63117d6654b4fa111947919c Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 11:29:10 +0000 Subject: [PATCH 05/11] Done --- q05_feature_engineering/build.py | 5 +++++ q05_feature_engineering/tests/test_sol.pkl | Bin 0 -> 99 bytes q05_feature_engineering/tests/user_sol.pkl | Bin 0 -> 87 bytes 3 files changed, 5 insertions(+) create mode 100644 q05_feature_engineering/tests/test_sol.pkl create mode 100644 q05_feature_engineering/tests/user_sol.pkl diff --git a/q05_feature_engineering/build.py b/q05_feature_engineering/build.py index 97e29e7..4799134 100644 --- a/q05_feature_engineering/build.py +++ b/q05_feature_engineering/build.py @@ -1,9 +1,14 @@ +# %load q05_feature_engineering/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering(path): + df_shape,df = q01_load_data(path) + plt.scatter(df['Temperature'],df['Demand']) +q05_feature_engineering('data/elecdemand.csv') diff --git a/q05_feature_engineering/tests/test_sol.pkl b/q05_feature_engineering/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c8990f6faf7b3093f4ad2a9063625b47c6c393dd GIT binary patch literal 99 zcmZ|F$q9fk5Cu@do5c)4u%02yH;iO4+aWhg*9JVl_pH`eOx<$~VyH_5H&j7mrzfkO e@0u^zPDLGjLibU@NL~`YmOPmu!EHafm4#kvmnAj; literal 0 HcmV?d00001 diff --git a/q05_feature_engineering/tests/user_sol.pkl b/q05_feature_engineering/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..9f2b9eccc8898617f94e628a44c1bb332af295e3 GIT binary patch literal 87 zcmZo*PAN{TPfJZKDJ@EkPt8lu%u59- U%S+cwD$UGE;ld%yRmh+R0IR4VX#fBK literal 0 HcmV?d00001 From 23b53df3d1820337c0b4aa7e1e50756aac028c13 Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 11:37:19 +0000 Subject: [PATCH 06/11] Done --- q05_feature_engineering_part2/build.py | 14 ++++++++++++-- q05_feature_engineering_part2/tests/test_sol.pkl | Bin 0 -> 111 bytes q05_feature_engineering_part2/tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 q05_feature_engineering_part2/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part2/tests/user_sol.pkl diff --git a/q05_feature_engineering_part2/build.py b/q05_feature_engineering_part2/build.py index 53e6749..a62600f 100644 --- a/q05_feature_engineering_part2/build.py +++ b/q05_feature_engineering_part2/build.py @@ -1,8 +1,18 @@ +# %load q05_feature_engineering_part2/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt -from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data +from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering_part2(path): + df_shape, df = q01_load_data(path) + df['hour'] = df['Datetime'].dt.hour + hours = [] + for i in range(24): + one = df[df['hour'] == i]['Demand'].values + hours.append(one) + plt.boxplot(hours, labels=[str(i) for i in range(24)]) + plt.show() + - diff --git a/q05_feature_engineering_part2/tests/test_sol.pkl b/q05_feature_engineering_part2/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2f666a14df1eb6caf5167db48841dd663be1e31b GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb(n~7M%t_(GtA?wPK@R{~@Fe*F literal 0 HcmV?d00001 From 46a51eac1ae69fb5181f12605967f5380ebb4b99 Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 11:46:49 +0000 Subject: [PATCH 07/11] Done --- q05_feature_engineering_part3/build.py | 13 ++++++++++++- q05_feature_engineering_part3/tests/test_sol.pkl | Bin 0 -> 111 bytes q05_feature_engineering_part3/tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 q05_feature_engineering_part3/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part3/tests/user_sol.pkl diff --git a/q05_feature_engineering_part3/build.py b/q05_feature_engineering_part3/build.py index 7da14f7..26c4ba9 100644 --- a/q05_feature_engineering_part3/build.py +++ b/q05_feature_engineering_part3/build.py @@ -1,8 +1,19 @@ +# %load q05_feature_engineering_part3/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering_part3(path): + df_shape,df = q01_load_data(path) + df['month'] = df['Datetime'].dt.strftime('%b') + demands=[] + months = ['Jan','Feb','Mar','Apr','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] + for month in months: + temp = df[df['month']==month]['Demand'] + demands.append(list(temp)) + plt.boxplot(demands,labels=months) +q05_feature_engineering_part3('data/elecdemand.csv') + - diff --git a/q05_feature_engineering_part3/tests/test_sol.pkl b/q05_feature_engineering_part3/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..017cf665feddbd720b48d7ba68ba334a780efe40 GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb)=MhQ%t_(GtA?wPK@R|09whq! literal 0 HcmV?d00001 From 9541fe8ff8f100a57825f7caf1c94a4b7467ea26 Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 12:24:06 +0000 Subject: [PATCH 08/11] Done --- q05_feature_engineering_part4/build.py | 14 +++++++++++--- q05_feature_engineering_part4/tests/test_sol.pkl | Bin 0 -> 111 bytes q05_feature_engineering_part4/tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 q05_feature_engineering_part4/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part4/tests/user_sol.pkl diff --git a/q05_feature_engineering_part4/build.py b/q05_feature_engineering_part4/build.py index 2731397..140800d 100644 --- a/q05_feature_engineering_part4/build.py +++ b/q05_feature_engineering_part4/build.py @@ -1,9 +1,17 @@ +# %load q05_feature_engineering_part2/build.py import pandas as pd import numpy as np -from sklearn.model_selection import train_test_split import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') -def q05_feature_engineering_part4(): - +def q05_feature_engineering_part4(path): + df_shape,df = q01_load_data(path) + df['hour'] = df['Datetime'].dt.hour + df['month'] = df['Datetime'].dt.strftime('%b') + df['Peakhours'] = ((df['hour']>=6) & (df['hour']<20))*1 + df['Peakmonths'] = df['month'].apply(lambda x : 1 if x in ['Feb', 'May', 'Jun', 'Jul', 'Aug'] else 0) + return df +q05_feature_engineering_part4('data/elecdemand.csv') + + diff --git a/q05_feature_engineering_part4/tests/test_sol.pkl b/q05_feature_engineering_part4/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fca58170afc11aa0ba91572679fb87f1db092e35 GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb(Mu}L%t_(GtA?wPK@R|0OeFmP literal 0 HcmV?d00001 From f59ced79c35d09e915a1481a6c47ff206daa339a Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 12:27:41 +0000 Subject: [PATCH 09/11] Done --- q06_linear_regression/build.py | 26 +++++++++++++++++++++-- q06_linear_regression/tests/test_sol.pkl | Bin 0 -> 95 bytes q06_linear_regression/tests/user_sol.pkl | Bin 0 -> 83 bytes 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 q06_linear_regression/tests/test_sol.pkl create mode 100644 q06_linear_regression/tests/user_sol.pkl diff --git a/q06_linear_regression/build.py b/q06_linear_regression/build.py index 8c11052..7615197 100644 --- a/q06_linear_regression/build.py +++ b/q06_linear_regression/build.py @@ -1,3 +1,4 @@ +# %load q06_linear_regression/build.py import pandas as pd import numpy as np import math @@ -5,8 +6,29 @@ from sklearn.metrics import mean_squared_error from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter +from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data -fe = ["WorkDay", "Peakhours", "Peakmonths"] - +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] +def q06_linear_regression(path,columns = fe, random_state =9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + splits = q02_data_splitter(path) + 'write your solution here' + rmse = [] + for i in splits: + train = i[0] + valid = i[1] + x_train, y_train = data[fe].values[train], data['Demand'].values[train] + x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] + model = LinearRegression() + model.fit(x_train, y_train) + pred = model.predict(x_valid) + measure = math.pow(mean_squared_error(y_valid, pred), 0.5) + rmse.append(measure) + return np.mean(rmse) + + + + diff --git a/q06_linear_regression/tests/test_sol.pkl b/q06_linear_regression/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e0cbf28ec8ae05c55b655d1ba771b898848292f2 GIT binary patch literal 95 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{R&&kY7 eO)QEpN(JgEF3!x)(@QGN%t_(GD#umGpa%ffmLtLd literal 0 HcmV?d00001 diff --git a/q06_linear_regression/tests/user_sol.pkl b/q06_linear_regression/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6794af11eb97c6033013830c6dee62f85f8732a4 GIT binary patch literal 83 zcmZo*PAN{R&&kY7O)QEpN=+|HEiTT? T&(ljP&CE&R!YapA$e;%RD(W8f literal 0 HcmV?d00001 From 41edafa9e37a20225f87edcc5a41f2abb41bbf4f Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 12:30:46 +0000 Subject: [PATCH 10/11] Done --- q07_randomforest_regressor/build.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/q07_randomforest_regressor/build.py b/q07_randomforest_regressor/build.py index 4cdb470..8c713c3 100644 --- a/q07_randomforest_regressor/build.py +++ b/q07_randomforest_regressor/build.py @@ -1,3 +1,4 @@ +# %load q07_randomforest_regressor/build.py import pandas as pd import numpy as np import math @@ -6,7 +7,29 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] +def q07_randomforest_regressor(path,columns = fe, random_state =9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + splits = q02_data_splitter(path) + 'write your solution here' + + rmse = [] + for i in splits: + train = i[0] + valid = i[1] + x_train, y_train = data[fe].values[train], data['Demand'].values[train] + x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] + model = RandomForestRegressor(n_estimators=50, min_samples_leaf=30, random_state=10) + model.fit(x_train, y_train) + pred = model.predict(x_valid) + measure = math.pow(mean_squared_error(y_valid, pred), 0.5) + rmse.append(measure) + return np.mean(rmse) +q07_randomforest_regressor('data/elecdemand.csv',columns = fe, random_state =9) + + + From dabf25ddf42782671455921ee5917444f4016da6 Mon Sep 17 00:00:00 2001 From: rushabh-mehta Date: Thu, 3 Jan 2019 12:38:23 +0000 Subject: [PATCH 11/11] Done --- q08_gradientboosting_regressor/build.py | 26 +++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/q08_gradientboosting_regressor/build.py b/q08_gradientboosting_regressor/build.py index e661aac..31fbe0e 100644 --- a/q08_gradientboosting_regressor/build.py +++ b/q08_gradientboosting_regressor/build.py @@ -1,3 +1,4 @@ +# %load q08_gradientboosting_regressor/build.py import pandas as pd import numpy as np import math @@ -6,5 +7,26 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] - +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] +def q08_gradientboosting_regressor(path,columns = fe, random_state =9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + splits = q02_data_splitter(path) + 'write your solution here' + + rmse = [] + for i in splits: + train = i[0] + valid = i[1] + x_train, y_train = data[fe].values[train], data['Demand'].values[train] + x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] + model = GradientBoostingRegressor(n_estimators=200, min_samples_leaf=10, learning_rate=0.01, random_state=random_state) + model.fit(x_train, y_train) + pred = model.predict(x_valid) + measure = math.pow(mean_squared_error(y_valid, pred), 0.5) + rmse.append(measure) + return np.mean(rmse) + +q08_gradientboosting_regressor('data/elecdemand.csv',columns = fe, random_state =9) + +