From 616ea1585245b13dbb6abb684e261fec05d409a3 Mon Sep 17 00:00:00 2001 From: sannidh Date: Tue, 22 Jan 2019 02:06:26 +0000 Subject: [PATCH 01/11] Done --- q01_load_data/build.py | 11 +++++++++-- q01_load_data/tests/test_sol.pkl | Bin 0 -> 79 bytes q01_load_data/tests/user_sol.pkl | Bin 0 -> 67 bytes 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 q01_load_data/tests/test_sol.pkl create mode 100644 q01_load_data/tests/user_sol.pkl diff --git a/q01_load_data/build.py b/q01_load_data/build.py index a29c139..ad654eb 100644 --- a/q01_load_data/build.py +++ b/q01_load_data/build.py @@ -1,7 +1,14 @@ +# %load q01_load_data/build.py import pandas as pd import numpy as np -from sklearn.model_selection import train_test_split + +path = 'data/elecdemand.csv' + +def q01_load_data(path): + data = pd.read_csv(path) + data['Datetime'] = pd.to_datetime(data['Datetime']) + return data.shape, data + - diff --git a/q01_load_data/tests/test_sol.pkl b/q01_load_data/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7912fb668f4a4bff9f60d47546462bb183207435 GIT binary patch literal 79 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{S&&f|r W0g9I->LrzC=A>|;h;bD%=m7w&xE!N{S&&f|r0g9I->LrzC=A>|; Kh;bD%=m7vu+!yBn literal 0 HcmV?d00001 From 31a01834747d3ac26a397de45f24779625e23ddf Mon Sep 17 00:00:00 2001 From: sannidh Date: Wed, 23 Jan 2019 02:49:31 +0000 Subject: [PATCH 02/11] Done --- q02_data_splitter/build.py | 13 ++++++++++++- q02_data_splitter/tests/test_sol.pkl | Bin 0 -> 87 bytes q02_data_splitter/tests/user_sol.pkl | Bin 0 -> 75 bytes 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 q02_data_splitter/tests/test_sol.pkl create mode 100644 q02_data_splitter/tests/user_sol.pkl diff --git a/q02_data_splitter/build.py b/q02_data_splitter/build.py index b6c715f..9ab0206 100644 --- a/q02_data_splitter/build.py +++ b/q02_data_splitter/build.py @@ -1,7 +1,18 @@ +# %load q02_data_splitter/build.py import pandas as pd import numpy as np from sklearn.model_selection import TimeSeriesSplit from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data +def q02_data_splitter(path): + path = 'data/elecdemand.csv' + shape, df = q01_load_data(path) + tscv = TimeSeriesSplit(n_splits=2) + com_idx = [] + for train_index, valid_index in tscv.split(df): + com_idx.append((train_index, valid_index)) + return com_idx + + + - diff --git a/q02_data_splitter/tests/test_sol.pkl b/q02_data_splitter/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a3e9cc57e21382b4149f86a10e44fd859a5d4213 GIT binary patch literal 87 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D+IDr5=#=} ZiwklxOG;9U^pZ+5b5gi4#JLI?^Z>TKAY=dl literal 0 HcmV?d00001 diff --git a/q02_data_splitter/tests/user_sol.pkl b/q02_data_splitter/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..70c23336a7d13cc99ae68a1dcdbdc3e6646e261a GIT binary patch literal 75 zcmZo*PA Date: Wed, 23 Jan 2019 03:02:07 +0000 Subject: [PATCH 03/11] Done --- q03_time_plot/build.py | 12 ++++++++++++ test_sol.pkl | Bin 0 -> 79 bytes user_sol.pkl | Bin 0 -> 67 bytes 3 files changed, 12 insertions(+) create mode 100644 test_sol.pkl create mode 100644 user_sol.pkl diff --git a/q03_time_plot/build.py b/q03_time_plot/build.py index bf18743..a7d5307 100644 --- a/q03_time_plot/build.py +++ b/q03_time_plot/build.py @@ -1,7 +1,19 @@ +# %load q03_time_plot/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q03_time_plot(path): + path = 'data/elecdemand.csv' + shape, df = q01_load_data(path) + plt.plot(df['Datetime'], df['Demand']) + plt.title('Electricity Demand for Australia for a year') + plt.xlabel('Year-Month') + plt.ylabel('Demand') + plt.show(); + + + diff --git a/test_sol.pkl b/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..cec104b8f2d3d8445aa4256f364f0ca31036739d GIT binary patch literal 79 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{U2TK>^ UN{U2TK>^ Date: Wed, 23 Jan 2019 07:56:30 +0000 Subject: [PATCH 04/11] Done --- q04_boxplot/build.py | 11 ++++++++++- q04_boxplot/tests/test_sol.pkl | Bin 0 -> 75 bytes q04_boxplot/tests/user_sol.pkl | Bin 0 -> 63 bytes 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 q04_boxplot/tests/test_sol.pkl create mode 100644 q04_boxplot/tests/user_sol.pkl diff --git a/q04_boxplot/build.py b/q04_boxplot/build.py index c69f931..44c9a09 100644 --- a/q04_boxplot/build.py +++ b/q04_boxplot/build.py @@ -1,7 +1,16 @@ +# %load q04_boxplot/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') - +def q04_boxplot(path): + path = 'data/elecdemand.csv' + shape, df = q01_load_data(path) + df.boxplot(column=['Demand'], by=['WorkDay']) + plt.show(); + + + + diff --git a/q04_boxplot/tests/test_sol.pkl b/q04_boxplot/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f863f64a8c994210960bcdfa9c645b392115b34a GIT binary patch literal 75 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{PPs*<- V$jL9!ODfIGN#Q~g;VNX%0{}mk8<+q9 literal 0 HcmV?d00001 diff --git a/q04_boxplot/tests/user_sol.pkl b/q04_boxplot/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..44dda795407bb9cf236e1b00c6328005eb47a0ac GIT binary patch literal 63 zcmZo*PAN{PPs*<-$jL9!ODfIGN#Q~g J;VNX%0|4{i7Rvwt literal 0 HcmV?d00001 From 7773270d2d460c3b2a028c48ecd199c2c2b5acf3 Mon Sep 17 00:00:00 2001 From: sannidh Date: Wed, 23 Jan 2019 10:00:28 +0000 Subject: [PATCH 05/11] Done --- q05_feature_engineering/build.py | 12 ++++++++++++ q05_feature_engineering/tests/test_sol.pkl | Bin 0 -> 99 bytes q05_feature_engineering/tests/user_sol.pkl | Bin 0 -> 87 bytes 3 files changed, 12 insertions(+) create mode 100644 q05_feature_engineering/tests/test_sol.pkl create mode 100644 q05_feature_engineering/tests/user_sol.pkl diff --git a/q05_feature_engineering/build.py b/q05_feature_engineering/build.py index 97e29e7..d7f3450 100644 --- a/q05_feature_engineering/build.py +++ b/q05_feature_engineering/build.py @@ -1,9 +1,21 @@ +# %load q05_feature_engineering/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt +from scipy.stats import pearsonr from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering(path): + path = 'data/elecdemand.csv' + shape, df = q01_load_data(path) + corr, p_value = pearsonr(df['Temperature'], df['Demand']) + plt.scatter(df['Temperature'], df['Demand']) + plt.xlabel('Temperature') + plt.ylabel('Demand') + plt.title('Temperature vs Demand') + plt.show(); + diff --git a/q05_feature_engineering/tests/test_sol.pkl b/q05_feature_engineering/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c8990f6faf7b3093f4ad2a9063625b47c6c393dd GIT binary patch literal 99 zcmZ|F$q9fk5Cu@do5c)4u%02yH;iO4+aWhg*9JVl_pH`eOx<$~VyH_5H&j7mrzfkO e@0u^zPDLGjLibU@NL~`YmOPmu!EHafm4#kvmnAj; literal 0 HcmV?d00001 diff --git a/q05_feature_engineering/tests/user_sol.pkl b/q05_feature_engineering/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..9f2b9eccc8898617f94e628a44c1bb332af295e3 GIT binary patch literal 87 zcmZo*PAN{TPfJZKDJ@EkPt8lu%u59- U%S+cwD$UGE;ld%yRmh+R0IR4VX#fBK literal 0 HcmV?d00001 From 90fd8f7e5765599b81a17add0cd32f7967f030d8 Mon Sep 17 00:00:00 2001 From: sannidh Date: Wed, 23 Jan 2019 10:28:41 +0000 Subject: [PATCH 06/11] Done --- q05_feature_engineering_part2/build.py | 31 ++++++++++++++++-- .../tests/test_sol.pkl | Bin 0 -> 111 bytes .../tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 q05_feature_engineering_part2/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part2/tests/user_sol.pkl diff --git a/q05_feature_engineering_part2/build.py b/q05_feature_engineering_part2/build.py index 53e6749..d6dbc7e 100644 --- a/q05_feature_engineering_part2/build.py +++ b/q05_feature_engineering_part2/build.py @@ -1,8 +1,35 @@ +# %load q05_feature_engineering_part2/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt -from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data +from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering_part2(path): + path = 'data/elecdemand.csv' + shape, data = q01_load_data(path) + data['hour'] = data['Datetime'].dt.hour + data['month'] = data['Datetime'].dt.month + plt.figure(figsize=(16, 6)) + demand_hours = [] + for i in range(1,25): + one = data[data['hour'] == i]['Demand'].values + demand_hours.append(one) + demand_months = [] + for j in range(1,13): + demand_months.append(data[data['month'] == j]['Demand'].values) + plt.subplot(211) + plt.boxplot(demand_hours, labels=[str(i) for i in range(1,25)]) + plt.xlabel('Hour') + plt.ylabel('Demand') + plt.title('Change in Electricity demand wrt to Hour') + plt.subplot(212) + plt.boxplot(demand_months, labels=[str(i) for i in range(1,13)]) + plt.xlabel('Months') + plt.ylabel('Demand') + plt.title('Change in Electricity demand wrt to months') + plt.show(); + + + - diff --git a/q05_feature_engineering_part2/tests/test_sol.pkl b/q05_feature_engineering_part2/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2f666a14df1eb6caf5167db48841dd663be1e31b GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb(n~7M%t_(GtA?wPK@R{~@Fe*F literal 0 HcmV?d00001 From 69fad90347e0e83e1ff8948b5130e1bcfdc334b1 Mon Sep 17 00:00:00 2001 From: sannidh Date: Wed, 23 Jan 2019 10:43:04 +0000 Subject: [PATCH 07/11] Done --- q05_feature_engineering_part3/build.py | 27 +++++++++++++++++- .../tests/test_sol.pkl | Bin 0 -> 111 bytes .../tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 q05_feature_engineering_part3/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part3/tests/user_sol.pkl diff --git a/q05_feature_engineering_part3/build.py b/q05_feature_engineering_part3/build.py index 7da14f7..1d2cd9f 100644 --- a/q05_feature_engineering_part3/build.py +++ b/q05_feature_engineering_part3/build.py @@ -1,8 +1,33 @@ +# %load q05_feature_engineering_part3/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering_part3(path): + path = 'data/elecdemand.csv' + shape, data = q01_load_data(path) + data['hour'] = data['Datetime'].dt.hour + data['month'] = data['Datetime'].dt.month + plt.figure(figsize=(16, 6)) + demand_hours = [] + for i in range(1,25): + one = data[data['hour'] == i]['Demand'].values + demand_hours.append(one) + demand_months = [] + for j in range(1,13): + demand_months.append(data[data['month'] == j]['Demand'].values) + plt.subplot(211) + plt.boxplot(demand_hours, labels=[str(i) for i in range(1,25)]) + plt.xlabel('Hour') + plt.ylabel('Demand') + plt.title('Change in Electricity demand wrt to Hour') + plt.subplot(212) + plt.boxplot(demand_months, labels=[str(i) for i in range(1,13)]) + plt.xlabel('Months') + plt.ylabel('Demand') + plt.title('Change in Electricity demand wrt to months') + plt.show(); + - diff --git a/q05_feature_engineering_part3/tests/test_sol.pkl b/q05_feature_engineering_part3/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..017cf665feddbd720b48d7ba68ba334a780efe40 GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb)=MhQ%t_(GtA?wPK@R|09whq! literal 0 HcmV?d00001 From e95a2f1661693b02c2b1deff2719aa83eb824862 Mon Sep 17 00:00:00 2001 From: sannidh Date: Thu, 24 Jan 2019 17:41:38 +0000 Subject: [PATCH 08/11] Done --- q05_feature_engineering_part4/build.py | 17 +++++++++++++---- .../tests/test_sol.pkl | Bin 0 -> 111 bytes .../tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 q05_feature_engineering_part4/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part4/tests/user_sol.pkl diff --git a/q05_feature_engineering_part4/build.py b/q05_feature_engineering_part4/build.py index 2731397..cf7ec16 100644 --- a/q05_feature_engineering_part4/build.py +++ b/q05_feature_engineering_part4/build.py @@ -1,9 +1,18 @@ +# %load q05_feature_engineering_part2/build.py import pandas as pd import numpy as np -from sklearn.model_selection import train_test_split import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data -plt.switch_backend('agg') -def q05_feature_engineering_part4(): - +path = 'data/elecdemand.csv' + +def q05_feature_engineering_part4(path): + shape, data = q01_load_data(path) + data['hour'] = data['Datetime'].dt.hour + data['month'] = data['Datetime'].dt.strftime('%b') + data['Peakhours'] = list(map(lambda x: 1 if x in range(6,20) else 0, data['Datetime'].dt.hour)) + data['Peakmonths'] = list(map(lambda x: 1 if x in ['Feb', 'May', 'Jun', 'Jul', 'Aug'] else 0, data['Datetime'].dt.strftime('%b'))) + return data + + + diff --git a/q05_feature_engineering_part4/tests/test_sol.pkl b/q05_feature_engineering_part4/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fca58170afc11aa0ba91572679fb87f1db092e35 GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb(Mu}L%t_(GtA?wPK@R|0OeFmP literal 0 HcmV?d00001 From 88e059906b8e579b2ff339a6fa3e560d29d3033c Mon Sep 17 00:00:00 2001 From: sannidh Date: Fri, 25 Jan 2019 01:16:28 +0000 Subject: [PATCH 09/11] Done --- q06_linear_regression/build.py | 21 +++++++++++++++++++-- q06_linear_regression/tests/test_sol.pkl | Bin 0 -> 95 bytes q06_linear_regression/tests/user_sol.pkl | Bin 0 -> 83 bytes 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 q06_linear_regression/tests/test_sol.pkl create mode 100644 q06_linear_regression/tests/user_sol.pkl diff --git a/q06_linear_regression/build.py b/q06_linear_regression/build.py index 8c11052..e779f4c 100644 --- a/q06_linear_regression/build.py +++ b/q06_linear_regression/build.py @@ -1,3 +1,4 @@ +# %load q06_linear_regression/build.py import pandas as pd import numpy as np import math @@ -6,7 +7,23 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] + +def q06_linear_regression(path, columns = fe, random_state = 9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + com_idx = q02_data_splitter(path) + rmse = [] + for i in com_idx: + train_idx = i[0] + valid_idx = i[1] + X_train, y_train = data.ix[train_idx, fe], data.ix[train_idx, 'Demand'] + X_valid, y_valid = data.ix[valid_idx, fe], data.ix[valid_idx, 'Demand'] + model = LinearRegression() + model.fit(X_train, y_train) + y_pred = model.predict(X_valid) + rms = mean_squared_error(y_valid, y_pred)**0.5 + rmse.append(rms) + return np.mean(rmse) - diff --git a/q06_linear_regression/tests/test_sol.pkl b/q06_linear_regression/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e0cbf28ec8ae05c55b655d1ba771b898848292f2 GIT binary patch literal 95 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{R&&kY7 eO)QEpN(JgEF3!x)(@QGN%t_(GD#umGpa%ffmLtLd literal 0 HcmV?d00001 diff --git a/q06_linear_regression/tests/user_sol.pkl b/q06_linear_regression/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6794af11eb97c6033013830c6dee62f85f8732a4 GIT binary patch literal 83 zcmZo*PAN{R&&kY7O)QEpN=+|HEiTT? T&(ljP&CE&R!YapA$e;%RD(W8f literal 0 HcmV?d00001 From 9f43567dc7641718774c492981ff30f6a80a2a11 Mon Sep 17 00:00:00 2001 From: sannidh Date: Fri, 25 Jan 2019 01:34:10 +0000 Subject: [PATCH 10/11] Done --- q07_randomforest_regressor/build.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/q07_randomforest_regressor/build.py b/q07_randomforest_regressor/build.py index 4cdb470..12949db 100644 --- a/q07_randomforest_regressor/build.py +++ b/q07_randomforest_regressor/build.py @@ -1,3 +1,4 @@ +# %load q07_randomforest_regressor/build.py import pandas as pd import numpy as np import math @@ -6,7 +7,23 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] + +def q07_randomforest_regressor(path, columns = fe, random_state = 9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + com_idx = q02_data_splitter(path) + rmse = [] + for i in com_idx: + train_idx = i[0] + valid_idx = i[1] + X_train, y_train = data.ix[train_idx, fe], data.ix[train_idx, 'Demand'] + X_valid, y_valid = data.ix[valid_idx, fe], data.ix[valid_idx, 'Demand'] + model = RandomForestRegressor(n_estimators=50, min_samples_leaf=30, random_state=10) + model.fit(X_train, y_train) + y_pred = model.predict(X_valid) + rms = mean_squared_error(y_valid, y_pred)**0.5 + rmse.append(rms) + return np.mean(rmse) - From 008f84089f1696485ea8a9d1ab8cdf36dde7392a Mon Sep 17 00:00:00 2001 From: sannidh Date: Fri, 25 Jan 2019 01:37:14 +0000 Subject: [PATCH 11/11] Done --- q08_gradientboosting_regressor/build.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/q08_gradientboosting_regressor/build.py b/q08_gradientboosting_regressor/build.py index e661aac..47a5438 100644 --- a/q08_gradientboosting_regressor/build.py +++ b/q08_gradientboosting_regressor/build.py @@ -1,3 +1,4 @@ +# %load q08_gradientboosting_regressor/build.py import pandas as pd import numpy as np import math @@ -6,5 +7,23 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] +def q08_gradientboosting_regressor(path, columns = fe, random_state = 9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + com_idx = q02_data_splitter(path) + rmse = [] + for i in com_idx: + train_idx = i[0] + valid_idx = i[1] + X_train, y_train = data.ix[train_idx, fe], data.ix[train_idx, 'Demand'] + X_valid, y_valid = data.ix[valid_idx, fe], data.ix[valid_idx, 'Demand'] + model = GradientBoostingRegressor(n_estimators=200, min_samples_leaf=10, learning_rate=0.01, random_state=random_state) + model.fit(X_train, y_train) + y_pred = model.predict(X_valid) + rms = mean_squared_error(y_valid, y_pred)**0.5 + rmse.append(rms) + return np.mean(rmse) + +