From da2da546b8c30df6320265e452565d7e6d0dcece Mon Sep 17 00:00:00 2001 From: HardikaB Date: Fri, 10 Aug 2018 13:56:22 +0000 Subject: [PATCH 01/11] Done --- q01_load_data/build.py | 9 +++++++++ q01_load_data/tests/test_sol.pkl | Bin 0 -> 79 bytes q01_load_data/tests/user_sol.pkl | Bin 0 -> 67 bytes 3 files changed, 9 insertions(+) create mode 100644 q01_load_data/tests/test_sol.pkl create mode 100644 q01_load_data/tests/user_sol.pkl diff --git a/q01_load_data/build.py b/q01_load_data/build.py index a29c139..b68fcc9 100644 --- a/q01_load_data/build.py +++ b/q01_load_data/build.py @@ -1,7 +1,16 @@ +# %load q01_load_data/build.py import pandas as pd import numpy as np from sklearn.model_selection import train_test_split +def q01_load_data(path): + tss=pd.read_csv(path) + tss['Datetime']=pd.to_datetime(tss['Datetime']) + return tss.shape, tss + + + + diff --git a/q01_load_data/tests/test_sol.pkl b/q01_load_data/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7912fb668f4a4bff9f60d47546462bb183207435 GIT binary patch literal 79 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{S&&f|r W0g9I->LrzC=A>|;h;bD%=m7w&xE!N{S&&f|r0g9I->LrzC=A>|; Kh;bD%=m7vu+!yBn literal 0 HcmV?d00001 From 43d6931f22552e2dcc2eb5d33953cf9cdbc642e1 Mon Sep 17 00:00:00 2001 From: HardikaB Date: Mon, 13 Aug 2018 12:29:50 +0000 Subject: [PATCH 02/11] Done --- q02_data_splitter/build.py | 14 +++++++++++++- q02_data_splitter/tests/test_sol.pkl | Bin 0 -> 87 bytes q02_data_splitter/tests/user_sol.pkl | Bin 0 -> 75 bytes 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 q02_data_splitter/tests/test_sol.pkl create mode 100644 q02_data_splitter/tests/user_sol.pkl diff --git a/q02_data_splitter/build.py b/q02_data_splitter/build.py index b6c715f..cbf824e 100644 --- a/q02_data_splitter/build.py +++ b/q02_data_splitter/build.py @@ -1,7 +1,19 @@ +# %load q02_data_splitter/build.py import pandas as pd import numpy as np from sklearn.model_selection import TimeSeriesSplit from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data +def q02_data_splitter(path): + seed=9 + shape,df = q01_load_data(path) + tssf= TimeSeriesSplit(n_splits=3) + trainl=() + validl=() + for train_index,valid_index in tssf.split(df): + trainl=trainl+ tuple(train_index) + validl=validl + tuple(valid_index) + return [[trainl,trainl],[validl,validl]] + + - diff --git a/q02_data_splitter/tests/test_sol.pkl b/q02_data_splitter/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a3e9cc57e21382b4149f86a10e44fd859a5d4213 GIT binary patch literal 87 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D+IDr5=#=} ZiwklxOG;9U^pZ+5b5gi4#JLI?^Z>TKAY=dl literal 0 HcmV?d00001 diff --git a/q02_data_splitter/tests/user_sol.pkl b/q02_data_splitter/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..70c23336a7d13cc99ae68a1dcdbdc3e6646e261a GIT binary patch literal 75 zcmZo*PA Date: Mon, 13 Aug 2018 12:42:47 +0000 Subject: [PATCH 03/11] Done --- q03_time_plot/build.py | 11 +++++++++++ test_sol.pkl | Bin 0 -> 79 bytes user_sol.pkl | Bin 0 -> 67 bytes 3 files changed, 11 insertions(+) create mode 100644 test_sol.pkl create mode 100644 user_sol.pkl diff --git a/q03_time_plot/build.py b/q03_time_plot/build.py index bf18743..cb5cfa7 100644 --- a/q03_time_plot/build.py +++ b/q03_time_plot/build.py @@ -1,7 +1,18 @@ +# %load q03_time_plot/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt + from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q03_time_plot(path): + shp,df=q01_load_data(path) + plt.figure(figsize=(16, 6)) + plt.plot(df['Datetime'], df['Demand']) + plt.xlabel('Time') + plt.ylabel('Demand') + plt.title('SateTime vs Demand in Australia') + plt.show() + diff --git a/test_sol.pkl b/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..cec104b8f2d3d8445aa4256f364f0ca31036739d GIT binary patch literal 79 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{U2TK>^ UN{U2TK>^ Date: Mon, 13 Aug 2018 12:52:52 +0000 Subject: [PATCH 04/11] Done --- q04_boxplot/build.py | 13 ++++++++++++- q04_boxplot/tests/test_sol.pkl | Bin 0 -> 75 bytes q04_boxplot/tests/user_sol.pkl | Bin 0 -> 63 bytes 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 q04_boxplot/tests/test_sol.pkl create mode 100644 q04_boxplot/tests/user_sol.pkl diff --git a/q04_boxplot/build.py b/q04_boxplot/build.py index c69f931..2c46da1 100644 --- a/q04_boxplot/build.py +++ b/q04_boxplot/build.py @@ -1,7 +1,18 @@ +# %load q04_boxplot/build.py import pandas as pd import numpy as np +import seaborn as sns import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') - +def q04_boxplot(path): + + shp,df=q01_load_data(path) + plt.figure(figsize=(16, 7)) + sns.factorplot(x='WorkDay', y='Demand', data=df, kind='box', size=8, aspect=float(16/7)) + plt.xlabel('Workday') + plt.ylabel('Demand') + plt.title('Each workday demand in Australia') + + diff --git a/q04_boxplot/tests/test_sol.pkl b/q04_boxplot/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f863f64a8c994210960bcdfa9c645b392115b34a GIT binary patch literal 75 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{PPs*<- V$jL9!ODfIGN#Q~g;VNX%0{}mk8<+q9 literal 0 HcmV?d00001 diff --git a/q04_boxplot/tests/user_sol.pkl b/q04_boxplot/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..44dda795407bb9cf236e1b00c6328005eb47a0ac GIT binary patch literal 63 zcmZo*PAN{PPs*<-$jL9!ODfIGN#Q~g J;VNX%0|4{i7Rvwt literal 0 HcmV?d00001 From ea044d8f8596c2522a1521ad58110f825286275e Mon Sep 17 00:00:00 2001 From: HardikaB Date: Thu, 16 Aug 2018 11:53:15 +0000 Subject: [PATCH 05/11] Done --- q05_feature_engineering_part2/build.py | 19 ++++++++++++++++-- .../tests/test_sol.pkl | Bin 0 -> 111 bytes .../tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 q05_feature_engineering_part2/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part2/tests/user_sol.pkl diff --git a/q05_feature_engineering_part2/build.py b/q05_feature_engineering_part2/build.py index 53e6749..51f7201 100644 --- a/q05_feature_engineering_part2/build.py +++ b/q05_feature_engineering_part2/build.py @@ -1,8 +1,23 @@ +# %load q05_feature_engineering_part2/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt -from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data +from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') +def q05_feature_engineering_part2(path): + shape, data = q01_load_data(path) + 'write your solution here' + data['hour'] = data['Datetime'].dt.hour + + plt.figure(figsize=(16, 6)) + + hours = [] + for i in range(24): + one = data[data['hour'] == i]['Demand'].values + hours.append(one) + plt.boxplot(hours, labels=[str(i) for i in range(24)]) + plt.xlabel('Hour') + plt.ylabel('Demand') + plt.title('Change in Electricity demand wrt to Hour') - diff --git a/q05_feature_engineering_part2/tests/test_sol.pkl b/q05_feature_engineering_part2/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2f666a14df1eb6caf5167db48841dd663be1e31b GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb(n~7M%t_(GtA?wPK@R{~@Fe*F literal 0 HcmV?d00001 From 1c3f89f8425c7a104c644d8dc5e83e7f4c8515a9 Mon Sep 17 00:00:00 2001 From: HardikaB Date: Wed, 29 Aug 2018 09:31:27 +0000 Subject: [PATCH 06/11] Done --- q05_feature_engineering/build.py | 9 +++++++++ q05_feature_engineering/tests/test_sol.pkl | Bin 0 -> 99 bytes q05_feature_engineering/tests/user_sol.pkl | Bin 0 -> 87 bytes 3 files changed, 9 insertions(+) create mode 100644 q05_feature_engineering/tests/test_sol.pkl create mode 100644 q05_feature_engineering/tests/user_sol.pkl diff --git a/q05_feature_engineering/build.py b/q05_feature_engineering/build.py index 97e29e7..caabbed 100644 --- a/q05_feature_engineering/build.py +++ b/q05_feature_engineering/build.py @@ -1,3 +1,4 @@ +# %load q05_feature_engineering/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt @@ -6,4 +7,12 @@ +def q05_feature_engineering(path): + tss,df=q01_load_data(path) + p_corf=np.corrcoef(df['Temperature'], df['Demand']) + plt.scatter(df['Temperature'], df['Demand']) + plt.show() + +#path='data/elecdemand.csv' +#q05_feature_engineering(path) diff --git a/q05_feature_engineering/tests/test_sol.pkl b/q05_feature_engineering/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..c8990f6faf7b3093f4ad2a9063625b47c6c393dd GIT binary patch literal 99 zcmZ|F$q9fk5Cu@do5c)4u%02yH;iO4+aWhg*9JVl_pH`eOx<$~VyH_5H&j7mrzfkO e@0u^zPDLGjLibU@NL~`YmOPmu!EHafm4#kvmnAj; literal 0 HcmV?d00001 diff --git a/q05_feature_engineering/tests/user_sol.pkl b/q05_feature_engineering/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..9f2b9eccc8898617f94e628a44c1bb332af295e3 GIT binary patch literal 87 zcmZo*PAN{TPfJZKDJ@EkPt8lu%u59- U%S+cwD$UGE;ld%yRmh+R0IR4VX#fBK literal 0 HcmV?d00001 From b63ded416fb58a2f6c33c891542ac2877871d61b Mon Sep 17 00:00:00 2001 From: HardikaB Date: Wed, 29 Aug 2018 11:41:53 +0000 Subject: [PATCH 07/11] Done --- q05_feature_engineering_part3/build.py | 14 ++++++++++++-- q05_feature_engineering_part3/tests/test_sol.pkl | Bin 0 -> 111 bytes q05_feature_engineering_part3/tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 q05_feature_engineering_part3/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part3/tests/user_sol.pkl diff --git a/q05_feature_engineering_part3/build.py b/q05_feature_engineering_part3/build.py index 7da14f7..1714766 100644 --- a/q05_feature_engineering_part3/build.py +++ b/q05_feature_engineering_part3/build.py @@ -1,8 +1,18 @@ +# %load q05_feature_engineering_part3/build.py import pandas as pd import numpy as np import matplotlib.pyplot as plt +import seaborn as sns from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data plt.switch_backend('agg') - - +def q05_feature_engineering_part3(path): + tss,df=q01_load_data(path) + df.info() + df['hour']=df['Datetime'].dt.hour + df['month']=df['Datetime'].dt.strftime('%b') + sns.factorplot(x='month', y='Demand', data=df, kind='box', order=['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug','Sep', 'Oct', 'Nov', 'Dec'], size=8, aspect=float(16/7)) + +#path='data/elecdemand.csv' +#q05_feature_engineering_part3(path) + diff --git a/q05_feature_engineering_part3/tests/test_sol.pkl b/q05_feature_engineering_part3/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..017cf665feddbd720b48d7ba68ba334a780efe40 GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb)=MhQ%t_(GtA?wPK@R|09whq! literal 0 HcmV?d00001 From 1943c7ce63a212a151651ea833ae9380fc8046cd Mon Sep 17 00:00:00 2001 From: HardikaB Date: Wed, 29 Aug 2018 12:21:10 +0000 Subject: [PATCH 08/11] Done --- q05_feature_engineering_part4/build.py | 17 +++++++++++++---- .../tests/test_sol.pkl | Bin 0 -> 111 bytes .../tests/user_sol.pkl | Bin 0 -> 99 bytes 3 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 q05_feature_engineering_part4/tests/test_sol.pkl create mode 100644 q05_feature_engineering_part4/tests/user_sol.pkl diff --git a/q05_feature_engineering_part4/build.py b/q05_feature_engineering_part4/build.py index 2731397..a8fa4f1 100644 --- a/q05_feature_engineering_part4/build.py +++ b/q05_feature_engineering_part4/build.py @@ -1,9 +1,18 @@ +# %load q05_feature_engineering_part2/build.py import pandas as pd import numpy as np -from sklearn.model_selection import train_test_split import matplotlib.pyplot as plt from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data -plt.switch_backend('agg') -def q05_feature_engineering_part4(): - + +def q05_feature_engineering_part4(path): + shape, data = q01_load_data(path) + data['hour'] = data['Datetime'].dt.hour + data['month'] = data['Datetime'].dt.strftime('%b') + data['Peakhours']=data['hour'].apply(lambda x : 1 if x in range(6,20) else 0) + data['Peakmonths']=data['month'].apply(lambda x : 1 if x in ['Feb', 'May', 'Jun', 'Jul', 'Aug'] else 0) + return data + +#path='data/elecdemand.csv' +#q05_feature_engineering_part4(path) + diff --git a/q05_feature_engineering_part4/tests/test_sol.pkl b/q05_feature_engineering_part4/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fca58170afc11aa0ba91572679fb87f1db092e35 GIT binary patch literal 111 zcmaLNO%6an3N{TPfJZKDJ@EkPt8lu%u59- a%S(?hNGvKb(Mu}L%t_(GtA?wPK@R|0OeFmP literal 0 HcmV?d00001 From b6f2765ccaae58ce07d17b37bf5f8e19a792f1c5 Mon Sep 17 00:00:00 2001 From: HardikaB Date: Wed, 29 Aug 2018 13:06:18 +0000 Subject: [PATCH 09/11] Done --- q06_linear_regression/build.py | 26 +++++++++++++++++++++-- q06_linear_regression/tests/test_sol.pkl | Bin 0 -> 95 bytes q06_linear_regression/tests/user_sol.pkl | Bin 0 -> 83 bytes 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 q06_linear_regression/tests/test_sol.pkl create mode 100644 q06_linear_regression/tests/user_sol.pkl diff --git a/q06_linear_regression/build.py b/q06_linear_regression/build.py index 8c11052..4de1e06 100644 --- a/q06_linear_regression/build.py +++ b/q06_linear_regression/build.py @@ -1,12 +1,34 @@ +# %load q06_linear_regression/build.py import pandas as pd import numpy as np import math from sklearn.linear_model import LinearRegression from sklearn.metrics import mean_squared_error + +from greyatomlib.time_series_day_02_project.q01_load_data.build import q01_load_data from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] - +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] +def q06_linear_regression(path,columns = fe, random_state =9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + splits = q02_data_splitter(path) + 'write your solution here' + rmse = [] + for i in splits: + train = i[0] + valid = i[1] + x_train, y_train = data[fe].values[train], data['Demand'].values[train] + x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] + model = LinearRegression() + model.fit(x_train, y_train) + pred = model.predict(x_valid) + measure = math.pow(mean_squared_error(y_valid, pred), 0.5) + rmse.append(measure) + return np.mean(rmse) + + + diff --git a/q06_linear_regression/tests/test_sol.pkl b/q06_linear_regression/tests/test_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e0cbf28ec8ae05c55b655d1ba771b898848292f2 GIT binary patch literal 95 zcmZo*PEIdMtxPP*&&|n9(ksc#O^q*3Ey_$Sj!#Lfj5jcfFDS~-N=+`&D>N{R&&kY7 eO)QEpN(JgEF3!x)(@QGN%t_(GD#umGpa%ffmLtLd literal 0 HcmV?d00001 diff --git a/q06_linear_regression/tests/user_sol.pkl b/q06_linear_regression/tests/user_sol.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6794af11eb97c6033013830c6dee62f85f8732a4 GIT binary patch literal 83 zcmZo*PAN{R&&kY7O)QEpN=+|HEiTT? T&(ljP&CE&R!YapA$e;%RD(W8f literal 0 HcmV?d00001 From 64593672267f76fd8ba7cb6d294d8c11b5dcd45b Mon Sep 17 00:00:00 2001 From: HardikaB Date: Wed, 29 Aug 2018 13:27:28 +0000 Subject: [PATCH 10/11] Done --- q07_randomforest_regressor/build.py | 24 +++++++++++++++++++- q08_gradientboosting_regressor/build.py | 29 ++++++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/q07_randomforest_regressor/build.py b/q07_randomforest_regressor/build.py index 4cdb470..ab42019 100644 --- a/q07_randomforest_regressor/build.py +++ b/q07_randomforest_regressor/build.py @@ -1,3 +1,4 @@ +# %load q07_randomforest_regressor/build.py import pandas as pd import numpy as np import math @@ -6,7 +7,28 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] +def q07_randomforest_regressor(path,columns = fe, random_state =9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + splits = q02_data_splitter(path) + rmse = [] + for i in splits: + train = i[0] + valid = i[1] + x_train, y_train = data[fe].values[train], data['Demand'].values[train] + x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] + model = RandomForestRegressor( n_estimators=50, min_samples_leaf=30, random_state=random_state) + model.fit(x_train, y_train) + pred = model.predict(x_valid) + measure = math.pow(mean_squared_error(y_valid, pred), 0.5) + rmse.append(measure) + return np.mean(rmse) + + +#path= 'data/elecdemand.csv' +#q07_randomforest_regressor(path) + diff --git a/q08_gradientboosting_regressor/build.py b/q08_gradientboosting_regressor/build.py index e661aac..bfdba77 100644 --- a/q08_gradientboosting_regressor/build.py +++ b/q08_gradientboosting_regressor/build.py @@ -1,3 +1,4 @@ +# %load q08_gradientboosting_regressor/build.py import pandas as pd import numpy as np import math @@ -6,5 +7,31 @@ from greyatomlib.time_series_day_02_project.q05_feature_engineering_part4.build import q05_feature_engineering_part4 from greyatomlib.time_series_day_02_project.q02_data_splitter.build import q02_data_splitter -fe = ["WorkDay", "Peakhours", "Peakmonths"] +fe = ['WorkDay', 'Peakhours', 'Peakmonths'] + + +def q08_gradientboosting_regressor(path,columns = fe, random_state =9): + np.random.seed(random_state) + data = q05_feature_engineering_part4(path) + splits = q02_data_splitter(path) + rmse = [] + + for i in splits: + train = i[0] + valid = i[1] + x_train, y_train = data[fe].values[train], data['Demand'].values[train] + x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] + model = GradientBoostingRegressor( n_estimators=200, min_samples_leaf=10, learning_rate=0.01, random_state=random_state) + model.fit(x_train, y_train) + pred = model.predict(x_valid) + measure = math.pow(mean_squared_error(y_valid, pred), 0.5) + rmse.append(measure) + return np.mean(rmse) + + + +#path= 'data/elecdemand.csv' +#q08_gradientboosting_regressor(path) + + From fd17d0c736fe7ceae06a8306e241a2a052df3e99 Mon Sep 17 00:00:00 2001 From: HardikaB Date: Thu, 30 Aug 2018 11:32:40 +0000 Subject: [PATCH 11/11] Done --- q07_randomforest_regressor/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/q07_randomforest_regressor/build.py b/q07_randomforest_regressor/build.py index ab42019..d300887 100644 --- a/q07_randomforest_regressor/build.py +++ b/q07_randomforest_regressor/build.py @@ -20,7 +20,7 @@ def q07_randomforest_regressor(path,columns = fe, random_state =9): valid = i[1] x_train, y_train = data[fe].values[train], data['Demand'].values[train] x_valid, y_valid = data[fe].values[valid], data['Demand'].values[valid] - model = RandomForestRegressor( n_estimators=50, min_samples_leaf=30, random_state=random_state) + model = RandomForestRegressor( n_estimators=50, min_samples_leaf=30, random_state=10) model.fit(x_train, y_train) pred = model.predict(x_valid) measure = math.pow(mean_squared_error(y_valid, pred), 0.5)