Featurization Model Selection and Tuning

 1# NumPy: For mathematical funcations, array, matrices operations
 2import numpy as np 
 3
 4# Graph: Plotting graphs and other visula tools
 5import pandas as pd
 6import seaborn as sns
 7
 8sns.set(color_codes=True)
 9
10#To enable inline plotting graphs
11import matplotlib.pyplot as plt
12%matplotlib inline
13
14# Disable warning
15import warnings
16warnings.filterwarnings("ignore")

 1# Load data set
 2# Import CSV data using pandas data frame
 3df_original = pd.read_csv('concrete.csv')
 4
 5# Print total columns
 6print("Total Colums in dataframe: ", len(df_original.columns))
 7
 8# Prepare columns names
 9df_original_columns = []
10for column in df_original.columns:
11    df_original_columns.append(column)
12
13
14    
15print("Columns list {}".format(df_original_columns))
16print("***********************************************************************************************************************")
17
18# Prepare mapping of column names for quick access
19df_original_columns_map = {}
20map_index: int = 0
21for column in df_original_columns:
22    df_original_columns_map[map_index] = column
23    map_index = map_index + 1
24    
25print("Columns Map {}".format(df_original_columns_map))

1df_original.shape

1df_original.info()
2# All row items are numeric in nature, hence no label encoding is required

1df_original.head(16)

1df_original.describe().T

1df_original.isna().sum()
2# No null values are present in data

 1df_duplicates = df_original.duplicated()
 2
 3print('Number of duplicate rows = {}'.format(df_duplicates.sum()))
 4
 5# 25 duplicates
 6
 7# We are not modyfying original dataframe instead our all operations
 8# will be on `df_main`
 9df_main = df_original.drop_duplicates() 
10
11df_main.shape

1df_main.corr(method ='pearson')
2# Eg. Present of cement, superplastic and more value of age result more strength of the mixture

1df_main.hist(bins=20, xlabelsize=10, ylabelsize=10)
2# We can see that data is not distributed properly and there are tails present on either side of gaussian curve

1sns.pairplot(df_main)
2# We do not see any positive curve or negative curve but there is a cloud 
3# which indicates the presence of cement add strength though not direct relation

  1from sklearn.model_selection import cross_val_score
  2from sklearn.model_selection import train_test_split
  3from sklearn.pipeline import Pipeline
  4from sklearn.model_selection import KFold, GridSearchCV, RandomizedSearchCV
  5from sklearn.preprocessing import MinMaxScaler
  6from sklearn.linear_model import LinearRegression, Ridge,Lasso
  7from sklearn.neighbors import KNeighborsRegressor
  8from sklearn.ensemble import AdaBoostRegressor, ExtraTreesRegressor, RandomForestRegressor, GradientBoostingRegressor
  9from sklearn.tree import DecisionTreeRegressor
 10
 11
 12class ModelEvaluator:
 13    
 14    def __init__(self, dataFrame):
 15        self.dataFrame = dataFrame
 16        self.modelResullt = {}
 17        self.algorithms = {
 18            "linearRegression" :  LinearRegression(),
 19            "lasso" :  Lasso(),
 20            "ridge" :  Ridge(),
 21            "adaBoostRegressor" : AdaBoostRegressor(),
 22            "extraTreeRegressor" :  ExtraTreesRegressor(),
 23            "randomForestRegressor" : RandomForestRegressor(),
 24            "gradientBoostRegressor" : GradientBoostingRegressor(),
 25            "decisionTreeRegressor" : DecisionTreeRegressor(),
 26            "knnRegressor" : KNeighborsRegressor()
 27        }
 28    
 29    def printDf_X(self):
 30        print(self.df_X)
 31        
 32    def printDf_Y(self):
 33        print(self.df_Y)
 34        
 35    def splitData(self):
 36        print("Splitting data into train ,test and validation")
 37        test_size = 0.2
 38        seed = 29
 39        self.df_X = self.dataFrame.copy().drop(['strength'], axis = 1)
 40        self.df_Y = self.dataFrame['strength']
 41        
 42        self.X_train, self.X_test, self.y_train, self.y_test = \
 43            train_test_split(self.df_X, self.df_Y, test_size=test_size, random_state=seed)
 44        
 45        self.X_train, self.X_val, self.y_train, self.y_val = \
 46            train_test_split(self.df_X, self.df_Y, test_size=test_size, random_state=seed)
 47        
 48    
 49    def runSimplePipelines(self):       
 50        print("Running simple pipelines")
 51        
 52        self.decentModels  = {}
 53        
 54        for item in self.algorithms:
 55            # Init pipe
 56            itemPipe = Pipeline([("scaler", MinMaxScaler()), (item, self.algorithms[item])])
 57            
 58            #fit data 
 59            itemPipe.fit(self.X_train, self.y_train)
 60            
 61            print ("**********   {}   ************".format(item))
 62            
 63            print("")
 64            score  = itemPipe.score(self.X_test, self.y_test)
 65            print("Test score is {:.2f}". format(score))
 66            print("")
 67           
 68            cross_val = cross_val_score(self.algorithms[item], self.X_train, self.y_train)
 69            cross_val = cross_val.ravel()
 70            print ("cross validatioion score")
 71            print ("cv-mean :",cross_val.mean())
 72            print ("cv-std  :",cross_val.std())
 73            print ("cv-max  :",cross_val.max())
 74            print ("cv-min  :",cross_val.min())
 75            print("")
 76            
 77            # We aree considering 0.85 s the minim score we need for our prediction
 78            if score >= 0.85:
 79                self.decentModels[item] = score
 80           
 81    def printDecentModels(self):
 82        print("Top performing models are: ")
 83        for topModel in self.decentModels:
 84            print("{} score {}".format(topModel, self.decentModels[topModel]))
 85
 86    def deecentModels(self):
 87        return selff.decentModels
 88    
 89    def runGridSearchCV_HyperParameterTuningPipelines(self, param1, param2):
 90        print("Running gridsearch cv hyper parameter pipelines")
 91        for topModel in self.decentModels:
 92            
 93            # Init pipe
 94            regressor = self.algorithms[topModel]
 95
 96            itemPipe = Pipeline([("scaler", MinMaxScaler()), (topModel, regressor)])
 97            
 98            if topModel == 'extraTreeRegressor' or topModel == 'randomForestRegressor':
 99
100                gsCv = GridSearchCV(estimator = regressor, param_grid = param1, 
101                              cv = 3, n_jobs = 1, verbose = 0, return_train_score=True)
102                #fit data 
103                gsCv.fit(self.X_train, self.y_train)
104
105                print ("**********   {}   ************".format(topModel))
106
107                print("")
108
109                print("Best parameter {} ".format(gsCv.best_params_))
110
111                best_grid = gsCv.best_estimator_
112
113                score = best_grid.score(self.X_val, self.y_val)
114
115                print("Best score is {} ".format(score))
116            elif topModel == 'decisionTreeRegressor':
117                gsCv = GridSearchCV(estimator = regressor, param_grid = param2, 
118                              cv = 3, n_jobs = 1, verbose = 0, return_train_score=True)
119                #fit data 
120                gsCv.fit(self.X_train, self.y_train)
121
122                print ("**********   {}   ************".format(topModel))
123
124                print("")
125
126                print("Best parameter {} ".format(gsCv.best_params_))
127
128                best_grid = gsCv.best_estimator_
129
130                score = best_grid.score(self.X_val, self.y_val)
131
132                print("Best score is {} ".format(score))
133                
134        
135    def runRandomSearchCV_HyperParameterTuningPipelines(self, param1, param2):
136        print("Running randomsearch cv hyper parameter pipelines")
137        for topModel in self.decentModels:
138            print("")
139            print ("**********   {}   ************".format(topModel))
140
141            # Init pipe
142            regressor = self.algorithms[topModel]
143
144            itemPipe = Pipeline([("scaler", MinMaxScaler()), (topModel, regressor)])
145            
146            if topModel == 'extraTreeRegressor' or topModel == 'randomForestRegressor':
147
148                randCv = RandomizedSearchCV(estimator = regressor, param_distributions = param1, 
149                              cv = 3, n_jobs = 1, verbose = 0, return_train_score=True)
150                #fit data 
151                randCv.fit(self.X_train, self.y_train)
152
153               
154                print("Best parameter {} ".format(randCv.best_params_))
155
156                best_random = randCv.best_estimator_
157
158                score = best_random.score(self.X_val, self.y_val)
159
160                print("Best score is {} ".format(score))
161                
162            elif topModel == 'gradientBoostRegressor':
163                randCv = RandomizedSearchCV(estimator = regressor, param_distributions = param2, 
164                              cv = 3, n_jobs = 1, verbose = 0, return_train_score=True)
165                #fit data 
166                randCv.fit(self.X_train, self.y_train)
167                
168                print("Best parameter {} ".format(randCv.best_params_))
169
170                best_random = randCv.best_estimator_
171
172                score = best_random.score(self.X_val, self.y_val)
173                print("Best score is {} ".format(score))
174                
175                

 1# Instantiation of Class
 2modelEvaluator = ModelEvaluator(df_main)
 3
 4# Splitting data in train, test, validation
 5modelEvaluator.splitData()
 6
 7# Print Data X
 8# modelEvaluator.printDf_X()
 9
10# Print Data Y
11
12#modelEvaluator.printDf_Y()

1# run simple pipelines
2
3modelEvaluator.runSimplePipelines()

1#print deceent models
2modelEvaluator.printDecentModels()

 1for topModel in modelEvaluator.decentModels:
 2    model = modelEvaluator.algorithms[topModel]
 3    print("")
 4    print("***************************")
 5    print(" Model {} ".format(topModel))
 6    print(model.get_params())
 7    print("***************************")
 8    print("")
 9    
10# We see all regressor has ~similar attributes hence we shall
11# use same cutomizing parameters for them

 1param_regressor = {
 2    'bootstrap': [True, False],
 3    'max_depth': [int(x) for x in np.linspace(5, 96, num = 2)],
 4    'max_features': ['auto', 'log2'],
 5    'min_samples_leaf': [1, 2, 4, 6, 8, 10, 12, 16, 32, 48],
 6    'min_samples_split': [2, 4, 6, 8, 10, 12, 15],
 7    'n_estimators': [int(x) for x in np.linspace(start = 2 , stop = 512, num = 2)]
 8}
 9
10param_gb_regressor = {
11    'max_depth': [int(x) for x in np.linspace(5, 96, num = 2)],
12    'max_features': ['auto', 'log2'],
13    'min_samples_leaf': [1, 2, 4, 6, 8, 10, 12, 16, 32, 48],
14    'min_samples_split': [2, 4, 6, 8, 10, 12, 15],
15    'n_estimators': [int(x) for x in np.linspace(start = 2 , stop = 512, num = 2)]
16}

1modelEvaluator.runRandomSearchCV_HyperParameterTuningPipelines(param_regressor, param_gb_regressor)

1modelEvaluator.runGridSearchCV_HyperParameterTuningPipelines(param_regressor, param_gb_regressor)