Regression Models

import pandas as pd

data = pd.read_csv("data/winequality-white.csv", sep=';')
data.head()

from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(data.iloc[:,:-1], 
                                                    data.iloc[:,-1], 
                                                    test_size=0.3, 
                                                    random_state=123)

from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error, mean_absolute_percentage_error

def evaluate_model(y_pred, y_test, verbose=True):
    # Performance
    perf = {}
    perf['r2'] = r2_score(y_test, y_pred)
    perf['MSE'] = mean_squared_error(y_test, y_pred, squared=True)
    perf['RMSE'] = mean_squared_error(y_test, y_pred, squared=False)
    perf['MAE'] = mean_absolute_error(y_test, y_pred)
    perf['MAPE'] = mean_absolute_percentage_error(y_test, y_pred)
    
    if verbose:
        for p in perf:
            print("%s: %.3f" % (p, perf[p]))
    return perf

from sklearn.linear_model import LinearRegression

model = LinearRegression(fit_intercept=True)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.linear_model import Ridge

model = Ridge(alpha=.1, # regularization parameter : ||y - Xw||^2_2 + alpha * ||w||^2_2
              fit_intercept=True)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.linear_model import Lasso

model = Lasso(alpha=.1, # regularization parameter : ||y - Xw||^2_2 + alpha * ||w||_1
              fit_intercept=True)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.linear_model import ElasticNet

# Objective function:
# 1 / (2 * n_samples) * ||y - Xw||^2_2 + alpha * l1_ratio * ||w||_1 + 0.5 * alpha * (1 - l1_ratio) * ||w||^2_2

model = ElasticNet(l1_ratio=0.5, # regularization parameter
                   alpha=.1, # regularization parameter
                   fit_intercept=True)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.linear_model import QuantileRegressor

model = QuantileRegressor(quantile=.5, 
                          alpha=1.0, # regularization parameter for L1 penalty
                          fit_intercept=True) 

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.neighbors import KNeighborsRegressor
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
x_train_scaled = scaler.fit_transform(x_train)
x_test_scaled = scaler.transform(x_test)

model = KNeighborsRegressor(n_neighbors=5, # number of neighbors
                            weights='uniform', # or ‘distance’
                            p=2, # Minkowski metric: p=1 is manhattan distance and p=2 is euclidean distance
                            metric='minkowski')  

model.fit(x_train_scaled, y_train)
y_pred = model.predict(x_test_scaled)

from sklearn.svm import SVR
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
x_train_scaled = scaler.fit_transform(x_train)
x_test_scaled = scaler.transform(x_test)

model = SVR(kernel='rbf', 
            C=1.0)  

model.fit(x_train_scaled, y_train)
y_pred = model.predict(x_test_scaled)

from sklearn.tree import DecisionTreeRegressor

model = DecisionTreeRegressor(max_depth=None, # if None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples
                              min_samples_split=2, # minimum number of samples required to split an internal node
                              min_samples_leaf=1, # minimum number of samples required to be at a leaf node
                              max_features=None, # if None, then max_features=n_features; can be 'sqrt' for sqrt(n_features)  
                              random_state=123)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.ensemble import RandomForestRegressor

model = RandomForestRegressor(n_estimators=100,
                              max_depth=None, # if None, then nodes are expanded until all leaves are pure or until all leaves contain less than min_samples_split samples
                              min_samples_split=2, # minimum number of samples required to split an internal node
                              min_samples_leaf=1, # minimum number of samples required to be at a leaf node
                              max_features='sqrt', # if None, then max_features=n_features; can be 'sqrt' for sqrt(n_features)  
                              random_state=123)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

from sklearn.ensemble import AdaBoostRegressor

model = AdaBoostRegressor(n_estimators=100,
                          learning_rate=1.0,
                          loss='linear',
                          random_state=123)  

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

import xgboost as xgb

dtrain = xgb.DMatrix(data=x_train, label=y_train)
dtest = xgb.DMatrix(data=x_test, label=y_test)

params = {'objective':'reg:squarederror',
          'eval_metric': 'rmse',
          'colsample_bytree': 0.6,
          'min_child_weight':1.0,
          'max_depth':6,
          'eta':0.05, # learning_rate
          'lambda':1.0, # L2 regularization parameter
          'alpha':0.0, # L1 regularization parameter
          'random_state':123}

model = xgb.train(params, 
                  dtrain, 
                  500,
                  evals=[(dtrain, 'train'), (dtest, 'test')], 
                  maximize=False, 
                  early_stopping_rounds=5,
                  verbose_eval=10)
    
y_pred = model.predict(dtest)

import lightgbm as lgb

params = {
    "objective" : "regression",
    "metric" : "rmse",
    "num_leaves" : 100,
    "learning_rate" : 0.001,
    "bagging_fraction" : 0.6,
    "feature_fraction" : 0.6,
    "bagging_frequency" : 6,
    "bagging_seed" : 42,
    "verbosity" : -1,
    "seed": 42
}

lg_train = lgb.Dataset(data=x_train, label=y_train)
lg_test = lgb.Dataset(data=x_test, label=y_test)

model = lgb.train(params, 
                  lg_train, 
                  5000,
                  valid_sets=[lg_train, lg_test],
                  early_stopping_rounds=100,
                  verbose_eval=150,
                  evals_result=evals_result_lgbm)

y_pred = model.predict(x_test, num_iteration=model.best_iteration)

from sklearn.neural_network import MLPRegressor

model = MLPRegressor(hidden_layer_sizes=(100,),
                     activation='relu',
                     solver='adam',
                     alpha=0.0001, # L2 penalty,
                     learning_rate='adaptive',
                     random_state=123)

model.fit(x_train, y_train)
y_pred = model.predict(x_test)

import lazypredict
from lazypredict.Supervised import LazyRegressor

reg = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None)
models, predictions = reg.fit(x_train, x_test, y_train, y_test)

print(models)