python实现线性回归之简单回归

首先定义一个基本的回归类，作为各种回归方法的基类：

; "复制代码")

class Regression(object): """ Base regression model. Models the relationship between a scalar dependent variable y and the independent

variables X. 
Parameters:
-----------
n\_iterations: float
    The number of training iterations the algorithm will tune the weights for.
learning\_rate: float
    The step length that will be used when updating the weights. """
def \_\_init\_\_(self, n\_iterations, learning\_rate):
    self.n\_iterations \= n\_iterations
    self.learning\_rate \= learning\_rate def initialize\_wights(self, n\_features): """ Initialize weights randomly \[-1/N, 1/N\] """ limit \= 1 / math.sqrt(n\_features)
    self.w \= np.random.uniform(-limit, limit, (n\_features, )) def fit(self, X, y): # Insert constant ones for bias weights
    X = np.insert(X, 0, 1, axis=1)
    self.training\_errors \= \[\]
    self.initialize\_weights(n\_features\=X.shape\[1\]) # Do gradient descent for n\_iterations
    for i in range(self.n\_iterations):
        y\_pred \= X.dot(self.w) # Calculate l2 loss
        mse = np.mean(0.5 \* (y - y\_pred)\*\*2 + self.regularization(self.w))
        self.training\_errors.append(mse) # Gradient of l2 loss w.r.t w
        grad\_w = -(y - y\_pred).dot(X) + self.regularization.grad(self.w) # Update the weights
        self.w -= self.learning\_rate \* grad\_w def predict(self, X): # Insert constant ones for bias weights
    X = np.insert(X, 0, 1, axis=1)
    y\_pred \= X.dot(self.w) return y\_pred

; "复制代码")

说明：初始化时传入两个参数，一个是迭代次数，另一个是学习率。initialize_weights()用于初始化权重。fit()用于训练。需要注意的是，对于原始的输入X，需要将其最前面添加一项为偏置项。predict()用于输出预测值。

接下来是简单线性回归，继承上面的基类：

; "复制代码")

class LinearRegression(Regression): """Linear model.

Parameters:
-----------
n\_iterations: float
    The number of training iterations the algorithm will tune the weights for.
learning\_rate: float
    The step length that will be used when updating the weights.
gradient\_descent: boolean
    True or false depending if gradient descent should be used when training. If 
    false then we use batch optimization by least squares. """
def \_\_init\_\_(self, n\_iterations=100, learning\_rate=0.001, gradient\_descent=True):
    self.gradient\_descent \= gradient\_descent # No regularization
    self.regularization = lambda x: 0
    self.regularization.grad \= lambda x: 0
    super(LinearRegression, self).\_\_init\_\_(n\_iterations=n\_iterations,
                                        learning\_rate\=learning\_rate) def fit(self, X, y): # If not gradient descent => Least squares approximation of w
    if not self.gradient\_descent: # Insert constant ones for bias weights
        X = np.insert(X, 0, 1, axis=1) # Calculate weights by least squares (using Moore-Penrose pseudoinverse)
        U, S, V = np.linalg.svd(X.T.dot(X))
        S \= np.diag(S)
        X\_sq\_reg\_inv \= V.dot(np.linalg.pinv(S)).dot(U.T)
        self.w \= X\_sq\_reg\_inv.dot(X.T).dot(y) else:
        super(LinearRegression, self).fit(X, y)

; "复制代码")

这里使用两种方式进行计算。如果规定gradient_descent=True，那么使用随机梯度下降算法进行训练，否则使用标准方程法进行训练。

最后是使用：

; "复制代码")

import numpy as np import pandas as pd import matplotlib.pyplot as plt from sklearn.datasets import make_regression import sys
sys.path.append("/content/drive/My Drive/learn/ML-From-Scratch/") from mlfromscratch.utils import train_test_split, polynomial_features from mlfromscratch.utils import mean_squared_error, Plot from mlfromscratch.supervised_learning import LinearRegression def main():

X, y \= make\_regression(n\_samples=100, n\_features=1, noise=20)

X\_train, X\_test, y\_train, y\_test \= train\_test\_split(X, y, test\_size=0.4)

n\_samples, n\_features \= np.shape(X)

model \= LinearRegression(n\_iterations=100)

model.fit(X\_train, y\_train) # Training error plot
n = len(model.training\_errors)
training, \= plt.plot(range(n), model.training\_errors, label="Training Error")
plt.legend(handles\=\[training\])
plt.title("Error Plot")
plt.ylabel('Mean Squared Error')
plt.xlabel('Iterations')
plt.savefig("test1.png")
plt.show()

y\_pred \= model.predict(X\_test)
mse \= mean\_squared\_error(y\_test, y\_pred) print ("Mean squared error: %s" % (mse))

y\_pred\_line \= model.predict(X) # Color map
cmap = plt.get\_cmap('viridis') # Plot the results
m1 = plt.scatter(366 \* X\_train, y\_train, color=cmap(0.9), s=10)
m2 \= plt.scatter(366 \* X\_test, y\_test, color=cmap(0.5), s=10)
plt.plot(366 \* X, y\_pred\_line, color='black', linewidth=2, label="Prediction")
plt.suptitle("Linear Regression")
plt.title("MSE: %.2f" % mse, fontsize=10)
plt.xlabel('Day')
plt.ylabel('Temperature in Celcius')
plt.legend((m1, m2), ("Training data", "Test data"), loc='lower right')
plt.savefig("test2.png")
plt.show() if \_\_name\_\_ == "\_\_main\_\_":
main()

; "复制代码")

利用sklearn库生成线性回归数据，然后将其拆分为训练集和测试集。

utils下的mean_squared_error()：

def mean_squared_error(y_true, y_pred): """ Returns the mean squared error between y_true and y_pred """ mse = np.mean(np.power(y_true - y_pred, 2)) return mse

结果：

Mean squared error: 532.3321383700828