93 lines
2.6 KiB
Python
93 lines
2.6 KiB
Python
import math
|
|
from math import sin, pi
|
|
from random import random
|
|
|
|
|
|
def f(x):
|
|
return sin(pi * x)
|
|
|
|
|
|
def generate_training_examples(n=2):
|
|
xs = [random() * 2 - 1 for _ in range(n)]
|
|
return [(x, f(x)) for x in xs]
|
|
|
|
|
|
def fit_without_reg(examples):
|
|
"""Computes values of w0 and w1 that minimize the sum-of-squared-errors cost function
|
|
|
|
Args:
|
|
- examples: a list of two (x, y) tuples, where x is the feature and y is the label
|
|
"""
|
|
w0 = 0
|
|
w1 = 0
|
|
|
|
x1, y1 = examples[0]
|
|
x2, y2 = examples[1]
|
|
|
|
w1 = (y2 - y1) / (x2 - x1)
|
|
w0 = y1 - w1 * x1
|
|
|
|
return w0, w1
|
|
|
|
|
|
def fit_with_reg(examples, lambda_hp, learning_rate=0.01, max_iter=1000, tol=1e-6):
|
|
"""Computes values of w0 and w1 that minimize the regularized sum-of-squared-errors cost function
|
|
|
|
Args:
|
|
- examples: a list of two (x, y) tuples, where x is the feature and y is the label
|
|
- lambda_hp: a float representing the value of the lambda hyperparameter; a larger value means more regularization
|
|
"""
|
|
w0 = 0
|
|
w1 = 0
|
|
for _ in range(max_iter):
|
|
# Compute the sum of squared errors
|
|
sse = sum((w0 + w1 * x - y) ** 2 for x, y in examples)
|
|
|
|
# Compute the regularization term
|
|
reg_term = lambda_hp * (w0 ** 2 + w1 ** 2)
|
|
|
|
# Compute the total cost function
|
|
cost = sse + reg_term
|
|
|
|
# Compute the partial derivatives of the cost function with respect to w0 and w1
|
|
d_w0 = 2 * sum(w0 + w1 * x - y for x, y in examples) + 2 * lambda_hp * w0
|
|
d_w1 = 2 * sum((w0 + w1 * x - y) * x for x, y in examples) + 2 * lambda_hp * w1
|
|
|
|
# Update w0 and w1 using gradient descent
|
|
w0_new = w0 - learning_rate * d_w0
|
|
w1_new = w1 - learning_rate * d_w1
|
|
|
|
# Check for convergence
|
|
if abs(w0_new - w0) < tol and abs(w1_new - w1) < tol:
|
|
break
|
|
|
|
w0, w1 = w0_new, w1_new
|
|
|
|
return w0, w1
|
|
|
|
|
|
def test_error(w0, w1):
|
|
n = 100
|
|
xs = [i/n for i in range(-n, n + 1)]
|
|
return sum((w0 + w1 * x - f(x)) ** 2 for x in xs) / len(xs)
|
|
|
|
def main():
|
|
print("Starting...")
|
|
no_reg_errors=[]
|
|
reg_erros = []
|
|
exp_size = 1000
|
|
for _ in range(exp_size):
|
|
|
|
train_examples = generate_training_examples(n=2)
|
|
|
|
w0, w1 = fit_without_reg(train_examples)
|
|
w0_r, w1_r = fit_with_reg(train_examples, lambda_hp=1)
|
|
|
|
no_reg_errors.append(test_error(w0, w1))
|
|
reg_erros.append(test_error(w0_r, w1_r))
|
|
|
|
print(f"Avg loss, regularization: {sum(no_reg_errors)/(len(no_reg_errors))}")
|
|
print(f"Avg loss, no regularization: {sum(reg_erros)/(len(reg_erros))}")
|
|
|
|
if __name__=="__main__":
|
|
main() |