%% Gradient Descent
%% Code for Example 12.1

f = @(x) x^4 - 2*x^3 - 23*x^2 + 24*x + 147;
g = @(x) 4*x^3 - 6*x^2 - 46*x + 24;

x_prev = 2.0;     % initial guess
alpha = 0.01;     % step size

fprintf('i=%i, x=%g, f(x)=%g\n', 0, x_prev, f(x_prev));
for i = 1:10
    x_next = x_prev - alpha*g(x_prev);
    fprintf('i=%i, x=%g, f(x)=%g\n', i, x_next, f(x_next));
    x_prev = x_next;
end

%
%% Code for Example 12.2

x_true = [0.07 0.16 0.48 0.68 0.83]';
y_true = [-0.05 0.40 0.66 0.65 1.12]';

% the model matrix X = [1 x]
X = [ones(size(x_true)) x_true];

loss = @(b) 0.5*sum( (b(1) + b(2)*x_true - y_true).^2 )
g = @(b) [ sum( (b(1) + b(2)*x_true - y_true) );
           sum( (b(1) + b(2)*x_true - y_true) .* x_true ) ]
alpha = 0.1;    % step size
b = [0; 0];     % initial guess
N = 500;        % number of iterations
fprintf('i=%i, b0=%g, b1=%g, loss=%g\n',0,b(1),b(2),loss(b));
for i = 1:N
    b = b - alpha*g(b);
    if rem(i,N/10) == 0
        fprintf('i=%i, b0=%g, b1=%g, loss=%g\n',i,b(1),b(2),loss(b));
        %fprintf('%i & %g & %g & %g \\\\\n',i,loss(b),b(1),b(2));  % for printing LaTeX
    end
end

% use the pseudoinverse to find the true values (for comparison)
beta = pinv(X)*y_true
loss(beta)