From 98f221936661d665fac7ca0b62b956f8ee35e3c9 Mon Sep 17 00:00:00 2001
From: Ziver Koc <ziver@koc.se>
Date: Mon, 1 Oct 2018 17:06:53 +0200
Subject: [PATCH] Fixed gradient decent

---
 src/zutil/ml/LinearRegression.java      | 27 +++++++----------
 test/zutil/ml/LinearRegressionTest.java | 39 +++++++++++++++++++++----
 2 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/src/zutil/ml/LinearRegression.java b/src/zutil/ml/LinearRegression.java
index 5dd97d1..94cb6ce 100755
--- a/src/zutil/ml/LinearRegression.java
+++ b/src/zutil/ml/LinearRegression.java
@@ -40,17 +40,8 @@ public class LinearRegression {
                 Matrix.Elemental.pow(normalized,2));
     }
 
-    /**
-     * Calculates the gradiant of the current provided theta.
-     */
-    protected static double calculateGradiant(double[][] x, double[] y, double[] theta){
-        int m = y.length; // number of training examples
-        double[] hypothesis = calculateHypothesis(x, theta);
-        double[] normalized = Matrix.subtract(hypothesis, y);
-
-        return 1/m * Matrix.sum(
-                Matrix.Elemental.multiply(Matrix.transpose(x), normalized));
-
+    private static double calculateDiff(double[] vector1, double[] vector2){
+        return Math.abs(Matrix.sum(vector1) - Matrix.sum(vector2));
     }
 
     /**
@@ -58,12 +49,16 @@ public class LinearRegression {
      */
     public static double[] gradientDescent(double[][] x, double[] y, double[] theta, double alpha){
         double[] newTheta = theta.clone();
-        double gradient;
+        double[] prevTheta = new double[newTheta.length];
+        double thetaDiff = 0;
+        int i = 0;
 
-        for (int i=0; (gradient = calculateGradiant(x, y, newTheta)) != 0; i++) {
-            logger.fine("Gradient Descent iteration " + i + ", gradiant: " + gradient);
+        do {
+            logger.fine("Gradient Descent iteration " + i + ", diff to previous iteration: " + thetaDiff);
+            System.arraycopy(newTheta, 0, prevTheta, 0, newTheta.length);
             newTheta = gradientDescentIteration(x, y, newTheta, alpha);
-        }
+            ++i;
+        } while ((thetaDiff=calculateDiff(prevTheta, newTheta)) > 0.0001);
 
         return newTheta;
     }
@@ -84,7 +79,7 @@ public class LinearRegression {
         double[] normalized = Matrix.subtract(hypothesis, y);
 
         for (int j= 0; j < theta.length; j++) {
-            newTheta[j] = theta[j] - alpha * (1.0/m) * Matrix.sum(
+            newTheta[j] = theta[j] - (alpha/m) * Matrix.sum(
                     Matrix.Elemental.multiply(normalized, Matrix.getColumn(x, j)));
         }
 
diff --git a/test/zutil/ml/LinearRegressionTest.java b/test/zutil/ml/LinearRegressionTest.java
index 9ba9c25..7bed265 100755
--- a/test/zutil/ml/LinearRegressionTest.java
+++ b/test/zutil/ml/LinearRegressionTest.java
@@ -1,6 +1,7 @@
 package zutil.ml;
 
 import org.junit.Test;
+import zutil.io.MultiPrintStream;
 import zutil.log.LogUtil;
 
 import java.util.logging.Level;
@@ -36,8 +37,8 @@ public class LinearRegressionTest {
     }
 
     // Does not work
-    //@Test
-    public void gradientAscent() {
+    @Test
+    public void gradientDescent() {
         double[][] x = {
                 {1.0, 0.1, 0.6, 1.1},
                 {1.0, 0.2, 0.7, 1.2},
@@ -59,14 +60,40 @@ public class LinearRegressionTest {
                 2
         };
 
-        double[] resultTheta = LinearRegression.gradientDescent(x, y, theta, 0);
+        // Alpha zero
 
-        assertEquals(0.73482, LinearRegression.calculateCost(x, y, resultTheta), 0.000001);
+        double[] resultTheta = LinearRegression.gradientDescent(x, y, theta, 0);
+        System.out.println("Result Theta (alpha = 0):");
+        System.out.println(MultiPrintStream.dumpToString(resultTheta));
+
+        assertArrayEquals(theta, resultTheta, 0.000001);
+
+        // Alpha +
+
+        resultTheta = LinearRegression.gradientDescent(x, y, theta, 0.1);
+        System.out.println("Result Theta (alpha = 0.1):");
+        System.out.println(MultiPrintStream.dumpToString(resultTheta));
+
+        assertArrayEquals(
+                new double[]{-1.31221, -1.98259, 0.36131, 1.70520},
+                resultTheta, 0.001);
     }
 
     @Test
-    public void gradientAscentIteration() {
-        double[] theta = LinearRegression.gradientDescentIteration( // one iteration
+    public void gradientDescentIteration() {
+        // Zero iterations
+
+        double[] theta = LinearRegression.gradientDescentIteration(
+                /* x */ new double[][]{{1, 5},{1, 2},{1, 4},{1, 5}},
+                /* y */ new double[]{1, 6, 4, 2},
+                /* theta */ new double[]{0, 0},
+                /* alpha */0.0);
+
+        assertArrayEquals(new double[]{0.0, 0.0}, theta, 0.000001);
+
+        // One iteration
+
+        theta = LinearRegression.gradientDescentIteration(
                 /* x */ new double[][]{{1, 5},{1, 2},{1, 4},{1, 5}},
                 /* y */ new double[]{1, 6, 4, 2},
                 /* theta */ new double[]{0, 0},