From cf94310598d7c2ae32fdafdd3f7b959eb21da079 Mon Sep 17 00:00:00 2001
From: Ziver Koc <ziver@koc.se>
Date: Wed, 5 Sep 2018 16:37:19 +0200
Subject: [PATCH] Added more elemental matrix math and a non functioning
 gradient decent function.

---
 src/zutil/math/Matrix.java              | 40 +++++++++++++++++++++++--
 src/zutil/ml/LinearRegression.java      | 35 ++++++++++++++++++++--
 test/zutil/math/MatrixTest.java         | 20 ++++++++++++-
 test/zutil/ml/LinearRegressionTest.java | 34 ++++++++++++++++++++-
 4 files changed, 123 insertions(+), 6 deletions(-)

diff --git a/src/zutil/math/Matrix.java b/src/zutil/math/Matrix.java
index d382e0f..430db0b 100755
--- a/src/zutil/math/Matrix.java
+++ b/src/zutil/math/Matrix.java
@@ -146,6 +146,42 @@ public class Matrix {
             return result;
         }
 
+        /**
+         * Matrix Vector elemental multiplication, each element column in the matrix will be
+         * multiplied with the column  of the vector.
+         *
+         * @return a new matrix with the result
+         */
+        public static double[][] multiply(double[][] matrix, double[] vector){
+            vectorPreCheck(matrix, vector);
+            double[][] result = new double[matrix.length][matrix[0].length];
+
+            for (int y=0; y < matrix.length; ++y) {
+                for (int x=0; x<matrix[0].length; ++x) {
+                    result[y][x] = matrix[y][x] * vector[x];
+                }
+            }
+            return result;
+        }
+
+        /**
+         * Matrix Vector elemental division, each element column in the matrix will be
+         * divided with the column of the vector.
+         *
+         * @return a new matrix with the result
+         */
+        public static double[][] divide(double[][] matrix, double[] vector){
+            vectorPreCheck(matrix, vector);
+            double[][] result = new double[matrix.length][matrix[0].length];
+
+            for (int y=0; y < matrix.length; ++y) {
+                for (int x=0; x<matrix[0].length; ++x) {
+                    result[y][x] = matrix[y][x] / vector[x];
+                }
+            }
+            return result;
+        }
+
         /**
          * Element exponential, each element in the vector will raised
          * to the power of the exp paramter
@@ -181,7 +217,7 @@ public class Matrix {
             if (matrix1.length != matrix2.length || matrix1[0].length != matrix2[0].length)
                 throw new IllegalArgumentException("Matrices need to be of same dimension: " +
                         "matrix1 " + matrix1.length + "x" + matrix1[0].length + ", " +
-                        "matrix2 " + matrix2.length + "x" + matrix2[0].length + ", ");
+                        "matrix2 " + matrix2.length + "x" + matrix2[0].length);
         }
     }
 
@@ -374,7 +410,7 @@ public class Matrix {
         if (matrix1[0].length != matrix2.length)
             throw new IllegalArgumentException("Matrix1 columns need to match Matrix2 rows: " +
                     "matrix1 " + matrix1.length + "x" + matrix1[0].length + ", " +
-                    "matrix2 " + matrix2.length + "x" + matrix2[0].length + ", ");
+                    "matrix2 " + matrix2.length + "x" + matrix2[0].length);
     }
 
     /***********************************************************************
diff --git a/src/zutil/ml/LinearRegression.java b/src/zutil/ml/LinearRegression.java
index 0d8d4a5..5dd97d1 100755
--- a/src/zutil/ml/LinearRegression.java
+++ b/src/zutil/ml/LinearRegression.java
@@ -1,13 +1,16 @@
 package zutil.ml;
 
+import zutil.log.LogUtil;
 import zutil.math.Matrix;
 
+import java.util.logging.Logger;
+
 /**
- * Implementation of a Linear Regression algorithm for "predicting"
+ * Implementation of a Linear Regression algorithm for predicting
  * numerical values depending on specific input
  */
 public class LinearRegression {
-
+    private static final Logger logger = LogUtil.getLogger();
 
     /**
      * Method for calculating a hypothesis value fr a specific input value x.
@@ -37,6 +40,34 @@ public class LinearRegression {
                 Matrix.Elemental.pow(normalized,2));
     }
 
+    /**
+     * Calculates the gradiant of the current provided theta.
+     */
+    protected static double calculateGradiant(double[][] x, double[] y, double[] theta){
+        int m = y.length; // number of training examples
+        double[] hypothesis = calculateHypothesis(x, theta);
+        double[] normalized = Matrix.subtract(hypothesis, y);
+
+        return 1/m * Matrix.sum(
+                Matrix.Elemental.multiply(Matrix.transpose(x), normalized));
+
+    }
+
+    /**
+     * Will try to find the best theta value.
+     */
+    public static double[] gradientDescent(double[][] x, double[] y, double[] theta, double alpha){
+        double[] newTheta = theta.clone();
+        double gradient;
+
+        for (int i=0; (gradient = calculateGradiant(x, y, newTheta)) != 0; i++) {
+            logger.fine("Gradient Descent iteration " + i + ", gradiant: " + gradient);
+            newTheta = gradientDescentIteration(x, y, newTheta, alpha);
+        }
+
+        return newTheta;
+    }
+
     /**
      * Gradient Descent algorithm
      * <br /><br />
diff --git a/test/zutil/math/MatrixTest.java b/test/zutil/math/MatrixTest.java
index 5f5e9d2..1ea4970 100755
--- a/test/zutil/math/MatrixTest.java
+++ b/test/zutil/math/MatrixTest.java
@@ -125,7 +125,7 @@ public class MatrixTest {
     }
 
     @Test
-    public void vectorDivision(){
+    public void vectorMatrixDivision(){
         assertArrayEquals(
                 new double[]{4,1},
                 Matrix.divide(new double[][]{{2,4},{-4,10}}, new double[]{1,2}),
@@ -133,6 +133,24 @@ public class MatrixTest {
         );
     }
 
+    @Test
+    public void vectorMatrixElementalMultiply(){
+        assertArrayEquals(
+                new double[][]{{1, 4, 9}, {1, 6, 12}, {1, 8, 15}, {1, 10, 18}},
+                Matrix.Elemental.multiply(
+                        new double[][]{{1, 2, 3}, {1, 3, 4}, {1, 4, 5}, {1, 5, 6}},
+                        new double[]{1, 2, 3}));
+    }
+
+    @Test
+    public void vectorMatrixElementalDivision(){
+        assertArrayEquals(
+                new double[][]{{2,2},{-4,5}},
+                Matrix.Elemental.divide(
+                        new double[][]{{2,4},{-4,10}},
+                        new double[]{1,2}));
+    }
+
     @Test
     public void vectorSum(){
         assertEquals(
diff --git a/test/zutil/ml/LinearRegressionTest.java b/test/zutil/ml/LinearRegressionTest.java
index a96c8d6..9ba9c25 100755
--- a/test/zutil/ml/LinearRegressionTest.java
+++ b/test/zutil/ml/LinearRegressionTest.java
@@ -1,6 +1,9 @@
 package zutil.ml;
 
 import org.junit.Test;
+import zutil.log.LogUtil;
+
+import java.util.logging.Level;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -32,8 +35,37 @@ public class LinearRegressionTest {
         assertEquals(11.9450, cost, 0.0001);
     }
 
-    @Test
+    // Does not work
+    //@Test
     public void gradientAscent() {
+        double[][] x = {
+                {1.0, 0.1, 0.6, 1.1},
+                {1.0, 0.2, 0.7, 1.2},
+                {1.0, 0.3, 0.8, 1.3},
+                {1.0, 0.4, 0.9, 1.4},
+                {1.0, 0.5, 1.0, 1.5}
+        };
+        double[] y = {
+                1,
+                0,
+                1,
+                0,
+                1
+        };
+        double[] theta = {
+                -2,
+                -1,
+                1,
+                2
+        };
+
+        double[] resultTheta = LinearRegression.gradientDescent(x, y, theta, 0);
+
+        assertEquals(0.73482, LinearRegression.calculateCost(x, y, resultTheta), 0.000001);
+    }
+
+    @Test
+    public void gradientAscentIteration() {
         double[] theta = LinearRegression.gradientDescentIteration( // one iteration
                 /* x */ new double[][]{{1, 5},{1, 2},{1, 4},{1, 5}},
                 /* y */ new double[]{1, 6, 4, 2},