Add Adadelta optimizer (#150)

* Add Adadelta optimizer

* Post review update

Author: Lewuathe

src/graph/optimizers/adadelta_optimizer.ts

@@ -0,0 +1,108 @@
+/**
+ * @license
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =============================================================================
+ */
+
+import {NDArrayMath} from '../../math/math';
+import {NDArray, Scalar} from '../../math/ndarray';
+import {Node} from '../graph';
+import {SessionRuntime} from '../session';
+import {SummedTensorArrayMap, TensorArrayMap} from '../tensor_array_map';
+
+import {Optimizer} from './optimizer';
+
+export class AdadeltaOptimizer extends Optimizer {
+  constructor(
+      protected learningRate: number, private gamma: number,
+      specifiedVariableList?: Node[]) {
+    super(learningRate, specifiedVariableList);
+    this.eps = Scalar.new(1e-6);
+    this.g = Scalar.new(this.gamma);
+  }
+
+  beforeBatch(
+      math: NDArrayMath, batchSize: number, runtime: SessionRuntime,
+      activationArrayMap: TensorArrayMap,
+      gradientArrayMap: SummedTensorArrayMap) {
+    super.beforeBatch(
+        math, batchSize, runtime, activationArrayMap, gradientArrayMap);
+    if (this.accumulatedSquaredGradients.size() === 0) {
+      this.variableNodes.forEach(node => {
+        this.accumulatedSquaredGradients.set(
+            node.output, NDArray.zeros(node.output.shape));
+        this.accumulatedUpdates.set(
+            node.output, NDArray.zeros(node.output.shape));
+      });
+    }
+  }
+
+  afterBatch(
+      math: NDArrayMath, batchSize: number, runtime: SessionRuntime,
+      activationArrayMap: TensorArrayMap,
+      gradientArrayMap: SummedTensorArrayMap) {
+    math.scope((keep) => {
+      this.variableNodes.forEach(node => {
+        const oldVariable = activationArrayMap.get(node.output);
+        const gradient = this.variableGradients.get(node.output);
+        const oldCache = this.accumulatedSquaredGradients.get(node.output);
+        const oldUpdates = this.accumulatedUpdates.get(node.output);
+
+        const gradientSquare = math.multiply(gradient, gradient);
+        // Exponential decay of average squared gradients.
+        const cache = math.scaledArrayAdd(
+            this.g, oldCache, math.sub(this.one, this.g), gradientSquare);
+
+        const updates = math.multiply(
+            math.divide(
+                math.sqrt(math.add(oldUpdates, this.eps)),
+                math.sqrt(math.add(oldCache, this.eps))),
+            gradient);
+
+        const variable =
+            math.scaledArrayAdd(this.c, updates, this.one, oldVariable);
+
+        const updateSquare = math.multiply(updates, updates);
+        // Exponential decay of average updated values.
+        const newUpdates = math.scaledArrayAdd(
+            this.g, oldUpdates, math.sub(this.one, this.g), updateSquare);
+
+        this.accumulatedSquaredGradients.set(node.output, keep(cache));
+        this.accumulatedUpdates.set(node.output, keep(newUpdates));
+        activationArrayMap.set(node.output, keep(variable));
+        node.data = variable;
+
+        oldVariable.dispose();
+        oldCache.dispose();
+        oldUpdates.dispose();
+      });
+    });
+
+    this.variableGradients.dispose();
+    this.variableGradients = new TensorArrayMap();
+  }
+
+  dispose() {
+    super.dispose();
+    this.eps.dispose();
+    this.g.dispose();
+    this.accumulatedSquaredGradients.dispose();
+    this.accumulatedUpdates.dispose();
+  }
+
+  private accumulatedSquaredGradients = new TensorArrayMap();
+  private accumulatedUpdates = new TensorArrayMap();
+  private eps: Scalar;
+  private g: Scalar;
+}

src/graph/session_test.ts

@@ -26,6 +26,7 @@ import {AdagradOptimizer} from './optimizers/adagrad_optimizer';
 import {MomentumOptimizer} from './optimizers/momentum_optimizer';
 import {RMSPropOptimizer} from './optimizers/rmsprop_optimizer';
 import {SGDOptimizer} from './optimizers/sgd_optimizer';
+import {AdadeltaOptimizer} from './optimizers/adadelta_optimizer';
 import {FeedDictionary, FeedEntry, Session} from './session';
 
 
@@ -447,4 +448,56 @@ describe('Session', () => {
             session.train(w, [{tensor: x, data: inputProvider}], 1, optimizer))
         .toThrowError();
   });
+
+  it('adadelta', () => {
+    const x = g.placeholder('x', [2]);
+    const w = g.variable('w', NDArray.zeros([1, 2]));
+    const b = g.variable('b', NDArray.zeros([1]));
+    const y = g.reduceSum(g.add(g.matmul(w, x), b));
+
+    const safeMode = true;
+    const optimizer = new AdadeltaOptimizer(0.1, 0.8);
+    const math = new NDArrayMathCPU(safeMode);
+    const session = new Session(g, math);
+    const inputProvider: InputProvider = {
+      getNextCopy() {
+        return Array1D.new([2, 4]);
+      },
+      disposeCopy(math, example) {}
+    };
+
+    math.scope(() => {
+      // w = reduce_sum(w_1*x_1 + w_2*x_2 + b)
+      // cache = [gamma*old_cache_w1 + (1-gamma)*grad_w1**2,
+      //            gamma*old_cache_w2 + (1-gamma)*grad_w2**2]
+      //            = [.8, 3.2]
+      // updates = [sqrt(old_updates_w1 + eps)/sqrt(old_cache_w1 + eps)*grad_w1,
+      //            sqrt(old_updates_w2 + eps)/sqrT(old_cache_w2 + eps)*grad_w2]
+      //            = [2, 4]
+      // w = [ w1_old - lr*updates_w1,
+      //            w2_old - lr*updates_w2]
+      //            = [-0.2, -0.4]
+      // new_updates = [gamma * old_updates_w1 + (1 - gamma) * 2**2,
+      //                gamma * old_updates_w2 + (1 - gamma) * 4**2]
+      //             = [0.8, 3.2]
+      // 
+      session.train(y, [{tensor: x, data: inputProvider}], 1, optimizer);
+      const dydw = session.activationArrayMap.get(w).getValues();
+      test_util.expectArraysClose(
+          dydw, new Float32Array([-0.2, -0.4]), 1e-5);
+      // cache = [gamma*old_cache_w1 + (1-gamma)*grad_w1**2,
+      //            gamma*old_cache_w2 + (1-gamma)*grad_w2**2]
+      //            = [1.44, 5.76]
+      // updates = [sqrt(old_updates_w1 + eps)/sqrt(old_cache_w1 + eps)*grad_w1,
+      //            sqrt(old_updates_w2 + eps)/sqrT(old_cache_w2 + eps)*grad_w2]
+      //            = [2, 4]
+      // w = [ w1_old - lr*updates_w1,
+      //            w2_old - lr*updates_w2]
+      //            = [-0.4, -0.8]
+      session.train(y, [{tensor: x, data: inputProvider}], 1, optimizer);
+      const dydw2 = session.activationArrayMap.get(w).getValues();
+      test_util.expectArraysClose(
+          dydw2, new Float32Array([-.4, -.8]), 2e-5);
+    });
+  });
 });

src/index.ts

@@ -31,6 +31,7 @@ export {ENV, Features} from './environment';
 export {Graph, Tensor} from './graph/graph';
 export {AdagradOptimizer} from './graph/optimizers/adagrad_optimizer';
 export {MomentumOptimizer} from './graph/optimizers/momentum_optimizer';
+export {AdadeltaOptimizer} from './graph/optimizers/adadelta_optimizer';
 export {Optimizer} from './graph/optimizers/optimizer';
 export {RMSPropOptimizer} from './graph/optimizers/rmsprop_optimizer';
 export {SGDOptimizer} from './graph/optimizers/sgd_optimizer';