Feature addition: Backtracking Line Search for optimization

tensorflow_probability/python/optimizer/linesearch/BUILD

@@ -29,6 +29,7 @@ multi_substrate_py_library(
     srcs = ["__init__.py"],
     deps = [
         ":hager_zhang",
+        ":backtracking",
         "//tensorflow_probability/python/internal:all_util",
         "//tensorflow_probability/python/optimizer/linesearch/internal",
     ],
@@ -45,6 +46,16 @@ multi_substrate_py_library(
     ],
 )
 
+multi_substrate_py_library(
+    name = "backtracking",
+    srcs = ["backtracking.py"],
+    srcs_version = "PY3",
+    deps = [
+        # tensorflow dep,
+        "//tensorflow_probability/python/internal:dtype_util",
+        ],
+)
+
 multi_substrate_py_test(
     name = "hager_zhang_test",
     size = "medium",
@@ -59,3 +70,18 @@ multi_substrate_py_test(
         "//tensorflow_probability/python/internal:test_util",
     ],
 )
+
+multi_substrate_py_test(
+    name = "backtracking_test",
+    size = "small",
+    srcs = ["backtracking_test.py"],
+    numpy_tags = ["notap"],
+    shard_count = 5,
+    deps = [
+        # absl/testing:parameterized dep,
+        # numpy dep,
+        # tensorflow dep,
+        "//tensorflow_probability",
+        "//tensorflow_probability/python/internal:test_util",
+    ],
+)

tensorflow_probability/python/optimizer/linesearch/__init__.py

@@ -20,10 +20,12 @@
 
 from tensorflow_probability.python.internal import all_util
 from tensorflow_probability.python.optimizer.linesearch.hager_zhang import hager_zhang
+from tensorflow_probability.python.optimizer.linesearch.backtracking import backtracking
 
 
 _allowed_symbols = [
     'hager_zhang',
+    'backtracking',
 ]
 
 all_util.remove_undocumented(__name__, _allowed_symbols)

tensorflow_probability/python/optimizer/linesearch/backtracking.py

@@ -0,0 +1,34 @@
+# Copyright 2018 The TensorFlow Probability Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Implements the Backtracking line search algorithm.
+Line searches are a central component for many optimization algorithms (e.g.
+BFGS, conjugate gradient, ISTA, FISTA etc). Sophisticated line search methods 
+aim to find the appropriate step length.
+This module implements the Backtracking Line Search Algorithm.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+def backtracking ( function,
+                   differentiation,
+                   value,
+                   beta = 0.707,
+                   alpha = 1): 
+
+  while function(value-(alpha*differentiation(value)))>function(value) - 
+  (alpha/2)*((differentiation(value))**2):
+    alpha *= beta
+  return alpha

tensorflow_probability/python/optimizer/linesearch/backtracking_test.py

@@ -0,0 +1,23 @@
+import unittest
+import backtracking
+from tensorflow_probability.python.internal import test_util
+
+class TestBacktracking(unittest.TestCase):
+
+
+
+    def test_ndegree(self):
+         self.assertEqual(backtracking.backtracking
+                          (lambda x: x**2 +3*x, lambda x: 2*x + 3,11), 0.49984899999999993)
+
+         self.assertEqual(backtracking.backtracking
+                          (lambda x: x**10 +3*x, lambda x: 10*(x**9) + 3,2), 6.07776055631376e-05)
+
+         self.assertEqual(backtracking.backtracking
+                          (lambda x: x**5 - 3*x, lambda x: 5*(x**4) - 3,2),1)
+
+if __name__ == '__main__':
+    unittest.main()
+
+if __name__ == '__main__':
+  test_util.main()