Initial commit of jep-11, the let function

Proposed in jmespath/jmespath.site#6.

I tried to pick an implementation that was as minimally
invasive as possible.

It works by making three changes.

First we need to track scope, and share this information between
the interpreter and the function module.  They both take a
reference to a scope object that allows you to push/pop scopes.
The ``let()`` function will push the user provided lexical scope
onto the scope chain before evaluating the expref, and pop the
scope after evaluating the expref.

The second change needed is to change how identifiers are resolved.
This corresponds to visiting the ``field`` AST node.  As detailed
in JEP 11, after failing to resolve the field in the current object,
we call back to the scope chain.

The third change is to bind the current value (the context) in which
an expref is first created.  This wasn't needed before because for
functions that take an expref, such as ``sort_by``, ``max_by``, and
``min_by``, they evaluate the expref in the context of each list
element.  However, with ``let()``, we want to evaluate the expref
in the context of the current object as specified when the expref
was created.  This also tracks the current object properly in the
case of nested ``let()`` calls.

jmespath/functions.py

@@ -61,8 +61,9 @@ class RuntimeFunctions(object):
     FUNCTION_TABLE = {
     }
 
-    def __init__(self):
+    def __init__(self, scope):
         self._interpreter = None
+        self._scope = scope
 
     @property
     def interpreter(self):
@@ -326,6 +327,14 @@ def _func_max_by(self, array, expref):
                                         'min_by')
         return max(array, key=keyfunc)
 
+    @builtin_function({'types': ['object']}, {'types': ['expref']})
+    def _func_let(self, lexical_scope, expref):
+        self._scope.push_scope(lexical_scope)
+        try:
+            return self.interpreter.visit(expref.expression, expref.context)
+        finally:
+            self._scope.pop_scope()
+
     def _create_key_func(self, expr_node, allowed_types, function_name):
         interpreter = self.interpreter
 

jmespath/visitor.py

@@ -1,4 +1,5 @@
 import operator
+from collections import deque
 
 from jmespath import functions
 
@@ -34,8 +35,48 @@ def _is_special_integer_case(x, y):
 
 
 class _Expression(object):
-    def __init__(self, expression):
+    def __init__(self, expression, context):
         self.expression = expression
+        self.context = context
+
+
+class ScopedChainDict(object):
+    """Dictionary that can delegate lookups to multiple dicts.
+
+    This provides a basic get/set dict interface that is
+    backed by multiple dicts.  Each dict is searched from
+    the top most (most recently pushed) scope dict until
+    a match is found.
+
+    """
+    def __init__(self, *scopes):
+        # The scopes are evaluated starting at the top of the stack (the most
+        # recently pushed scope via .push_scope()).  If we use a normal list()
+        # and push/pop scopes by adding/removing to the end of the list, we'd
+        # have to always call reversed(self._scopes) whenever we resolve a key,
+        # because the end of the list is the top of the stack.
+        # To avoid this, we're using a deque so we can append to the front of
+        # the list via .appendleft() in constant time, and iterate over scopes
+        # without having to do so with a reversed() call each time.
+        self._scopes = deque(scopes)
+
+    def __getitem__(self, key):
+        for scope in self._scopes:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def get(self, key, default=None):
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def push_scope(self, scope):
+        self._scopes.appendleft(scope)
+
+    def pop_scope(self):
+        self._scopes.popleft()
 
 
 class Visitor(object):
@@ -69,7 +110,8 @@ class TreeInterpreter(Visitor):
 
     def __init__(self):
         super(TreeInterpreter, self).__init__()
-        self._functions = functions.RuntimeFunctions()
+        self._scope = ScopedChainDict()
+        self._functions = functions.RuntimeFunctions(self._scope)
         # Note that .interpreter is a property that uses
         # a weakref so that the cyclic reference can be
         # properly freed.
@@ -83,8 +125,13 @@ def visit_subexpression(self, node, value):
 
     def visit_field(self, node, value):
         try:
-            return value.get(node['value'])
-        except AttributeError:
+            return value[node['value']]
+        except KeyError:
+            # If the field is not defined in the current object, then fall back
+            # to checking in the scope chain, if there's any that has been
+            # created.
+            return self._scope.get(node['value'])
+        except (AttributeError, TypeError):
             return None
 
     def visit_comparator(self, node, value):
@@ -98,7 +145,7 @@ def visit_current(self, node, value):
         return value
 
     def visit_expref(self, node, value):
-        return _Expression(node['children'][0])
+        return _Expression(node['children'][0], value)
 
     def visit_function_expression(self, node, value):
         resolved_args = []

tests/compliance/functions.json

@@ -692,4 +692,121 @@
       ]
     }
   ]
+}, {
+  "given":
+  {
+     "search_for": "foo",
+     "people": [
+         {"name": "a"},
+         {"name": "b"},
+         {"name": "c"},
+         {"name": "foo"},
+         {"name": "bar"},
+         {"name": "baz"},
+         {"name": "qux"},
+         {"name": "x"},
+         {"name": "y"},
+         {"name": "z"}
+     ]
+  },
+  "cases": [
+    {
+      "description": "Let function with filters",
+      "expression": "let({search_for: search_for}, &people[?name==search_for].name | [0])",
+      "result": "foo"
+    }
+  ]
+}, {
+  "given":
+  {
+     "a": {
+         "mylist": [
+           {"l1": "1", "result": "foo"},
+           {"l2": "2", "result": "bar"},
+           {"l1": "8", "l2": "9"},
+           {"l1": "8", "l2": "9"}
+         ],
+         "level2": "2"
+     },
+     "level1": "1",
+     "nested": {
+         "a": {
+             "b": {
+                 "c": {
+                     "fourth": "fourth"
+                 },
+                 "third": "third"
+             },
+             "second": "second"
+         },
+         "first": "first"
+     },
+     "precedence": {
+         "a": {
+             "b": {
+                 "c": {
+                     "variable": "fourth"
+                 },
+                 "variable": "third",
+                 "other": "y"
+             },
+             "variable": "second",
+             "other": "x"
+         },
+         "variable": "first",
+         "other": "w"
+     }
+  },
+  "cases": [
+    {
+      "description": "Basic let from scope",
+      "expression": "let({level1: level1}, &a.[level2, level1])",
+      "result": ["2", "1"]
+    },
+    {
+      "description": "Current object has precedence",
+      "expression": "let({level1: `other`}, &level1)",
+      "result": "1"
+    },
+    {
+      "description": "No scope specified using literal hash",
+      "expression": "let(`{}`, &a.level2)",
+      "result": "2"
+    },
+    {
+      "description": "Arbitrary variable added",
+      "expression": "let({foo: `anything`}, &[level1, foo])",
+      "result": ["1", "anything"]
+    },
+    {
+      "description": "Basic let from current object",
+      "expression": "let({other: level1}, &level1)",
+      "result": "1"
+    },
+    {
+      "description": "Nested let function with filters",
+      "expression": "let({level1: level1}, &a.[mylist[?l1==level1].result, let({level2: level2}, &mylist[?l2==level2].result)])[]",
+      "result": ["foo", "bar"]
+    },
+    {
+      "description": "Nested let function with filters with literal scope binding",
+      "expression": "let(`{\"level1\": \"1\"}`, &a.[mylist[?l1==level1].result, let({level2: level2}, &mylist[?l2==level2].result)])[]",
+      "result": ["foo", "bar"]
+    },
+    {
+      "description": "Nested let functions",
+      "expression": "nested.let({level1: first}, &a.let({level2: second}, &b.let({level3: third}, &c.{first: level1, second: level2, third: level3, fourth: fourth})))",
+      "result": {"first": "first", "second": "second", "third": "third", "fourth": "fourth"}
+    },
+    {
+      "description": "Precedence of lexical vars from scope object",
+      "expression": "precedence.let({other: other}, &a.let({other: other}, &b.let({other: other}, &c.{other: other})))",
+      "result": {"other": "y"}
+    },
+    {
+      "description": "Precedence of lexical vars from current object",
+      "expression": "precedence.let({variable: variable}, &a.let({variable: variable}, &b.let({variable: variable}, &c.let({variable: `override`}, &variable))))",
+      "result": "fourth"
+    }
+  ]
 }]