Merge pull request github#4665 from yoff/python-dataflow-modernize-tests

Python: Add new-style tests

Author: RasmusWL

python/ql/src/semmle/python/dataflow/new/internal/DataFlowPrivate.qll

@@ -703,17 +703,6 @@ class SpecialCall extends DataFlowCall, TSpecialCall {
   }
 }
 
-/** A data flow node that represents a call argument. */
-class ArgumentNode extends Node {
-  ArgumentNode() { this = any(DataFlowCall c).getArg(_) }
-
-  /** Holds if this argument occurs at the given position in the given call. */
-  predicate argumentOf(DataFlowCall call, int pos) { this = call.getArg(pos) }
-
-  /** Gets the call in which this node is an argument. */
-  final DataFlowCall getCall() { this.argumentOf(result, _) }
-}
-
 /** Gets a viable run-time target for the call `call`. */
 DataFlowCallable viableCallable(DataFlowCall call) { result = call.getCallable() }
 

python/ql/src/semmle/python/dataflow/new/internal/DataFlowPublic.qll

@@ -193,6 +193,17 @@ class ParameterNode extends CfgNode {
   Parameter getParameter() { result = def.getParameter() }
 }
 
+/** A data flow node that represents a call argument. */
+class ArgumentNode extends Node {
+  ArgumentNode() { this = any(DataFlowCall c).getArg(_) }
+
+  /** Holds if this argument occurs at the given position in the given call. */
+  predicate argumentOf(DataFlowCall call, int pos) { this = call.getArg(pos) }
+
+  /** Gets the call in which this node is an argument. */
+  final DataFlowCall getCall() { this.argumentOf(result, _) }
+}
+
 /**
  * A node associated with an object after an operation that might have
  * changed its state.

python/ql/test/experimental/dataflow/TestUtil/FlowTest.qll

@@ -0,0 +1,40 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import TestUtilities.InlineExpectationsTest
+import experimental.dataflow.TestUtil.PrintNode
+
+abstract class FlowTest extends InlineExpectationsTest {
+  bindingset[this]
+  FlowTest() { any() }
+
+  abstract string flowTag();
+
+  abstract predicate relevantFlow(DataFlow::Node fromNode, DataFlow::Node toNode);
+
+  override string getARelevantTag() { result = this.flowTag() }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(DataFlow::Node fromNode, DataFlow::Node toNode | this.relevantFlow(fromNode, toNode) |
+      location = toNode.getLocation() and
+      tag = this.flowTag() and
+      value =
+        "\"" + prettyNode(fromNode).replaceAll("\"", "'") + lineStr(fromNode, toNode) + " -> " +
+          prettyNode(toNode).replaceAll("\"", "'") + "\"" and
+      element = toNode.toString()
+    )
+  }
+
+  pragma[inline]
+  private string lineStr(DataFlow::Node fromNode, DataFlow::Node toNode) {
+    exists(int delta |
+      delta = fromNode.getLocation().getStartLine() - toNode.getLocation().getStartLine()
+    |
+      if delta = 0
+      then result = ""
+      else
+        if delta > 0
+        then result = ", l:+" + delta.toString()
+        else result = ", l:" + delta.toString()
+    )
+  }
+}

python/ql/test/experimental/dataflow/TestUtil/LocalFlowStepTest.qll

@@ -0,0 +1,13 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import FlowTest
+
+class LocalFlowStepTest extends FlowTest {
+  LocalFlowStepTest() { this = "LocalFlowStepTest" }
+
+  override string flowTag() { result = "step" }
+
+  override predicate relevantFlow(DataFlow::Node fromNode, DataFlow::Node toNode) {
+    DataFlow::localFlowStep(fromNode, toNode)
+  }
+}

python/ql/test/experimental/dataflow/TestUtil/MaximalFlowTest.qll

@@ -0,0 +1,42 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+private import semmle.python.dataflow.new.internal.DataFlowPrivate
+import FlowTest
+
+class MaximalFlowTest extends FlowTest {
+  MaximalFlowTest() { this = "MaximalFlowTest" }
+
+  override string flowTag() { result = "flow" }
+
+  override predicate relevantFlow(DataFlow::Node source, DataFlow::Node sink) {
+    source != sink and
+    exists(MaximalFlowsConfig cfg | cfg.hasFlow(source, sink))
+  }
+}
+
+/**
+ * A configuration to find all "maximal" flows.
+ * To be used on small programs.
+ */
+class MaximalFlowsConfig extends DataFlow::Configuration {
+  MaximalFlowsConfig() { this = "MaximalFlowsConfig" }
+
+  override predicate isSource(DataFlow::Node node) {
+    exists(node.getLocation().getFile().getRelativePath()) and
+    not node.asCfgNode() instanceof CallNode and
+    not node.asCfgNode().getNode() instanceof Return and
+    not node instanceof DataFlow::ParameterNode and
+    not node instanceof DataFlow::PostUpdateNode and
+    // not node.asExpr() instanceof FunctionExpr and
+    // not node.asExpr() instanceof ClassExpr and
+    not exists(DataFlow::Node pred | DataFlow::localFlowStep(pred, node))
+  }
+
+  override predicate isSink(DataFlow::Node node) {
+    exists(node.getLocation().getFile().getRelativePath()) and
+    not any(CallNode c).getArg(_) = node.asCfgNode() and
+    not node instanceof DataFlow::ArgumentNode and
+    not node.asCfgNode().(NameNode).getId().matches("SINK%") and
+    not exists(DataFlow::Node succ | DataFlow::localFlowStep(node, succ))
+  }
+}

python/ql/test/experimental/dataflow/TestUtil/PrintNode.qll

@@ -0,0 +1,31 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+
+string prettyExp(Expr e) {
+  not e instanceof Num and
+  not e instanceof StrConst and
+  not e instanceof Subscript and
+  not e instanceof Call and
+  not e instanceof Attribute and
+  result = e.toString()
+  or
+  result = e.(Num).getN()
+  or
+  result =
+    e.(StrConst).getPrefix() + e.(StrConst).getText() +
+      e.(StrConst).getPrefix().regexpReplaceAll("[a-zA-Z]+", "")
+  or
+  result = prettyExp(e.(Subscript).getObject()) + "[" + prettyExp(e.(Subscript).getIndex()) + "]"
+  or
+  (
+    if exists(e.(Call).getAnArg()) or exists(e.(Call).getANamedArg())
+    then result = prettyExp(e.(Call).getFunc()) + "(..)"
+    else result = prettyExp(e.(Call).getFunc()) + "()"
+  )
+  or
+  result = prettyExp(e.(Attribute).getObject()) + "." + e.(Attribute).getName()
+}
+
+string prettyNode(DataFlow::Node node) {
+  if exists(node.asExpr()) then result = prettyExp(node.asExpr()) else result = node.toString()
+}

python/ql/test/experimental/dataflow/TestUtil/RoutingTest.qll

@@ -0,0 +1,51 @@
+import python
+import semmle.python.dataflow.new.DataFlow
+import TestUtilities.InlineExpectationsTest
+import experimental.dataflow.TestUtil.PrintNode
+
+/**
+ * A routing test is designed to test that values are routed to the
+ * correct arguments of the correct functions. It is assumed that
+ * the functions tested sink their arguments sequentially, that is
+ * `SINK1(arg1)`, etc.
+ */
+abstract class RoutingTest extends InlineExpectationsTest {
+  bindingset[this]
+  RoutingTest() { any() }
+
+  abstract string flowTag();
+
+  abstract predicate relevantFlow(DataFlow::Node fromNode, DataFlow::Node toNode);
+
+  override string getARelevantTag() { result in ["func", this.flowTag()] }
+
+  override predicate hasActualResult(Location location, string element, string tag, string value) {
+    exists(DataFlow::Node fromNode, DataFlow::Node toNode | this.relevantFlow(fromNode, toNode) |
+      location = fromNode.getLocation() and
+      element = fromNode.toString() and
+      (
+        tag = this.flowTag() and
+        if "\"" + tag + "\"" = fromValue(fromNode) then value = "" else value = fromValue(fromNode)
+        or
+        tag = "func" and
+        value = toFunc(toNode) and
+        not value = fromFunc(fromNode)
+      )
+    )
+  }
+
+  pragma[inline]
+  private string fromValue(DataFlow::Node fromNode) {
+    result = "\"" + prettyNode(fromNode).replaceAll("\"", "'") + "\""
+  }
+
+  pragma[inline]
+  private string fromFunc(DataFlow::ArgumentNode fromNode) {
+    result = fromNode.getCall().getNode().(CallNode).getFunction().getNode().(Name).getId()
+  }
+
+  pragma[inline]
+  private string toFunc(DataFlow::Node toNode) {
+    result = toNode.getEnclosingCallable().getCallableValue().getScope().getQualifiedName() // TODO: More robust pretty printing?
+  }
+}

python/ql/test/experimental/dataflow/basic/localFlowStepTest.expected

@@ -0,0 +1 @@
+import experimental.dataflow.TestUtil.LocalFlowStepTest

python/ql/test/experimental/dataflow/basic/localFlowStepTest.ql

@@ -0,0 +1 @@
+import experimental.dataflow.TestUtil.MaximalFlowTest

python/ql/test/experimental/dataflow/basic/maximalFlowTest.expected

@@ -1,7 +1,7 @@
-def obfuscated_id(x):
-  y = x
-  z = y
-  return z
+def obfuscated_id(x): #$ step="FunctionExpr -> GSSA Variable obfuscated_id" step="x -> SSA variable x"
+  y = x #$ step="x -> SSA variable y" step="SSA variable x, l:-1 -> x"
+  z = y #$ step="y -> SSA variable z" step="SSA variable y, l:-1 -> y"
+  return z #$ flow="42, l:+2 -> z" step="SSA variable z, l:-1 -> z"
 
-a = 42
-b = obfuscated_id(a)
+a = 42 #$ step="42 -> GSSA Variable a"
+b = obfuscated_id(a) #$ flow="42, l:-1 -> GSSA Variable b" flow="FunctionExpr, l:-6 -> obfuscated_id" step="obfuscated_id(..) -> GSSA Variable b" step="GSSA Variable obfuscated_id, l:-6 -> obfuscated_id" step="GSSA Variable a, l:-1 -> a"

python/ql/test/experimental/dataflow/basic/maximalFlowTest.ql

@@ -11,7 +11,7 @@ class CallGraphConfig extends DataFlow::Configuration {
   override predicate isSource(DataFlow::Node node) {
     node instanceof DataFlowPrivate::ReturnNode
     or
-    node instanceof DataFlowPrivate::ArgumentNode
+    node instanceof DataFlow::ArgumentNode
   }
 
   override predicate isSink(DataFlow::Node node) {

python/ql/test/experimental/dataflow/basic/test.py

@@ -86,12 +86,12 @@ def argument_passing(
 
 @expects(7)
 def test_argument_passing1():
-    argument_passing(arg1, *(arg2, arg3, arg4), e=arg5, **{"f": arg6, "g": arg7})
+    argument_passing(arg1, *(arg2, arg3, arg4), e=arg5, **{"f": arg6, "g": arg7})  #$ arg1 arg7 func=argument_passing MISSING: arg2 arg3="arg3 arg4 arg5 arg6
 
 
 @expects(7)
 def test_argument_passing2():
-    argument_passing(arg1, arg2, arg3, f=arg6)
+    argument_passing(arg1, arg2, arg3, f=arg6)  #$ arg1 arg2 arg3
 
 
 def with_pos_only(a, /, b):
@@ -101,9 +101,9 @@ def with_pos_only(a, /, b):
 
 @expects(6)
 def test_pos_only():
-    with_pos_only(arg1, arg2)
-    with_pos_only(arg1, b=arg2)
-    with_pos_only(arg1, *(arg2,))
+    with_pos_only(arg1, arg2)  #$ arg1 arg2
+    with_pos_only(arg1, b=arg2)  #$ arg1 arg2
+    with_pos_only(arg1, *(arg2,))  #$ arg1 MISSING: arg2
 
 
 def with_multiple_kw_args(a, b, c):
@@ -114,13 +114,13 @@ def with_multiple_kw_args(a, b, c):
 
 @expects(12)
 def test_multiple_kw_args():
-    with_multiple_kw_args(b=arg2, c=arg3, a=arg1)
-    with_multiple_kw_args(arg1, *(arg2,), arg3)
-    with_multiple_kw_args(arg1, **{"c": arg3}, b=arg2)
-    with_multiple_kw_args(**{"b": arg2}, **{"c": arg3}, **{"a": arg1})
+    with_multiple_kw_args(b=arg2, c=arg3, a=arg1)  #$ arg1 arg2 arg3
+    with_multiple_kw_args(arg1, *(arg2,), arg3)  #$ arg1 MISSING: arg2 arg3
+    with_multiple_kw_args(arg1, **{"c": arg3}, b=arg2)  #$ arg1 arg3 func=with_multiple_kw_args MISSING: arg2
+    with_multiple_kw_args(**{"b": arg2}, **{"c": arg3}, **{"a": arg1})  #$ arg1 arg2 arg3 func=with_multiple_kw_args
 
 
-def with_default_arguments(a=arg1, b=arg2, c=arg3):
+def with_default_arguments(a=arg1, b=arg2, c=arg3):  # Need a mechanism to test default arguments
     SINK1(a)
     SINK2(b)
     SINK3(c)
@@ -129,9 +129,9 @@ def with_default_arguments(a=arg1, b=arg2, c=arg3):
 @expects(12)
 def test_default_arguments():
     with_default_arguments()
-    with_default_arguments(arg1)
-    with_default_arguments(b=arg2)
-    with_default_arguments(**{"c": arg3})
+    with_default_arguments(arg1)  #$ arg1
+    with_default_arguments(b=arg2)  #$ arg2
+    with_default_arguments(**{"c": arg3})  #$ arg3 func=with_default_arguments
 
 
 # Nested constructor pattern
@@ -157,55 +157,55 @@ def grab_baz(baz):
 
 @expects(4)
 def test_grab():
-    grab_foo_bar_baz(baz=arg3, bar=arg2, foo=arg1)
+    grab_foo_bar_baz(baz=arg3, bar=arg2, foo=arg1)  #$ arg1 arg2 arg3 func=grab_bar_baz func=grab_baz
 
 
 # All combinations
 def test_pos_pos():
     def with_pos(a):
         SINK1(a)
 
-    with_pos(arg1)
+    with_pos(arg1)  #$ arg1 func=test_pos_pos.with_pos
 
 
 def test_pos_pos_only():
     def with_pos_only(a, /):
         SINK1(a)
 
-    with_pos_only(arg1)
+    with_pos_only(arg1)  #$ arg1 func=test_pos_pos_only.with_pos_only
 
 
 def test_pos_star():
     def with_star(*a):
         if len(a) > 0:
             SINK1(a[0])
 
-    with_star(arg1)
+    with_star(arg1)  #$ arg1 func=test_pos_star.with_star
 
 
 def test_pos_kw():
     def with_kw(a=""):
         SINK1(a)
 
-    with_kw(arg1)
+    with_kw(arg1)  #$ arg1 func=test_pos_kw.with_kw
 
 
 def test_kw_pos():
     def with_pos(a):
         SINK1(a)
 
-    with_pos(a=arg1)
+    with_pos(a=arg1)  #$ arg1 func=test_kw_pos.with_pos
 
 
 def test_kw_kw():
     def with_kw(a=""):
         SINK1(a)
 
-    with_kw(a=arg1)
+    with_kw(a=arg1)  #$ arg1 func=test_kw_kw.with_kw
 
 
 def test_kw_doublestar():
     def with_doublestar(**a):
         SINK1(a["a"])
 
-    with_doublestar(a=arg1)
+    with_doublestar(a=arg1)  #$ arg1 func=test_kw_doublestar.with_doublestar