don't call staged functions on abstract types during inference

this is a bit drastic, but otherwise there is too much risk of type
unsoundness. fixes #8504

base/inference.jl

@@ -646,8 +646,14 @@ let stagedcache=Dict{Any,Any}()
         elseif haskey(stagedcache,(m,tt,env))
             return stagedcache[(m,tt,env)].code
         else
-            f=ccall(:jl_instantiate_staged,Any,(Any,Any,Any),m,tt,env)
-            stagedcache[(m,tt,env)]=f
+            if !isleaftype(tt)
+                # don't call staged functions on abstract types.
+                # (see issues #8504, #10230)
+                # we can't guarantee that their type behavior is monotonic.
+                error()
+            end
+            f = ccall(:jl_instantiate_staged,Any,(Any,Any,Any),m,tt,env)
+            stagedcache[(m,tt,env)] = f
             return f.code
         end
     end

base/reducedim.jl

@@ -190,7 +190,7 @@ stagedfunction _mapreducedim!{T,N}(f, op, R::AbstractArray, A::AbstractArray{T,N
     end
 end
 
-mapreducedim!(f, op, R::AbstractArray, A::AbstractArray) = _mapreducedim!(f, op, R, A)
+mapreducedim!(f, op, R::AbstractArray, A::AbstractArray) = (_mapreducedim!(f, op, R, A); R)
 
 to_op(op) = op
 function to_op(op::Function)
@@ -201,7 +201,7 @@ function to_op(op::Function)
 end
 
 mapreducedim!(f, op, R::AbstractArray, A::AbstractArray) =
-    _mapreducedim!(f, to_op(op), R, A)
+    (_mapreducedim!(f, to_op(op), R, A); R)
 
 reducedim!{RT}(op, R::AbstractArray{RT}, A::AbstractArray) =
     mapreducedim!(IdFun(), op, R, A, zero(RT))