Do rebase and upstreaming

Author: opencici2006

chainer/function.py

@@ -419,7 +419,48 @@ def backward_cpu_cosim(self, inputs, grad_outputs):
         chainer.enable_cosim()
         return output_cosim
 
-    def cpu_cosim_verify_result(self, mkl_result, numpy_result):
+    def cpu_cosim_dump_forward_inputs(self, in_data):
+        """dump all forward inputs into file
+
+        Aims to dump the inputs into a file in order to  reproduce offline,
+        if encountering a mismatch in cosim results of forward prop.
+        To support this feature, implement it for each function.
+
+        Args:
+            inputs: Tuple of input arrays.
+
+        """
+        pass
+
+    def cpu_cosim_dump_backward_inputs(self, in_data, out_grad):
+        """dump all backward inputs into file
+
+        Aims to dump the inputs into a file in order to  reproduce offline,
+        if encountering a mismatch in cosim results of backward prop.
+        To support this feature, implement it for each function.
+
+        Args:
+            inputs: Tuple of input arrays.
+            out_grad: Tuple of output gradient arrays.
+
+        """
+        pass
+
+    def cpu_cosim_dump_inputs(self, inputs, out_grad=None):
+        """dump all inputs into a file in order to reprocude errors offline.
+        """
+        inputs = [x if isinstance(x, variable.Variable)
+                  else variable.Variable(x)
+                  for x in inputs]
+
+        in_data = tuple([x.data for x in inputs])
+
+        if out_grad is None:
+            self.cpu_cosim_dump_forward_inputs(in_data)
+        else:
+            self.cpu_cosim_dump_backward_inputs(in_data, out_grad)
+
+    def cpu_cosim_verify_result(self, mkl_result, numpy_result, inputs, out_grad=None):
         """cosim verify result between MKLDNN and numpy
         """
         if not chainer.is_cosim():
@@ -449,7 +490,13 @@ def cpu_cosim_verify_result(self, mkl_result, numpy_result):
                 numpy_y_nd = np.array(numpy_y)
             i = i + 1
             if isinstance(mkl_x_nd, np.ndarray):
-                testing.assert_allclose(mkl_x_nd, numpy_y_nd, **check_options)
+                try:
+                    testing.assert_allclose(mkl_x_nd, numpy_y_nd, **check_options)
+                except AssertionError:
+                    self.cpu_cosim_dump_inputs(inputs, out_grad)
+                    raise
+                except:
+                    raise
             else:
                 raise KeyError('cosim, unexpected!')
 

chainer/functions/activation/relu.py

@@ -89,13 +89,14 @@ def relu(x):
         (3, 2)
 
     """
-    if mkld.check_with_mkld((x, ), (2, 4)):
+    if not isinstance(x.data, cuda.ndarray) and \
+       mkld.check_with_mkld((x, ), (2, 4)):
         func = ReLUMKLDNN()
         ret = func(x)
         if chainer.is_cosim():
             func.cosim_func = ReLU()
             numpy_result = func.cosim_func(x)
-            func.cpu_cosim_verify_result(ret, numpy_result)
+            func.cpu_cosim_verify_result(ret, numpy_result, (x, ))
         return ret
     else:
         if chainer.mkld.available and \

chainer/functions/array/concat.py

@@ -93,13 +93,14 @@ def concat(xs, axis=1):
 
     """
     x = xs[0]
-    if mkld.check_with_mkld((x, ), (4, )):
+    if not isinstance(x.data, cuda.ndarray) and \
+       mkld.check_with_mkld((x, ), (4, )):
         func = ConcatMKLDNN(axis=axis)
         ret = func(*xs)
         if chainer.is_cosim():
             func.cosim_func = Concat(axis=axis)
             numpy_result = func.cosim_func(*xs)
-            func.cpu_cosim_verify_result(ret, numpy_result)
+            func.cpu_cosim_verify_result(ret, numpy_result, xs)
         return ret
     else:
         return Concat(axis=axis)(*xs)

chainer/functions/connection/convolution_2d.py

@@ -330,10 +330,11 @@ def convolution_2d(x, W, b=None, stride=1, pad=0,
             if b is None:
                 ret = func(x, W)
                 numpy_result = func.cosim_func(x, W)
+                func.cpu_cosim_verify_result(ret, numpy_result, (x, W))
             else:
                 ret = func(x, W, b)
                 numpy_result = func.cosim_func(x, W, b)
-            func.cpu_cosim_verify_result(ret, numpy_result)
+                func.cpu_cosim_verify_result(ret, numpy_result, (x, W, b))
             return ret
     else:
         func = Convolution2DFunction(

chainer/functions/connection/deconvolution_2d.py

@@ -2,6 +2,7 @@
 
 import chainer
 from chainer import cuda
+from chainer import mkld
 from chainer import function
 from chainer.functions.connection import convolution_2d
 from chainer.utils import conv
@@ -18,6 +19,8 @@
         _bwd_data_pref = \
             libcudnn.CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT
 
+if mkld.available:
+    Deconvolution2DFunctionMKLDNN = mkld.deconvolution_2d.Deconvolution2DFunctionMKLDNN
 
 _check_cudnn_acceptable_type = convolution_2d._check_cudnn_acceptable_type
 
@@ -321,7 +324,24 @@ def deconvolution_2d(x, W, b=None, stride=1, pad=0,
        w_O &= s_X (w - 1) + k_W - 2p_W.
 
     """
-    func = Deconvolution2DFunction(stride, pad, outsize, deterministic)
+    # XXX: Switch the route
+    if not isinstance(x.data, cuda.ndarray) and \
+       mkld.check_with_mkld((x, W), ()):
+        func = Deconvolution2DFunctionMKLDNN(stride, pad, outsize, deterministic)
+        if chainer.is_cosim():
+            func.cosim_func = Deconvolution2DFunction(stride, pad, outsize, deterministic)
+            if b is None:
+                ret = func(x, W)
+                numpy_result = func.cosim_func(x, W)
+                func.cpu_cosim_verify_result(ret, numpy_result, (x, W))
+            else:
+                ret = func(x, W, b)
+                numpy_result = func.cosim_func(x, W, b)
+                func.cpu_cosim_verify_result(ret, numpy_result, (x, W, b))
+            return ret
+    else:
+        func = Deconvolution2DFunction(stride, pad, outsize, deterministic)
+
     if b is None:
         return func(x, W)
     else:

chainer/functions/connection/linear.py

@@ -150,7 +150,8 @@ def linear(x, W, b=None):
     """
     # XXX: switch the route, work on the critera
 
-    if not isinstance(x.data, cuda.ndarray) and \
+    if not isinstance(x, cuda.ndarray) and \
+       not isinstance(x.data, cuda.ndarray) and \
        mkld.check_with_mkld((x, W), (2, 4)):
         if b is None:
             return LinearFunctionMKLDNN()(x, W)

chainer/functions/normalization/batch_normalization.py

@@ -432,7 +432,7 @@ def fixed_batch_normalization(x, gamma, beta, mean, var, eps=2e-5):
             if chainer.is_cosim():
                 func.cosim_func = BatchNormalizationFunction(eps, None, None, 0.0)
                 numpy_result = func.cosim_func(x, gamma, beta, mean, var)
-                func.cpu_cosim_verify_result(ret, numpy_result)
+                func.cpu_cosim_verify_result(ret, numpy_result, (x, gamma, beta, mean, var))
             return ret
         else:
             return BatchNormalizationFunction(eps, None, None, 0.0)(

chainer/functions/normalization/local_response_normalization.py

@@ -140,7 +140,7 @@ def local_response_normalization(x, n=5, k=2, alpha=1e-4, beta=.75):
         if chainer.is_cosim():
             func.cosim_func = LocalResponseNormalization(n, k, alpha, beta)
             numpy_result = func.cosim_func(x)
-            func.cpu_cosim_verify_result(ret, numpy_result)
+            func.cpu_cosim_verify_result(ret, numpy_result, (x, ))
         return ret
     else:
         return LocalResponseNormalization(n, k, alpha, beta)(x)

chainer/functions/pooling/average_pooling_2d.py

@@ -145,7 +145,7 @@ def average_pooling_2d(x, ksize, stride=None, pad=0):
         if chainer.is_cosim():
             func.cosim_func = AveragePooling2D(ksize, stride, pad, False)
             numpy_result = func.cosim_func(x)
-            func.cpu_cosim_verify_result(ret, numpy_result)
+            func.cpu_cosim_verify_result(ret, numpy_result, (x, ))
         return ret
     else:
         return AveragePooling2D(ksize, stride, pad, False)(x)

chainer/functions/pooling/max_pooling_2d.py

@@ -179,7 +179,7 @@ def max_pooling_2d(x, ksize, stride=None, pad=0, cover_all=True):
         if chainer.is_cosim():
             func.cosim_func = MaxPooling2D(ksize, stride, pad, cover_all)
             numpy_result = func.cosim_func(x)
-            func.cpu_cosim_verify_result(ret, numpy_result)
+            func.cpu_cosim_verify_result(ret, numpy_result, (x, ))
         return ret
     else:
         return MaxPooling2D(ksize, stride, pad, cover_all)(x)