Skip to content

Commit d1cc073

Browse files
Oceania2018Oceania2018
committed
Merge branch 'master' into tensorflow2.x
2 parents 0c2ddfe + 85b64f3 commit d1cc073

49 files changed

Lines changed: 6152 additions & 6012 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* text=auto

‎TensorFlow.NET.sln‎

Lines changed: 133 additions & 133 deletions
Large diffs are not rendered by default.

‎docs/assets/Logo.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
TensorFlow.NET logo (c) 2019 by Meinrad Recheis.
2-
1+
TensorFlow.NET logo (c) 2019 by Meinrad Recheis.
2+
33
The logo is based on the original Tensorflow logo which is copyrighted by the respective creator.

‎src/TensorFlowNET.Core/APIs/tf.math.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -364,8 +364,8 @@ public Tensor div(Tensor x, Tensor y, string name = null)
364364
public Tensor divide<T>(Tensor x, T[] y, string name = null) where T : struct
365365
=> x / ops.convert_to_tensor(y, dtype: x.dtype.as_base_dtype(), name: "y");
366366

367-
public Tensor pow<T1, T2>(T1 x, T2 y)
368-
=> gen_math_ops.pow(x, y);
367+
public Tensor pow<T1, T2>(T1 x, T2 y, string name = "pow")
368+
=> gen_math_ops.pow(x, y, name: name);
369369

370370
/// <summary>
371371
/// Divides `x / y` elementwise, rounding toward the most negative integer.

‎src/TensorFlowNET.Core/Gradients/math_grad.cs‎

Lines changed: 49 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,13 @@ public static Tensor[] _AbsGrad(Operation op, Tensor[] grads)
3333
var x = op.inputs[0];
3434
var grad = grads[0];
3535

36-
return new Tensor[] { gen_ops.mul(grad, gen_math_ops.sign(x)) };
36+
return new Tensor[] { grad * math_ops.sign(x) };
3737
}
3838

39+
[RegisterGradient("AddV2")]
40+
public static Tensor[] _AddV2Grad(Operation op, Tensor[] grads)
41+
=> _AddGrad(op, grads);
42+
3943
[RegisterGradient("Add")]
4044
public static Tensor[] _AddGrad(Operation op, Tensor[] grads)
4145
{
@@ -107,7 +111,9 @@ public static Tensor[] _ExpGrad(Operation op, Tensor[] grads)
107111
var y = op.outputs[0]; // y = e^x
108112
return tf_with(ops.control_dependencies(new Operation[] { grad }), dp => {
109113
y = math_ops.conj(y);
110-
return new Tensor[] { math_ops.mul_no_nan(y, grad) };
114+
// forward_compatible(2019, 9, 14)
115+
// return new Tensor[] { math_ops.mul_no_nan(y, grad) };
116+
return new Tensor[] { grad * y };
111117
});
112118
}
113119

@@ -167,8 +173,7 @@ public static Tensor[] _MulGrad(Operation op, Tensor[] grads)
167173
new TF_DataType[] { tf.int32, tf.float32 }.Contains(grad.dtype))
168174
return new Tensor[] { gen_math_ops.mul(grad, y), gen_math_ops.mul(grad, x) };
169175

170-
var sx = array_ops.shape(x);
171-
var sy = array_ops.shape(y);
176+
var (sx, sy) = SmartBroadcastGradientArgs(x, y);
172177
var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
173178

174179
x = math_ops.conj(x);
@@ -355,8 +360,8 @@ private static Tensor[] _MaximumMinimumGrad(bool isMaximum, Operation op, Tensor
355360
: gen_math_ops.less_equal(x, y);
356361
var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
357362
var xgrad = array_ops.where(xmask, grad, zeros);
358-
var ygrad = array_ops.where(xmask, zeros, grad);
359363
var gx = array_ops.reshape(math_ops.reduce_sum(xgrad, rx), sx);
364+
var ygrad = array_ops.where(xmask, zeros, grad);
360365
var gy = array_ops.reshape(math_ops.reduce_sum(ygrad, ry), sy);
361366
return new Tensor[] { gx, gy };
362367
}
@@ -397,14 +402,13 @@ public static Tensor[] _SubGrad(Operation op, Tensor[] grads)
397402
_ShapesFullySpecifiedAndEqual(x, y, grad))
398403
return new Tensor[] { grad, -grad };
399404

400-
var sx = array_ops.shape(x);
401-
var sy = array_ops.shape(y);
405+
var (sx, sy) = SmartBroadcastGradientArgs(x, y);
402406
var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
403407

404-
var r1 = gen_array_ops.reshape(math_ops.reduce_sum(grad, rx), sx);
405-
var r2 = gen_array_ops.reshape(-math_ops.reduce_sum(grad, ry), sy);
408+
var gx = array_ops.reshape(math_ops.reduce_sum(grad, rx), sx);
409+
var gy = array_ops.reshape(math_ops.reduce_sum(-grad, ry), sy);
406410

407-
return new Tensor[] { r1, r2 };
411+
return new Tensor[] { gx, gy };
408412
}
409413

410414
public static bool _ShapesFullySpecifiedAndEqual(Tensor x, Tensor y, Tensor grad)
@@ -468,15 +472,16 @@ public static Tensor[] _RealDivGrad(Operation op, Tensor[] grads)
468472
x = math_ops.conj(x);
469473
y = math_ops.conj(y);
470474

471-
var realdiv1 = gen_math_ops.real_div(-x, y);
472-
var realdiv2 = gen_math_ops.real_div(realdiv1, y);
473-
var reduce_sum1 = math_ops.reduce_sum(grad * realdiv2, ry);
474-
var reshape1 = gen_array_ops.reshape(reduce_sum1, sy);
475-
var realdiv3 = gen_math_ops.real_div(grad, y);
476-
var reduce_sum2 = math_ops.reduce_sum(realdiv3, rx);
477-
var reshape2 = gen_array_ops.reshape(reduce_sum2, sx);
475+
var reshape1 = array_ops.reshape(
476+
math_ops.reduce_sum(
477+
math_ops.realdiv(grad, y), rx),
478+
sx);
479+
var reshape2 = array_ops.reshape(
480+
math_ops.reduce_sum(
481+
grad * math_ops.realdiv(math_ops.realdiv(-x, y), y), ry),
482+
sy);
478483

479-
return new Tensor[] { reshape2, reshape1 };
484+
return new Tensor[] { reshape1, reshape2 };
480485
}
481486

482487
[RegisterGradient("Sigmoid")]
@@ -602,14 +607,12 @@ public static Tensor[] _PowGrad(Operation op, Tensor[] grads)
602607
var y = op.inputs[1];
603608
var z = op.outputs[0];
604609

605-
var sx = array_ops.shape(x);
606-
var sy = array_ops.shape(y);
610+
var (sx, sy) = SmartBroadcastGradientArgs(x, y);
607611
var (rx, ry) = gen_array_ops.broadcast_gradient_args(sx, sy);
608612
x = math_ops.conj(x);
609613
y = math_ops.conj(y);
610614
z = math_ops.conj(z);
611-
var pow = gen_math_ops.pow(x, y - 1.0f);
612-
var mul = grad * y * pow;
615+
var mul = grad * y * math_ops.pow(x, y - 1.0f);
613616
var reduce_sum = math_ops.reduce_sum(mul, rx);
614617
var gx = gen_array_ops.reshape(reduce_sum, sx);
615618

@@ -630,5 +633,29 @@ public static Tensor[] _PowGrad(Operation op, Tensor[] grads)
630633

631634
return new Tensor[] { gx, gy };
632635
}
636+
637+
/// <summary>
638+
/// Optimized version of `broadcast_gradient_args` that caches results.
639+
/// </summary>
640+
/// <param name="x"></param>
641+
/// <param name="y"></param>
642+
/// <returns></returns>
643+
private static (Tensor, Tensor) SmartBroadcastGradientArgs(Tensor x, Tensor y)
644+
{
645+
Tensor sx, sy;
646+
if (x.TensorShape.is_fully_defined() &&
647+
y.TensorShape.is_fully_defined())
648+
{
649+
sx = array_ops.shape(x);
650+
sy = array_ops.shape(y);
651+
}
652+
else
653+
{
654+
sx = array_ops.shape_internal(x, optimize: false);
655+
sy = array_ops.shape_internal(y, optimize: false);
656+
}
657+
658+
return (sx, sy);
659+
}
633660
}
634661
}

‎src/TensorFlowNET.Core/Gradients/nn_grad.cs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,14 @@ public static Tensor[] _Conv2DGrad(Operation op, Tensor[] grads)
170170
public static Tensor[] _FusedBatchNormGrad(Operation op, Tensor[] grads)
171171
=> _BaseFusedBatchNormGrad(op, 0, grads);
172172

173+
[RegisterGradient("FusedBatchNormV2")]
174+
public static Tensor[] _FusedBatchNormV2Grad(Operation op, Tensor[] grads)
175+
=> _BaseFusedBatchNormGrad(op, 1, grads);
176+
177+
[RegisterGradient("FusedBatchNormV3")]
178+
public static Tensor[] _FusedBatchNormV3Grad(Operation op, Tensor[] grads)
179+
=> _BaseFusedBatchNormGrad(op, 2, grads);
180+
173181
/// <summary>
174182
/// Return the gradients for the 3 inputs of BatchNorm.
175183
/// </summary>
@@ -190,8 +198,10 @@ public static Tensor[] _BaseFusedBatchNormGrad(Operation op, int version, Tensor
190198
switch (version)
191199
{
192200
case 2:
193-
throw new NotImplementedException("");
201+
grad_fun = gen_nn_ops.fused_batch_norm_grad_v3;
202+
break;
194203
case 1:
204+
// grad_fun = gen_nn_ops.fused_batch_norm_grad_v2;
195205
throw new NotImplementedException("");
196206
default:
197207
grad_fun = gen_nn_ops.fused_batch_norm_grad;
@@ -225,8 +235,8 @@ public static Tensor[] _BaseFusedBatchNormGrad(Operation op, int version, Tensor
225235
YBackprop = grad_y,
226236
X = x,
227237
Scale = scale,
228-
ReserveSpace1 = op.outputs[3],
229-
ReserveSpace2 = op.outputs[4],
238+
ReserveSpace1 = pop_mean,
239+
ReserveSpace2 = pop_var,
230240
ReserveSpace3 = version == 2 ? op.outputs[5] : null,
231241
Epsilon = epsilon,
232242
DataFormat = data_format,

‎src/TensorFlowNET.Core/Graphs/Graph.Control.cs‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
/*****************************************************************************
2-
Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
3-
4-
Licensed under the Apache License, Version 2.0 (the "License");
5-
you may not use this file except in compliance with the License.
6-
You may obtain a copy of the License at
7-
8-
http://www.apache.org/licenses/LICENSE-2.0
9-
10-
Unless required by applicable law or agreed to in writing, software
11-
distributed under the License is distributed on an "AS IS" BASIS,
12-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13-
See the License for the specific language governing permissions and
14-
limitations under the License.
1+
/*****************************************************************************
2+
Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
1515
******************************************************************************/
1616

1717
using System.Collections.Generic;
@@ -77,8 +77,8 @@ public _ControlDependenciesController control_dependencies(ITensorOrOperation[]
7777
///
7878
/// Use with the `with` keyword to specify that all operations constructed
7979
/// within the context should have control dependencies on
80-
/// `control_inputs`.
81-
/// </summary>
80+
/// `control_inputs`.
81+
/// </summary>
8282
public _ControlDependenciesController control_dependencies(object[] control_inputs)
8383
{
8484
if (control_inputs == null)
@@ -92,20 +92,20 @@ public _ControlDependenciesController control_dependencies(object[] control_inpu
9292
// TODO: implement IndexedSlices
9393
//case IndexedSlices islice:
9494
// control_ops.Add(islice.op);
95-
// break;
95+
// break;
9696
case Tensor t:
9797
control_ops.Add(t.op);
9898
break;
9999
case Operation op:
100100
control_ops.Add(op);
101-
break;
101+
break;
102102
default:
103103
var t1 = _as_graph_element(c);
104104
if (t1 == null)
105105
throw new TypeError($"Control input must be Operation or Tensor:{c}");
106106
control_ops.Add(t1.op);
107-
break;
108-
}
107+
break;
108+
}
109109
}
110110
return new _ControlDependenciesController(this, control_ops);
111111
}
@@ -138,9 +138,9 @@ public void _pop_control_dependencies_controller(_ControlDependenciesController
138138
_control_dependencies_stack.RemoveAt(_control_dependencies_stack.Count-1);
139139
}
140140

141-
/// <summary>
142-
/// Record that the given op depends on all registered control dependencies.
143-
/// </summary>
141+
/// <summary>
142+
/// Record that the given op depends on all registered control dependencies.
143+
/// </summary>
144144
public void _record_op_seen_by_control_dependencies(Operation op)
145145
{
146146
foreach (var controller in _control_dependencies_stack)

‎src/TensorFlowNET.Core/Graphs/Graph.Operation.cs‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
1-
/*****************************************************************************
2-
Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
3-
4-
Licensed under the Apache License, Version 2.0 (the "License");
5-
you may not use this file except in compliance with the License.
6-
You may obtain a copy of the License at
7-
8-
http://www.apache.org/licenses/LICENSE-2.0
9-
10-
Unless required by applicable law or agreed to in writing, software
11-
distributed under the License is distributed on an "AS IS" BASIS,
12-
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13-
See the License for the specific language governing permissions and
14-
limitations under the License.
1+
/*****************************************************************************
2+
Copyright 2018 The TensorFlow.NET Authors. All Rights Reserved.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
1515
******************************************************************************/
1616

1717
using System;
@@ -38,8 +38,8 @@ public OpDef GetOpDef(string type)
3838
public OperationDescription NewOperation(string opType, string opName)
3939
{
4040
return c_api.TF_NewOperation(_handle, opType, opName);
41-
}
42-
41+
}
42+
4343
public Operation[] ReturnOperations(IntPtr results)
4444
{
4545
TF_Operation return_oper_handle = new TF_Operation();
@@ -89,14 +89,14 @@ public Operation OperationByName(string operName)
8989
public ITensorOrOperation[] get_operations()
9090
{
9191
return _nodes_by_name.Values.ToArray();
92-
}
93-
92+
}
93+
9494
/// <summary>
9595
/// Returns the `Operation` with the given `name`.
9696
///
97-
/// This method may be called concurrently from multiple threads.
98-
/// </summary>
99-
/// <param name="name">The name of the `Operation` to return.</param>
97+
/// This method may be called concurrently from multiple threads.
98+
/// </summary>
99+
/// <param name="name">The name of the `Operation` to return.</param>
100100
public Operation get_operation_by_name(string name)
101101
=> as_graph_element(name, allow_tensor: false, allow_operation: true) as Operation;
102102

@@ -109,8 +109,8 @@ public ITensorOrOperation _get_operation_by_tf_operation(IntPtr tf_oper)
109109
{
110110
var op_name = Marshal.PtrToStringAnsi(c_api.TF_OperationName(tf_oper));
111111
return _get_operation_by_name_unsafe(op_name);
112-
}
113-
112+
}
113+
114114
/// <summary>
115115
/// Creates an `Operation` in this graph from the supplied TF_Operation.
116116
///
@@ -125,7 +125,7 @@ public ITensorOrOperation _get_operation_by_tf_operation(IntPtr tf_oper)
125125
/// </summary>
126126
/// <param name="c_op">a wrapped TF_Operation</param>
127127
/// <param name="compute_device">(Optional.) If True, device functions will be executed
128-
/// to compute the device property of the Operation.</param>
128+
/// to compute the device property of the Operation.</param>
129129
/// <returns>An `Operation` object.</returns>
130130
public Operation _create_op_from_tf_operation(IntPtr c_op, bool compute_device = true)
131131
{

0 commit comments

Comments
 (0)