From 17b9147a91176365cf886ec0a702814123f3a325 Mon Sep 17 00:00:00 2001
From: Hallupa <328720+Hallupa@users.noreply.github.com>
Date: Tue, 20 Jul 2021 23:13:27 +0100
Subject: [PATCH 1/2] Add Softmax support and throw exception if activation
 isn't found

---
 .../Activations/Activations.Softmax.cs                | 11 +++++++++++
 src/TensorFlowNET.Keras/Layers/LayersApi.cs           |  6 ++++--
 2 files changed, 15 insertions(+), 2 deletions(-)
 create mode 100644 src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs

diff --git a/src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs b/src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs
new file mode 100644
index 000000000..02d86acea
--- /dev/null
+++ b/src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs
@@ -0,0 +1,11 @@
+using System;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Keras
+{
+    public partial class Activations
+    {
+        public Activation Softmax = (features, name) 
+            => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(features));
+    }
+}
diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs
index ed2f91d9b..6ffde8ef4 100644
--- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs
+++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs
@@ -1,4 +1,5 @@
-using Tensorflow.NumPy;
+using System;
+using Tensorflow.NumPy;
 using System.Collections.Generic;
 using Tensorflow.Keras.ArgsDefinition;
 using Tensorflow.Keras.Engine;
@@ -834,7 +835,8 @@ Activation GetActivationByName(string name)
                 "relu" => keras.activations.Relu,
                 "sigmoid" => keras.activations.Sigmoid,
                 "tanh" => keras.activations.Tanh,
-                _ => keras.activations.Linear
+                "softmax" => keras.activations.Softmax,
+                _ => throw new Exception($"Activation {name} not found")
             };
 
         /// <summary>

From ca6f9b477d4571d5a03e3d892dfc62f5b339735f Mon Sep 17 00:00:00 2001
From: Hallupa <328720+Hallupa@users.noreply.github.com>
Date: Mon, 26 Jul 2021 22:00:26 +0100
Subject: [PATCH 2/2] Add additional categorical cross entropy logic

---
 src/TensorFlowNET.Keras/BackendImpl.cs | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs
index e439eb9df..206c331fc 100644
--- a/src/TensorFlowNET.Keras/BackendImpl.cs
+++ b/src/TensorFlowNET.Keras/BackendImpl.cs
@@ -260,7 +260,19 @@ public Tensor categorical_crossentropy(Tensor target, Tensor output, bool from_l
             if (from_logits)
                 return tf.nn.softmax_cross_entropy_with_logits_v2(labels: target, logits: output, axis: axis);
 
-            throw new NotImplementedException("");
+            if (output.op != null && output.op.type == "Softmax")
+            {
+                if (output.op.inputs.Length != 1) throw new ApplicationException();
+                var o = output = output.op.inputs[0];
+                return tf.nn.softmax_cross_entropy_with_logits_v2(labels: target, logits: o, axis: axis);
+            }
+
+            // scale preds so that the class probas of each sample sum to 1
+            output = output / math_ops.reduce_sum(output, new Axis(axis), true);
+            // Compute cross entropy from probabilities.
+            var epsilon_ = constant_op.constant(epsilon(), output.dtype.as_base_dtype());
+            output = clip_ops.clip_by_value(output, epsilon_, 1.0 - epsilon_);
+            return -math_ops.reduce_sum(target * math_ops.log(output), new Axis(axis));
         }
 
         /// <summary>