Add LSE & Softmax functions to Maths category

New source code files added for each function.

Meta data for each function was added to maths.ini

Author: delphidabbler

collection/663.dat

@@ -0,0 +1,15 @@
+function LSE(const A: array of Double): Double;
+var
+  MaxElem: Double;
+  Elem: Double;
+  Sum: Double;
+begin
+  if System.Length(A) = 0 then
+    raise SysUtils.EArgumentException.Create('Empty array');
+  // Using the centering "trick": see https://rpubs.com/FJRubio/LSE
+  MaxElem := MaxOfArray(A);
+  Sum := 0.0;
+  for Elem in A do
+    Sum := Sum + System.Exp(Elem - MaxElem);
+  Result := System.Ln(Sum) + MaxElem;
+end;

collection/664.dat

@@ -0,0 +1,10 @@
+function SoftMax(const A: array of Double): Types.TDoubleDynArray;
+var
+  LSEOfA: Double;
+  Idx: Integer;
+begin
+  LSEOfA := LSE(A); // raise EArgumentException if A is empty
+  System.SetLength(Result, System.Length(A));
+  for Idx := 0 to Pred(System.Length(A)) do
+    Result[Idx] := System.Exp(A[Idx] - LSEOfA);
+end;

collection/maths.ini

@@ -1841,3 +1841,33 @@ AdvancedTest.URL="https://github.com/delphidabbler/code-snippets/tree/master/tes
 Snip=662.dat
 DelphiXE=Y
 Delphi12A=Y
+
+[LSE]
+DisplayName=LSE
+DescEx="<p>Returns the logarithm of the sum of the exponentials of the given array of floating pointing point numbers.</p><p>An <var>EArgumentException</var> exception is raised if the array is empty.</p>"
+Extra="<p>The mathematical definition of <em>LSE</em> is <em>LSE(x1,...,xn) = log(exp(x1) + ... + exp(xn))</em>. The version of the algorithm used here uses an <a href="https://gregorygundersen.com/blog/2020/02/09/log-sum-exp/">algebraic trick</a> to minimise the risk of overflow.</p><p>For more information see <a href="https://nhigham.com/2021/01/05/what-is-the-log-sum-exp-function/">What Is the Log-Sum-Exp Function?</a> by Nick Higham.</p>"
+Kind=routine
+Units=SysUtils
+Depends=MaxOfArray_Double
+SeeAlso=Softmax
+TestInfo=advanced
+AdvancedTest.Level=unit-tests
+AdvancedTest.URL="https://github.com/delphidabbler/code-snippets/tree/master/tests/Cat-Maths"
+Snip=663.dat
+DelphiXE=Y
+Delphi12A=Y
+
+[Softmax]
+DisplayName=Softmax
+DescEx="<p>Applies the <a href="https://en.wikipedia.org/wiki/Softmax_function"><em>softmax</em></a> function to each element of floating point array <var>A</var> and normalizes them into a probability distribution proportional to the exponentials of the elements of <var>A</var>. The normalised values are returned as an array of the same size as <var>A</var>.</p><p>An <var>EArgumentException</var> exception is raised if <var>A</var> is empty.</p>"
+Extra="<p>The <em>softmax</em> function is often used in statistics and in machine learning. It is closely related to the <em>log-sum-exp</em> (aka <em>LSE</em>) function since, for vector <strong>x</strong>=(x1,...,xn), <em>softmax(<strong>x</strong>)=exp[<strong>x</strong>&#8722;LSE(<strong>x</strong>)]</em>.</p>"
+Kind=routine
+Units=SysUtils
+Depends=LSE
+SeeAlso=LSE,MaxOfArray_Double
+TestInfo=advanced
+AdvancedTest.Level=unit-tests
+AdvancedTest.URL="https://github.com/delphidabbler/code-snippets/tree/master/tests/Cat-Maths"
+Snip=664.dat
+DelphiXE=Y
+Delphi12A=Y