Merge pull request #1071 from AsakusaRinne/add_linux_gpu_redist

build: add scripts to split large native library file.

Author: AsakusaRinne

.github/workflows/build_and_test.yml

@@ -28,9 +28,9 @@ jobs:
     - name: Test CPU version
       run: dotnet test --no-build --verbosity normal
     - name: uninstall redist cpu for unit tests
-      run: dotnet remove helpers/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist
+      run: dotnet remove tools/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist
     - name: install redist gpu for unit tests
-      run: dotnet add helpers/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist-Windows-GPU
+      run: dotnet add tools/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist-Windows-GPU
     - name: Restore dependencies
       run: dotnet restore
     - name: Build GPU version
@@ -52,12 +52,12 @@ jobs:
       run: dotnet restore
     - name: Build CPU version
       run: dotnet build --no-restore
-    # - name: Test CPU version
-    #   run: dotnet test --no-build --verbosity normal
+    - name: Test CPU version
+      run: dotnet test --no-build --verbosity normal
     - name: uninstall redist cpu for unit tests
-      run: dotnet remove helpers/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist
+      run: dotnet remove tools/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist
     - name: install redist gpu for unit tests
-      run: dotnet add helpers/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist-Linux-GPU
+      run: dotnet add tools/Tensorflow.UnitTest.RedistHolder package SciSharp.TensorFlow.Redist-Linux-GPU
     - name: Restore dependencies
       run: dotnet restore
     - name: Build GPU version

TensorFlow.NET.sln

@@ -86,6 +86,18 @@ public tensorflow()
 
             OpDefLib = new OpDefLibrary();
             InitGradientEnvironment();
+
+            try
+            {
+                var handle = c_api.TF_Version();
+            }
+            catch (DllNotFoundException)
+            {
+                throw new RuntimeError("Tensorflow.NET cannot find a backend. Please install one of the following packages for your program: " +
+                    "SciSharp.TensorFlow.Redist, SciSharp.TensorFlow.Redist-Linux-GPU, SciSharp.TensorFlow.Redist-Windows-GPU. For more details, " +
+                    "please visit https://github.com/SciSharp/TensorFlow.NET. If it still not work after installing the backend, please submit an " +
+                    "issue to https://github.com/SciSharp/TensorFlow.NET/issues");
+            }
         }
 
         public string VERSION => c_api.StringPiece(c_api.TF_Version());

src/TensorFlowNET.Core/tensorflow.cs

@@ -34,8 +34,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\helpers\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
+    <ProjectReference Include="..\..\tools\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
     <ProjectReference Include="..\TensorFlowNET.Keras.UnitTest\Tensorflow.Keras.UnitTest.csproj" />
   </ItemGroup>
 

test/TensorFlowNET.Graph.UnitTest/TensorFlowNET.Graph.UnitTest.csproj

@@ -23,8 +23,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\helpers\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Keras\Tensorflow.Keras.csproj" />
+    <ProjectReference Include="..\..\tools\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
   </ItemGroup>
 
   <ItemGroup>

test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj

@@ -13,13 +13,15 @@ namespace Tensorflow.Native.UnitTest
     public class TfLiteTest
     {
         [TestMethod]
+        [Ignore]
         public void TfLiteVersion()
         {
             var ver = c_api_lite.StringPiece(c_api_lite.TfLiteVersion());
             Assert.IsNotNull(ver);
         }
 
         [TestMethod]
+        [Ignore]
         public unsafe void SmokeTest()
         {
             var model = c_api_lite.TfLiteModelCreateFromFile("Lite/testdata/add.bin");
@@ -85,6 +87,7 @@ public unsafe void SmokeTest()
         }
 
         [TestMethod]
+        [Ignore]
         public unsafe void QuantizationParamsTest()
         {
             var model = c_api_lite.TfLiteModelCreateFromFile("Lite/testdata/add_quantized.bin");

test/TensorFlowNET.Native.UnitTest/Lite/TfLiteTest.cs

@@ -54,8 +54,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\helpers\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
+    <ProjectReference Include="..\..\tools\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
   </ItemGroup>
 
 </Project>

test/TensorFlowNET.Native.UnitTest/Tensorflow.Native.UnitTest.csproj

@@ -48,9 +48,9 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\helpers\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
     <ProjectReference Include="..\..\src\TensorFlowNET.Text\Tensorflow.Text.csproj" />
+    <ProjectReference Include="..\..\tools\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
   </ItemGroup>
 
   <ItemGroup>

test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj

@@ -16,8 +16,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\helpers\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
     <ProjectReference Include="..\..\src\TensorflowNET.Hub\Tensorflow.Hub.csproj" />
+    <ProjectReference Include="..\..\tools\Tensorflow.UnitTest.RedistHolder\Tensorflow.UnitTest.RedistHolder.csproj" />
   </ItemGroup>
 
 </Project>

test/TensorflowNET.Hub.Unittest/Tensorflow.Hub.Unittest.csproj

@@ -41,7 +41,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\TensorFlowNET.Keras\Tensorflow.Keras.csproj" />
+    <ProjectReference Include="..\..\src\TensorFlowNET.Keras\Tensorflow.Keras.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/TensorFlowNet.Benchmarks/Crash/RepeatDataSetCrash.cs renamed to tools/TensorFlowNET.Benchmarks/Crash/RepeatDataSetCrash.cs

@@ -24,8 +24,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\TensorFlowNET.Recommenders\Tensorflow.Recommenders.csproj" />
-    <ProjectReference Include="..\TensorFlowNET.Text\Tensorflow.Text.csproj" />
+    <ProjectReference Include="..\..\src\TensorFlowNET.Recommenders\Tensorflow.Recommenders.csproj" />
+    <ProjectReference Include="..\..\src\TensorFlowNET.Text\Tensorflow.Text.csproj" />
   </ItemGroup>
 
 </Project>

src/TensorFlowNet.Benchmarks/Leak/GpuLeakByCNN.cs renamed to tools/TensorFlowNET.Benchmarks/Leak/GpuLeakByCNN.cs

@@ -13,7 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\src\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
+    <ProjectReference Include="..\..\src\TensorFlowNET.Core\Tensorflow.Binding.csproj" />
   </ItemGroup>
 
 </Project>

src/TensorFlowNet.Benchmarks/Leak/SavedModelCleanup.cs renamed to tools/TensorFlowNET.Benchmarks/Leak/SavedModelCleanup.cs

@@ -0,0 +1,212 @@
+
+// =================================================================== //
+// This is a tool to split the native .so file of linux gpu library    //
+// =================================================================== //
+
+using System.Security.Cryptography;
+
+string filename = "libtensorflow.so";
+int count = 5;
+SplitFile(filename, count);
+
+static void SplitFile(string filename, int count)
+{
+    // 打开读取二进制文件的文件流
+    using (FileStream input = new FileStream(filename, FileMode.Open, FileAccess.Read))
+    {
+        long filesize = new FileInfo(filename).Length; // 获取文件大小
+        long fragmentSize = (long)(filesize / count + 1); // 计算每个分片的大小
+
+        byte[] buffer = new byte[fragmentSize]; // 设置缓冲区大小
+        int bytesRead; // 存储读取长度
+        int fragmentIndex = 1; // 分片计数器
+
+        // 使用循环遍历分片并写入相应的文件
+        while ((bytesRead = input.Read(buffer, 0, buffer.Length)) > 0)
+        {
+            string outputFileName = $"{filename}.fragment{fragmentIndex++}";
+            using (FileStream output = new FileStream(outputFileName, FileMode.Create, FileAccess.Write))
+            {
+                output.Write(buffer, 0, bytesRead);
+            }
+        }
+
+        // 计算整个文件的 SHA-256 哈希值并写入 .sha 文件
+        using (SHA256 sha256Hash = SHA256.Create())
+        {
+            input.Seek(0, SeekOrigin.Begin);
+            byte[] hashValue = sha256Hash.ComputeHash(input);
+
+            string shaFileName = $"{filename}.sha";
+            using (StreamWriter writer = new StreamWriter(shaFileName, false))
+            {
+                writer.Write(BitConverter.ToString(hashValue).Replace("-", ""));
+            }
+        }
+    }
+}
+
+// Resume the file from fregments. Thanks for the code in TorchSharp!
+static void Restitch(string RestitcherPackage)
+{
+    // !!!!!!!------------------------------NOTE------------------------------------!!!!!!
+    // !!!!!!! This code is manually copied into pkg\common\RestitchPackage.targets !!!!!!
+    // !!!!!!!------------------------------NOTE------------------------------------!!!!!!
+    //
+    // vvvvvvvvvvvvvvvvvvvvvvvvvvvvv START HERE vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
+    try
+    {
+        if (Directory.Exists(RestitcherPackage))
+        {
+            using (var writer = File.CreateText("obj/tensorflow_redist_build_log.txt"))
+            {
+                foreach (var p in Directory.EnumerateFiles(RestitcherPackage, "*", SearchOption.AllDirectories))
+                {
+
+                    var primaryFile = Path.GetFullPath(p);
+                    writer.WriteLine("Found primary file at {0}", primaryFile);
+
+                    // See if there are fragments in the parallel nuget packages. If the primary is 
+                    //        some-package-primary\runtimes\....\a.so 
+                    //        some-package-primary\runtimes\....\a.so.sha
+                    // then the expected fragments are
+                    //        some-package-fragment1\fragments\....\a.so 
+                    //        some-package-fragment2\fragments\....\a.so 
+                    //        some-package-fragment3\fragments\....\a.so 
+                    //        some-package-fragment4\fragments\....\a.so 
+                    //        some-package-fragment5\fragments\....\a.so 
+                    //        some-package-fragment6\fragments\....\a.so 
+                    //        some-package-fragment7\fragments\....\a.so 
+                    //        some-package-fragment8\fragments\....\a.so 
+                    //        some-package-fragment9\fragments\....\a.so 
+                    //        some-package-fragment10\fragments\....\a.so 
+                    var shaFile = primaryFile + ".sha";
+                    var fragmentFile1 = primaryFile.Replace("-primary", "-fragment1").Replace("runtimes", "fragments") + ".fragment1";
+                    var fragmentFile2 = primaryFile.Replace("-primary", "-fragment2").Replace("runtimes", "fragments") + ".fragment2";
+                    var fragmentFile3 = primaryFile.Replace("-primary", "-fragment3").Replace("runtimes", "fragments") + ".fragment3";
+                    var fragmentFile4 = primaryFile.Replace("-primary", "-fragment4").Replace("runtimes", "fragments") + ".fragment4";
+                    var fragmentFile5 = primaryFile.Replace("-primary", "-fragment5").Replace("runtimes", "fragments") + ".fragment5";
+
+
+                    if (File.Exists(fragmentFile1)) writer.WriteLine("Found fragment file at {0}", fragmentFile1);
+                    if (File.Exists(fragmentFile2)) writer.WriteLine("Found fragment file at {0}", fragmentFile2);
+                    if (File.Exists(fragmentFile3)) writer.WriteLine("Found fragment file at {0}", fragmentFile3);
+                    if (File.Exists(fragmentFile4)) writer.WriteLine("Found fragment file at {0}", fragmentFile4);
+                    if (File.Exists(fragmentFile5)) writer.WriteLine("Found fragment file at {0}", fragmentFile5);
+
+                    if (File.Exists(fragmentFile1))
+                    {
+                        var tmpFile = Path.GetTempFileName();
+
+                        {
+                            writer.WriteLine("Writing restored primary file at {0}", tmpFile);
+                            using (var os = File.OpenWrite(tmpFile))
+                            {
+
+                                //writer.WriteLine("Writing bytes from {0} to {1}", primaryFile, tmpFile);
+                                //var primaryBytes = File.ReadAllBytes(primaryFile);
+
+                                //os.Write(primaryBytes, 0, primaryBytes.Length);
+                                if (File.Exists(fragmentFile1))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile1, tmpFile);
+                                    var fragmentBytes1 = File.ReadAllBytes(fragmentFile1);
+                                    os.Write(fragmentBytes1, 0, fragmentBytes1.Length);
+                                }
+                                if (File.Exists(fragmentFile2))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile2, tmpFile);
+                                    var fragmentBytes2 = File.ReadAllBytes(fragmentFile2);
+                                    os.Write(fragmentBytes2, 0, fragmentBytes2.Length);
+                                }
+                                if (File.Exists(fragmentFile3))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile3, tmpFile);
+                                    var fragmentBytes3 = File.ReadAllBytes(fragmentFile3);
+                                    os.Write(fragmentBytes3, 0, fragmentBytes3.Length);
+                                }
+                                if (File.Exists(fragmentFile4))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile4, tmpFile);
+                                    var fragmentBytes4 = File.ReadAllBytes(fragmentFile4);
+                                    os.Write(fragmentBytes4, 0, fragmentBytes4.Length);
+                                }
+                                if (File.Exists(fragmentFile5))
+                                {
+                                    writer.WriteLine("Writing fragment bytes from {0} to {1}", fragmentFile5, tmpFile);
+                                    var fragmentBytes5 = File.ReadAllBytes(fragmentFile5);
+                                    os.Write(fragmentBytes5, 0, fragmentBytes5.Length);
+                                }
+                            }
+                        }
+
+                        var shaExpected = File.Exists(shaFile) ? File.ReadAllText(shaFile).ToUpper() : "";
+                        writer.WriteLine($"real sha: {shaExpected}");
+
+                        using (var sha256Hash = System.Security.Cryptography.SHA256.Create())
+                        {
+                            using (var os2 = File.OpenRead(tmpFile))
+                            {
+
+                                byte[] bytes = sha256Hash.ComputeHash(os2);
+                                var builder = new System.Text.StringBuilder();
+                                for (int i = 0; i < bytes.Length; i++)
+                                {
+                                    builder.Append(bytes[i].ToString("x2"));
+                                }
+                                var shaReconstituted = builder.ToString().ToUpper();
+                                if (shaExpected != shaReconstituted)
+                                {
+                                    string msg =
+                                            $"Error downloading and reviving packages. Reconsituted file contents have incorrect SHA\n\tExpected SHA: ${shaExpected}\n\tActual SHA: ${shaReconstituted}\n\tFile was reconstituted from:"
+                                          + $"\n\t{primaryFile} (length ${new FileInfo(primaryFile).Length})"
+                                          + (File.Exists(fragmentFile1) ? $"\n\t{fragmentFile1} (length ${new FileInfo(fragmentFile1).Length})" : "")
+                                          + (File.Exists(fragmentFile2) ? $"\n\t{fragmentFile2} (length ${new FileInfo(fragmentFile2).Length})" : "")
+                                          + (File.Exists(fragmentFile3) ? $"\n\t{fragmentFile3} (length ${new FileInfo(fragmentFile3).Length})" : "")
+                                          + (File.Exists(fragmentFile4) ? $"\n\t{fragmentFile4} (length ${new FileInfo(fragmentFile4).Length})" : "")
+                                          + (File.Exists(fragmentFile5) ? $"\n\t{fragmentFile5} (length ${new FileInfo(fragmentFile5).Length})" : "");
+                                    writer.WriteLine(msg);
+                                    throw new Exception(msg);
+                                }
+                            }
+
+                        }
+
+                        writer.WriteLine("Deleting {0}", primaryFile);
+                        File.Delete(primaryFile);
+                        if (File.Exists(primaryFile))
+                            throw new Exception("wtf?");
+
+                        writer.WriteLine("Moving {0} --> {1}", tmpFile, primaryFile);
+                        File.Move(tmpFile, primaryFile);
+
+                        writer.WriteLine("Deleting {0}", fragmentFile1);
+                        File.Delete(fragmentFile1);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile2);
+                        if (File.Exists(fragmentFile2))
+                            File.Delete(fragmentFile2);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile3);
+                        if (File.Exists(fragmentFile3))
+                            File.Delete(fragmentFile3);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile4);
+                        if (File.Exists(fragmentFile4))
+                            File.Delete(fragmentFile4);  // free up space and prevent us doing this again 
+
+                        writer.WriteLine("Deleting {0}", fragmentFile5);
+                        if (File.Exists(fragmentFile5))
+                            File.Delete(fragmentFile5);  // free up space and prevent us doing this again 
+                    }
+                }
+            }
+        }
+    }
+    catch (Exception ex)
+    {
+        Console.Error.WriteLine(ex.ToString());
+        Console.Error.WriteLine(ex.StackTrace);
+    }
+    // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ END HERE^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+}

src/TensorFlowNet.Benchmarks/Leak/TestModel/saved_model/saved_model.pb renamed to tools/TensorFlowNET.Benchmarks/Leak/TestModel/saved_model/saved_model.pb

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>