Implement encoding fallback for commit data

LibGit2Sharp.Tests/EncodingFixture.cs

@@ -0,0 +1,27 @@
+using LibGit2Sharp.Tests.TestHelpers;
+using System;
+using System.IO;
+using Xunit;
+using Xunit.Extensions;
+
+namespace LibGit2Sharp.Tests
+{
+    public class EncodingFixture : BaseFixture
+    {
+        [Theory]
+        //[InlineData("7bc349e5efdb52884cf47d860852e4912b12c244", "MãcRömãñ", 1000)]
+        [InlineData("782dced4c3930bc3c291b24c115ba9a231049d80", "Lãtïñ Òñê", 1250)]
+        //[InlineData("29b28e7d3e84aef886c0e00ff9de1a8cc1fda352", "Cõdêpâgê850", 850)]
+        public void CorruptAuthorNameEncoding(string commitId, string authorName, int commitEncoding)
+        {
+            string path = CloneEncodingTestRepo();
+
+            using (var repo = new Repository(path))
+            {
+                Commit commit = repo.Lookup<Commit>(commitId);
+                Assert.Equal(authorName, commit.Author.Name);
+            }
+        }
+
+    }
+}

LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj

@@ -116,6 +116,7 @@
     <Compile Include="TreeFixture.cs" />
     <Compile Include="TupleFixture.cs" />
     <Compile Include="UnstageFixture.cs" />
+    <Compile Include="EncodingFixture.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\LibGit2Sharp\LibGit2Sharp.csproj">

LibGit2Sharp.Tests/Resources/commitencodings.git/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

LibGit2Sharp.Tests/Resources/commitencodings.git/config

@@ -0,0 +1,9 @@
+[core]
+	repositoryformatversion = 0
+	filemode = false
+	bare = true
+	symlinks = false
+	ignorecase = true
+	hideDotFiles = dotGitOnly
+[remote "origin"]
+	url = D:/temp/.\\commitencodings

LibGit2Sharp.Tests/Resources/commitencodings.git/description

@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.

LibGit2Sharp.Tests/Resources/commitencodings.git/info/exclude

@@ -0,0 +1,6 @@
+# git ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~

LibGit2Sharp.Tests/Resources/commitencodings.git/objects/29/b28e7d3e84aef886c0e00ff9de1a8cc1fda352

@@ -0,0 +1,3 @@
+x
��M
+�0@a�9���i�� "x�4�F��#�wӃz�Gp�V/�e�u �w��� 8'3"t3E�F�2q�'FF�³_K��'���W��T[��
+K�ˡ�|Ҏ	�5�q@T�w���p����Q_~86

LibGit2Sharp.Tests/Resources/commitencodings.git/objects/59/7dd33db0c0f6300331a4f210423e3b8d983685

@@ -0,0 +1,2 @@
+x
5˱�0Qj$v�%4�ك�K�F��CCyһtjB�F��t)v�,,L}���a��\
+uǜb^@�k���6�(ڧ��?!�^�Cb#[_�&U

LibGit2Sharp.Tests/Resources/commitencodings.git/packed-refs

@@ -0,0 +1,3 @@
+# pack-refs with: peeled 
+29b28e7d3e84aef886c0e00ff9de1a8cc1fda352 refs/heads/dos850
+c62f9da614246e7f459a2a9d3dc2c325748435b3 refs/heads/master

LibGit2Sharp.Tests/Resources/commitencodings.git/refs/heads/master

@@ -0,0 +1 @@
+c62f9da614246e7f459a2a9d3dc2c325748435b3

LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs

@@ -23,6 +23,7 @@ static BaseFixture()
         public static string StandardTestRepoPath { get; private set; }
         public static string MergedTestRepoWorkingDirPath { get; private set; }
         public static string SubmoduleTestRepoWorkingDirPath { get; private set; }
+        public static string BareEncodingRepoPath { get; private set; }
         public static DirectoryInfo ResourcesDirectory { get; private set; }
 
         public static readonly Signature DummySignature = new Signature("Author N. Ame", "him@there.com", TruncateSubSeconds(DateTimeOffset.Now));
@@ -56,6 +57,7 @@ private static void SetUpTestEnvironment()
             StandardTestRepoPath = Path.Combine(StandardTestRepoWorkingDirPath, ".git");
             MergedTestRepoWorkingDirPath = Path.Combine(ResourcesDirectory.FullName, "mergedrepo_wd");
             SubmoduleTestRepoWorkingDirPath = Path.Combine(ResourcesDirectory.FullName, "submodule_wd");
+            BareEncodingRepoPath = Path.Combine(ResourcesDirectory.FullName, "commitencodings.git");
         }
 
         private static bool IsFileSystemCaseSensitiveInternal()
@@ -107,6 +109,11 @@ protected string CloneMergedTestRepo()
             return Clone(MergedTestRepoWorkingDirPath);
         }
 
+        protected string CloneEncodingTestRepo()
+        {
+            return Clone(BareEncodingRepoPath);
+        }
+
         public string CloneSubmoduleTestRepo()
         {
             var submoduleTarget = Path.Combine(ResourcesDirectory.FullName, "submodule_target_wd");

LibGit2Sharp/Core/Utf8Marshaler.cs

@@ -48,6 +48,18 @@ internal class Utf8Marshaler : ICustomMarshaler
     {
         private static readonly Utf8Marshaler staticInstance = new Utf8Marshaler();
 
+        private static readonly Encoding marshallerEncoding = UTF8EncodingWithFallBack;
+
+        public static Encoding UTF8EncodingWithFallBack
+        {
+            get
+            {
+                Encoding encoding = (Encoding)Encoding.UTF8.Clone();
+                encoding.DecoderFallback = new DecoderExceptionFallback();
+                return encoding;
+            }
+        }
+
         public static ICustomMarshaler GetInstance(String cookie)
         {
             return staticInstance;
@@ -141,7 +153,7 @@ public static unsafe String FromNative(IntPtr pNativeData)
                 return String.Empty;
             }
 
-            return new String((sbyte*)pNativeData.ToPointer(), 0, (int)(walk - start), Encoding.UTF8);
+            return FromNative(pNativeData, (int)(walk - start));
         }
 
         public static unsafe String FromNative(IntPtr pNativeData, int length)
@@ -156,7 +168,16 @@ public static unsafe String FromNative(IntPtr pNativeData, int length)
                 return String.Empty;
             }
 
-            return new String((sbyte*)pNativeData.ToPointer(), 0, length, Encoding.UTF8);
+            try
+            {
+                // Try UTF8 with fallback
+                return new String((sbyte*)pNativeData.ToPointer(), 0, length, marshallerEncoding);
+            }
+            catch (DecoderFallbackException)
+            {
+                // If this fails, try the platform default.
+                return new String((sbyte*)pNativeData.ToPointer(), 0, length, Encoding.Default);
+            }
         }
 
         public static String Utf8FromBuffer(byte[] buffer)
@@ -180,7 +201,16 @@ public static String Utf8FromBuffer(byte[] buffer)
                 return String.Empty;
             }
 
-            return Encoding.UTF8.GetString(buffer, 0, length);
+            try
+            {
+                // Try UTF8 with fallback
+                return marshallerEncoding.GetString(buffer, 0, length);
+            }
+            catch (DecoderFallbackException)
+            {
+                // If fails, use platform default
+                return Encoding.Default.GetString(buffer, 0, length);
+            }
         }
     }
 }