diff --git a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj index 0be3440c6..4ae6101dc 100644 --- a/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj +++ b/LibGit2Sharp.Tests/LibGit2Sharp.Tests.csproj @@ -64,7 +64,7 @@ - + diff --git a/LibGit2Sharp.Tests/PackBuilderFixture.cs b/LibGit2Sharp.Tests/PackDefinitionFixture.cs similarity index 54% rename from LibGit2Sharp.Tests/PackBuilderFixture.cs rename to LibGit2Sharp.Tests/PackDefinitionFixture.cs index 44e358158..db9ecc265 100644 --- a/LibGit2Sharp.Tests/PackBuilderFixture.cs +++ b/LibGit2Sharp.Tests/PackDefinitionFixture.cs @@ -1,12 +1,13 @@ using System; using System.IO; using System.Linq; +using System.Text; using LibGit2Sharp.Tests.TestHelpers; using Xunit; namespace LibGit2Sharp.Tests { - public class PackBuilderFixture : BaseFixture + public class PackDefinitionFixture : BaseFixture { [Fact] public void TestDefaultPackDelegate() @@ -17,16 +18,16 @@ public void TestDefaultPackDelegate() [Fact] public void TestCommitsPerBranchPackDelegate() { - TestIfSameRepoAfterPacking(AddingObjectIdsTestDelegate); + TestIfSameRepoAfterPacking(AddingObjectIdsPackBuilder); } [Fact] public void TestCommitsPerBranchIdsPackDelegate() { - TestIfSameRepoAfterPacking(AddingObjectsTestDelegate); + TestIfSameRepoAfterPacking(AddingObjectsPackBuilder); } - internal void TestIfSameRepoAfterPacking(Action packDelegate) + internal void TestIfSameRepoAfterPacking(Action packBuilder) { // read a repo // pack with the provided action @@ -34,22 +35,25 @@ internal void TestIfSameRepoAfterPacking(Action packDe // read new repo // compare - string orgRepoPath = SandboxPackBuilderTestRepo(); - string mrrRepoPath = SandboxPackBuilderTestRepo(); - string mrrRepoPackDirPath = Path.Combine(mrrRepoPath + "/.git/objects"); + string orgRepoPath = SandboxPackDefinitionTestRepo(); + string mrrRepoPath = SandboxPackDefinitionTestRepo(); + string mrrRepoPackDirPath = Path.Combine(mrrRepoPath + ".git", "objects"); DirectoryHelper.DeleteDirectory(mrrRepoPackDirPath); - Directory.CreateDirectory(mrrRepoPackDirPath + "/pack"); - - PackBuilderOptions packBuilderOptions = new PackBuilderOptions(mrrRepoPackDirPath + "/pack"); + string packDir = Path.Combine(mrrRepoPackDirPath, "pack"); + Directory.CreateDirectory(packDir); using (Repository orgRepo = new Repository(orgRepoPath)) { - PackBuilderResults results; - if (packDelegate != null) - results = orgRepo.ObjectDatabase.Pack(packBuilderOptions, b => packDelegate(orgRepo, b)); - else - results = orgRepo.ObjectDatabase.Pack(packBuilderOptions); + var packOptions = new PackOptions(); + + PackResults results; + if (packBuilder != null) + { + packOptions.PackBuilder = b => packBuilder(orgRepo, b); + } + + results = orgRepo.ObjectDatabase.Pack(packDir, packOptions); // written objects count is the same as in objects database Assert.Equal(orgRepo.ObjectDatabase.Count(), results.WrittenObjectsCount); @@ -68,7 +72,7 @@ internal void TestIfSameRepoAfterPacking(Action packDe } } - internal void AddingObjectIdsTestDelegate(IRepository repo, PackBuilder builder) + internal void AddingObjectIdsPackBuilder(IRepository repo, PackDefinition builder) { foreach (Branch branch in repo.Branches) { @@ -84,7 +88,7 @@ internal void AddingObjectIdsTestDelegate(IRepository repo, PackBuilder builder) } } - internal void AddingObjectsTestDelegate(IRepository repo, PackBuilder builder) + internal void AddingObjectsPackBuilder(IRepository repo, PackDefinition builder) { foreach (Branch branch in repo.Branches) { @@ -103,50 +107,35 @@ internal void AddingObjectsTestDelegate(IRepository repo, PackBuilder builder) [Fact] public void ExceptionIfPathDoesNotExist() { - PackBuilderOptions pbo; - - Assert.Throws(() => + using (Repository repo = new Repository(SandboxPackDefinitionTestRepo())) { - pbo = new PackBuilderOptions("aaa"); - }); + Assert.Throws(() => + { + repo.ObjectDatabase.Pack("aaa", new PackOptions()); + }); + } } [Fact] public void ExceptionIfPathEqualsNull() { - PackBuilderOptions pbo; - - Assert.Throws(() => - { - pbo = new PackBuilderOptions(null); - }); - } - - [Fact] - public void ExceptionIfOptionsEqualsNull() - { - string orgRepoPath = SandboxPackBuilderTestRepo(); - - using (Repository orgRepo = new Repository(orgRepoPath)) + using (Repository repo = new Repository(SandboxPackDefinitionTestRepo())) { Assert.Throws(() => { - orgRepo.ObjectDatabase.Pack(null); + repo.ObjectDatabase.Pack(default(string), new PackOptions()); }); } } [Fact] - public void ExceptionIfBuildDelegateEqualsNull() + public void ExceptionIfStreamIsNull() { - string orgRepoPath = SandboxPackBuilderTestRepo(); - PackBuilderOptions packBuilderOptions = new PackBuilderOptions(orgRepoPath); - - using (Repository orgRepo = new Repository(orgRepoPath)) + using (Repository repo = new Repository(SandboxPackDefinitionTestRepo())) { Assert.Throws(() => { - orgRepo.ObjectDatabase.Pack(packBuilderOptions, null); + repo.ObjectDatabase.Pack(default(Stream), new PackOptions()); }); } } @@ -154,48 +143,69 @@ public void ExceptionIfBuildDelegateEqualsNull() [Fact] public void ExceptionIfNegativeNumberOfThreads() { - string orgRepoPath = SandboxPackBuilderTestRepo(); - PackBuilderOptions packBuilderOptions = new PackBuilderOptions(orgRepoPath); - Assert.Throws(() => { - packBuilderOptions.MaximumNumberOfThreads = -1; + (new PackOptions()).MaximumNumberOfThreads = -1; }); } [Fact] public void ExceptionIfAddNullObjectID() { - string orgRepoPath = SandboxPackBuilderTestRepo(); - PackBuilderOptions packBuilderOptions = new PackBuilderOptions(orgRepoPath); + string orgRepoPath = SandboxPackDefinitionTestRepo(); + var packOptions = new PackOptions(); using (Repository orgRepo = new Repository(orgRepoPath)) { - Assert.Throws(() => + packOptions.PackBuilder = builder => builder.Add(null); + + using (var ms = new MemoryStream()) { - orgRepo.ObjectDatabase.Pack(packBuilderOptions, builder => + Assert.Throws(() => { - builder.Add(null); + orgRepo.ObjectDatabase.Pack(ms, packOptions); }); - }); + } } } [Fact] public void ExceptionIfAddRecursivelyNullObjectID() { - string orgRepoPath = SandboxPackBuilderTestRepo(); - PackBuilderOptions packBuilderOptions = new PackBuilderOptions(orgRepoPath); + string orgRepoPath = SandboxPackDefinitionTestRepo(); + var packOptions = new PackOptions(); using (Repository orgRepo = new Repository(orgRepoPath)) { - Assert.Throws(() => + packOptions.PackBuilder = builder => builder.AddRecursively(null); + + using (var ms = new MemoryStream()) { - orgRepo.ObjectDatabase.Pack(packBuilderOptions, builder => + Assert.Throws(() => { - builder.AddRecursively(null); + orgRepo.ObjectDatabase.Pack(ms, packOptions); }); - }); + } + } + } + + [Fact] + public void WriteToStreamWritesAllObjects() + { + var testRepoPath = SandboxPackDefinitionTestRepo(); + using (var testRepo = new Repository(testRepoPath)) + { + using (var packOutput = new MemoryStream()) + { + PackResults results = testRepo.ObjectDatabase.Pack(packOutput, new PackOptions()); + + const string packHeader = "PACK"; + Assert.Equal( + packHeader, + Encoding.UTF8.GetString(packOutput.GetBuffer(), 0, packHeader.Length)); + + Assert.Equal(testRepo.ObjectDatabase.Count(), results.WrittenObjectsCount); + } } } } diff --git a/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs b/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs index c1fbefb7f..84c93bec9 100644 --- a/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs +++ b/LibGit2Sharp.Tests/TestHelpers/BaseFixture.cs @@ -41,7 +41,7 @@ static BaseFixture() private static string SubmoduleTargetTestRepoWorkingDirPath { get; set; } private static string AssumeUnchangedRepoWorkingDirPath { get; set; } public static string SubmoduleSmallTestRepoWorkingDirPath { get; set; } - public static string PackBuilderTestRepoPath { get; private set; } + public static string PackDefinitionTestRepoPath { get; private set; } public static DirectoryInfo ResourcesDirectory { get; private set; } @@ -75,7 +75,7 @@ private static void SetUpTestEnvironment() SubmoduleTargetTestRepoWorkingDirPath = Path.Combine(sourceRelativePath, "submodule_target_wd"); AssumeUnchangedRepoWorkingDirPath = Path.Combine(sourceRelativePath, "assume_unchanged_wd"); SubmoduleSmallTestRepoWorkingDirPath = Path.Combine(sourceRelativePath, "submodule_small_wd"); - PackBuilderTestRepoPath = Path.Combine(sourceRelativePath, "packbuilder_testrepo_wd"); + PackDefinitionTestRepoPath = Path.Combine(sourceRelativePath, "packbuilder_testrepo_wd"); CleanupTestReposOlderThan(TimeSpan.FromMinutes(15)); } @@ -176,9 +176,9 @@ public string SandboxSubmoduleSmallTestRepo() return path; } - protected string SandboxPackBuilderTestRepo() + protected string SandboxPackDefinitionTestRepo() { - return Sandbox(PackBuilderTestRepoPath); + return Sandbox(PackDefinitionTestRepoPath); } protected string Sandbox(string sourceDirectoryPath, params string[] additionalSourcePaths) diff --git a/LibGit2Sharp/Core/NativeMethods.cs b/LibGit2Sharp/Core/NativeMethods.cs index 0e3d6b3fc..d4a71f5f1 100644 --- a/LibGit2Sharp/Core/NativeMethods.cs +++ b/LibGit2Sharp/Core/NativeMethods.cs @@ -937,6 +937,13 @@ internal static extern int git_patch_line_stats( /* Push network progress notification function */ internal delegate int git_push_transfer_progress(uint current, uint total, UIntPtr bytes, IntPtr payload); internal delegate int git_packbuilder_progress(int stage, uint current, uint total, IntPtr payload); + internal delegate int git_packbuilder_foreach_callback(IntPtr buf, UIntPtr size, IntPtr payload); + + [DllImport(libgit2)] + internal static extern int git_packbuilder_foreach( + PackBuilderSafeHandle packbuilder, + git_packbuilder_foreach_callback foreachCallback, + IntPtr payload); [DllImport(libgit2)] internal static extern void git_packbuilder_free(IntPtr packbuilder); diff --git a/LibGit2Sharp/Core/Proxy.cs b/LibGit2Sharp/Core/Proxy.cs index dec711d24..fd65cecd1 100644 --- a/LibGit2Sharp/Core/Proxy.cs +++ b/LibGit2Sharp/Core/Proxy.cs @@ -1572,6 +1572,13 @@ public static void git_packbuilder_insert(PackBuilderSafeHandle packbuilder, Obj Ensure.ZeroResult(res); } + public static int git_packbuilder_foreach( + PackBuilderSafeHandle packbuilder, + NativeMethods.git_packbuilder_foreach_callback callback) + { + return NativeMethods.git_packbuilder_foreach(packbuilder, callback, IntPtr.Zero); + } + internal static void git_packbuilder_insert_commit(PackBuilderSafeHandle packbuilder, ObjectId targetId) { GitOid oid = targetId.Oid; diff --git a/LibGit2Sharp/LibGit2Sharp.csproj b/LibGit2Sharp/LibGit2Sharp.csproj index c2fc74e2e..d8ef12665 100644 --- a/LibGit2Sharp/LibGit2Sharp.csproj +++ b/LibGit2Sharp/LibGit2Sharp.csproj @@ -130,7 +130,7 @@ - + diff --git a/LibGit2Sharp/ObjectDatabase.cs b/LibGit2Sharp/ObjectDatabase.cs index 0adb09afd..537297406 100644 --- a/LibGit2Sharp/ObjectDatabase.cs +++ b/LibGit2Sharp/ObjectDatabase.cs @@ -645,58 +645,74 @@ public virtual MergeTreeResult MergeCommits(Commit ours, Commit theirs, MergeTre } /// - /// Packs all the objects in the and write a pack (.pack) and index (.idx) files for them. + /// Packs objects in the and write the resulting pack to + /// the . Note that no index is outputted. /// + /// Stream where the pack will be written to. /// Packing options - /// This method will invoke the default action of packing all objects in an arbitrary order. /// Packing results - public virtual PackBuilderResults Pack(PackBuilderOptions options) + public virtual PackResults Pack(Stream outputPackStream, PackOptions options) { - return InternalPack(options, builder => + Ensure.ArgumentNotNull(outputPackStream, "outputPackStream"); + if (!outputPackStream.CanWrite) { - foreach (GitObject obj in repo.ObjectDatabase) - { - builder.Add(obj.Id); - } - }); + throw new ArgumentException("The outputPackStream must be writeable."); + } + + return InternalPack(options, builder => builder.WriteTo(outputPackStream)); + } /// - /// Packs objects in the chosen by the packDelegate action - /// and write a pack (.pack) and index (.idx) files for them + /// Packs objects in the , and write a pack (.pack) and index (.idx) + /// files for them to the /// + /// Output directory for pack and idx files. /// Packing options - /// Packing action /// Packing results - public virtual PackBuilderResults Pack(PackBuilderOptions options, Action packDelegate) + public virtual PackResults Pack(string packDirectory, PackOptions options) { - return InternalPack(options, packDelegate); + Ensure.ArgumentNotNullOrEmptyString(packDirectory, "packDirectory"); + if (!Directory.Exists(packDirectory)) + { + throw new DirectoryNotFoundException("The Directory " + packDirectory + " does not exist."); + } + + return InternalPack(options, builder => builder.Write(packDirectory)); } /// - /// Packs objects in the and write a pack (.pack) and index (.idx) files for them. + /// Packs objects in the and use the + /// to write the results. /// For internal use only. /// /// Packing options - /// Packing action + /// Writing action action /// Packing results - private PackBuilderResults InternalPack(PackBuilderOptions options, Action packDelegate) + private PackResults InternalPack(PackOptions options, Action writeAction) { - Ensure.ArgumentNotNull(options, "options"); - Ensure.ArgumentNotNull(packDelegate, "packDelegate"); + options = options ?? new PackOptions(); - PackBuilderResults results = new PackBuilderResults(); + PackResults results = new PackResults(); - using (PackBuilder builder = new PackBuilder(repo)) + using (PackDefinition builder = new PackDefinition(repo)) { // set pre-build options builder.SetMaximumNumberOfThreads(options.MaximumNumberOfThreads); - // call the provided action - packDelegate(builder); + if (options.PackBuilder == null) + { + foreach (GitObject obj in repo.ObjectDatabase) + { + builder.Add(obj.Id); + } + } + else + { + options.PackBuilder(builder); + } - // writing the pack and index files - builder.Write(options.PackDirectoryPath); + writeAction(builder); // adding the results to the PackBuilderResults object results.WrittenObjectsCount = builder.WrittenObjectsCount; diff --git a/LibGit2Sharp/PackBuilder.cs b/LibGit2Sharp/PackDefinition.cs similarity index 64% rename from LibGit2Sharp/PackBuilder.cs rename to LibGit2Sharp/PackDefinition.cs index 1f797c9bf..715ad7738 100644 --- a/LibGit2Sharp/PackBuilder.cs +++ b/LibGit2Sharp/PackDefinition.cs @@ -6,16 +6,16 @@ namespace LibGit2Sharp { /// - /// Representation of a git PackBuilder. + /// Provides access to packing capabilities. /// - public sealed class PackBuilder : IDisposable + public sealed class PackDefinition : IDisposable { private readonly PackBuilderSafeHandle packBuilderHandle; /// - /// Constructs a PackBuilder for a . + /// Constructs a for the . /// - internal PackBuilder(Repository repository) + internal PackDefinition(Repository repository) { Ensure.ArgumentNotNull(repository, "repository"); @@ -23,7 +23,7 @@ internal PackBuilder(Repository repository) } /// - /// Inserts a single to the PackBuilder. + /// Inserts a single into this . /// For an optimal pack it's mandatory to insert objects in recency order, commits followed by trees and blobs. (quoted from libgit2 API ref) /// /// The object to be inserted. @@ -49,7 +49,7 @@ public void AddRecursively(T gitObject) where T : GitObject } /// - /// Inserts a single object to the PackBuilder by its . + /// Inserts a single object to the by its . /// For an optimal pack it's mandatory to insert objects in recency order, commits followed by trees and blobs. (quoted from libgit2 API ref) /// /// The object ID to be inserted. @@ -75,7 +75,7 @@ public void AddRecursively(ObjectId id) } /// - /// Disposes the PackBuilder object. + /// Releases handles owned by this object. /// void IDisposable.Dispose() { @@ -92,7 +92,46 @@ internal void Write(string path) } /// - /// Sets number of threads to spawn. + /// Write the new pack to the . The index will not be + /// written, so the caller is responsible for indexing objects in the new pack. + /// + /// The output pack stream. + internal void WriteTo(Stream outputStream) + { + Ensure.ArgumentNotNull(outputStream, "outputStream"); + + // 64K is optimal buffer size per https://technet.microsoft.com/en-us/library/cc938632.aspx + const int defaultBufferSize = 64 * 1024; + + Proxy.git_packbuilder_foreach(packBuilderHandle, (IntPtr buf, UIntPtr size, IntPtr payload) => + { + try + { + long objectSize = (long)size; + unsafe + { + using (var input = new UnmanagedMemoryStream((byte*)buf.ToPointer(), objectSize)) + { + input.CopyTo( + outputStream, + objectSize > defaultBufferSize ? defaultBufferSize : (int)objectSize); + } + } + } + catch (Exception ex) + { + Log.Write(LogLevel.Error, "PackBuilder.WriteTo callback exception"); + Log.Write(LogLevel.Error, ex.ToString()); + Proxy.giterr_set_str(GitErrorCategory.Callback, ex); + return (int)GitErrorCode.Error; + } + + return (int)GitErrorCode.Ok; + }); + } + + /// + /// Sets number of threads to spawn during packing. /// /// Returns the number of actual threads to be used. /// The Number of threads to spawn. An argument of 0 ensures using all available CPUs @@ -103,7 +142,7 @@ internal int SetMaximumNumberOfThreads(int nThread) } /// - /// Number of objects the PackBuilder will write out. + /// Number of objects that will be written out during packing. /// internal long ObjectsCount { @@ -111,27 +150,21 @@ internal long ObjectsCount } /// - /// Number of objects the PackBuilder has already written out. - /// This is only correct after the pack file has been written. + /// Total number of objects that have been written out after packing. /// internal long WrittenObjectsCount { get { return Proxy.git_packbuilder_written(packBuilderHandle); } } - - internal PackBuilderSafeHandle Handle - { - get { return packBuilderHandle; } - } } /// /// The results of pack process of the . /// - public struct PackBuilderResults + public struct PackResults { /// - /// Number of objects the PackBuilder has already written out. + /// Total number of objects that have been written out after packing. /// public long WrittenObjectsCount { get; internal set; } } @@ -139,46 +172,10 @@ public struct PackBuilderResults /// /// Packing options of the . /// - public sealed class PackBuilderOptions + public sealed class PackOptions { - private string path; private int nThreads; - /// - /// Constructor - /// - /// Directory path to write the pack and index files to it - /// The default value for maximum number of threads to spawn is 0 which ensures using all available CPUs. - /// if packDirectory is null or empty - /// if packDirectory doesn't exist - public PackBuilderOptions(string packDirectory) - { - PackDirectoryPath = packDirectory; - MaximumNumberOfThreads = 0; - } - - /// - /// Directory path to write the pack and index files to it. - /// - public string PackDirectoryPath - { - set - { - Ensure.ArgumentNotNullOrEmptyString(value, "packDirectory"); - - if (!Directory.Exists(value)) - { - throw new DirectoryNotFoundException("The Directory " + value + " does not exist."); - } - - path = value; - } - get - { - return path; - } - } - /// /// Maximum number of threads to spawn. /// The default value is 0 which ensures using all available CPUs. @@ -187,11 +184,7 @@ public int MaximumNumberOfThreads { set { - if (value < 0) - { - throw new ArgumentException("Argument can not be negative", "value"); - } - + Ensure.ArgumentPositiveInt32(value, "value"); nThreads = value; } get @@ -199,5 +192,12 @@ public int MaximumNumberOfThreads return nThreads; } } + + /// + /// Packing action to perform instead of the default, which is to + /// pack all objects in the repository into a single packfile and index, + /// enumerating them in an arbitrary order. + /// + public Action PackBuilder { get; set; } } }