From 9c1ea031e74ff9421279f9b35f4236d0c1ba691d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 11 Nov 2020 10:04:50 +0200 Subject: [PATCH 1/9] Add support PackBuilder methods --- pygit2/__init__.py | 1 + pygit2/_run.py | 2 +- pygit2/decl/pack.h | 11 +++ pygit2/decl/types.h | 1 + pygit2/packbuilder.py | 80 ++++++++++++++++ pygit2/repository.py | 34 +++++++ src/packbuilder.c | 216 ++++++++++++++++++++++++++++++++++++++++++ src/packbuilder.h | 42 ++++++++ src/pygit2.c | 6 ++ src/types.h | 8 ++ 10 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 pygit2/packbuilder.py create mode 100755 src/packbuilder.c create mode 100755 src/packbuilder.h diff --git a/pygit2/__init__.py b/pygit2/__init__.py index 42988c3b9..2ed82db3a 100644 --- a/pygit2/__init__.py +++ b/pygit2/__init__.py @@ -40,6 +40,7 @@ from .index import Index, IndexEntry from .remote import Remote from .repository import Repository +from .packbuilder import PackBuilder from .settings import Settings from .submodule import Submodule from .utils import to_bytes, to_str diff --git a/pygit2/_run.py b/pygit2/_run.py index 5fdaf77a7..974d5ca37 100644 --- a/pygit2/_run.py +++ b/pygit2/_run.py @@ -61,10 +61,10 @@ 'strarray.h', 'diff.h', 'checkout.h', - 'pack.h', 'transport.h', 'proxy.h', 'indexer.h', + 'pack.h', 'remote.h', 'clone.h', 'common.h', diff --git a/pygit2/decl/pack.h b/pygit2/decl/pack.h index 314a272ac..42408670e 100644 --- a/pygit2/decl/pack.h +++ b/pygit2/decl/pack.h @@ -3,3 +3,14 @@ typedef int (*git_packbuilder_progress)( uint32_t current, uint32_t total, void *payload); + + +int git_packbuilder_new(git_packbuilder **out, git_repository *repo); +int git_packbuilder_insert(git_packbuilder *pb, const git_oid *id, const char *name); +size_t git_packbuilder_object_count(git_packbuilder *pb); +unsigned int git_packbuilder_set_threads(git_packbuilder *pb, unsigned int n); +int git_packbuilder_write(git_packbuilder *pb, const char *path, unsigned int mode, git_indexer_progress_cb progress_cb, void *progress_cb_payload); +int git_packbuilder_insert_recur(git_packbuilder *pb, const git_oid *id, const char *name); +uint32_t git_packbuilder_written(git_packbuilder *pb); + +const git_oid * git_packbuilder_hash(git_packbuilder *pb); \ No newline at end of file diff --git a/pygit2/decl/types.h b/pygit2/decl/types.h index 085270849..8770da7a5 100644 --- a/pygit2/decl/types.h +++ b/pygit2/decl/types.h @@ -10,6 +10,7 @@ typedef struct git_repository git_repository; typedef struct git_submodule git_submodule; typedef struct git_transport git_transport; typedef struct git_tree git_tree; +typedef struct git_packbuilder git_packbuilder; typedef int64_t git_off_t; typedef int64_t git_time_t; diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py new file mode 100644 index 000000000..82852d9c6 --- /dev/null +++ b/pygit2/packbuilder.py @@ -0,0 +1,80 @@ +# Copyright 2010-2020 The pygit2 contributors +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License, version 2, +# as published by the Free Software Foundation. +# +# In addition to the permissions in the GNU General Public License, +# the authors give you unlimited permission to link the compiled +# version of this file into combinations with other programs, +# and to distribute those combinations without any restriction +# coming from the use of this file. (The General Public License +# restrictions do apply in other respects; for example, they cover +# modification of the file, and distribution when not linked into +# a combined executable.) +# +# This file is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + + +# Import from pygit2 +from ._pygit2 import PackBuilder as _PackBuilder +from .errors import check_error +from .ffi import ffi, C +from .utils import to_bytes + + +class PackBuilder(_PackBuilder): + + def __init__(self, *args, **kwargs): + self.repo = args[0] + super().__init__(args[0]) + packbuilder_cptr = ffi.new('git_packbuilder **') + ffi.buffer(packbuilder_cptr)[:] = self._pointer[:] + self._packbuilder = packbuilder_cptr[0] + + @staticmethod + def convert_object_to_oid(git_object): + oid = ffi.new('git_oid *') + ffi.buffer(oid)[:] = git_object.raw[:] + return oid + + def add(self, git_object): + oid = self.convert_object_to_oid(git_object) + err = C.git_packbuilder_insert(self._packbuilder, oid, ffi.NULL) + check_error(err) + + def add_recursive(self, git_object): + oid = self.convert_object_to_oid(git_object) + err = C.git_packbuilder_insert_recur(self._packbuilder, oid, ffi.NULL) + check_error(err) + + def object_count(self): + return C.git_packbuilder_object_count(self._packbuilder) + + def set_max_threads(self, n_threads): + return C.git_packbuilder_set_threads(self._packbuilder, n_threads) + + def write(self, path): + err = C.git_packbuilder_write(self._packbuilder, to_bytes(path), 0, ffi.NULL, ffi.NULL) + check_error(err) + + @property + def written_objects_count(self): + return C.git_packbuilder_written(self._packbuilder) + + @classmethod + def _from_c(cls, ptr, repo): + cptr = ffi.new('git_packbuilder **') + cptr[0] = ptr + packbuilder = cls.__new__(cls) + packbuilder.repo = repo + super(cls, packbuilder)._from_c(bytes(ffi.buffer(cptr)[:]), repo) + return packbuilder diff --git a/pygit2/repository.py b/pygit2/repository.py index a7180f9b0..771260b83 100644 --- a/pygit2/repository.py +++ b/pygit2/repository.py @@ -49,6 +49,7 @@ from .blame import Blame from .utils import to_bytes, StrArray from .submodule import Submodule +from .packbuilder import PackBuilder class BaseRepository(_Repository): @@ -83,6 +84,39 @@ def write(self, *args, **kwargs): object.""" return self.odb.write(*args, **kwargs) + def pack(self, path, pack_delegate=None, max_threads=None): + """Pack the objects in the odb chosen by the pack_delegate function + and write .pack and .idx files for them. + + Returns: the number of objects written to the pack + + Parameters: + + path + The path to which the .pack and .idx files should be written. + + pack_delegate + The method which will provide add the objects to the pack builder. Defaults to all objects. + + max_threads + The maximum number of threads the PackBuilder will spawn. + """ + + def pack_all_objects(pack_builder): + for obj in self.odb: + pack_builder.add(obj) + + pack_delegate = pack_delegate or pack_all_objects + + builder = PackBuilder(self) + if max_threads: + builder.set_max_threads(max_threads) + pack_delegate(builder) + builder.write(path) + + return builder.written_objects_count + + def __iter__(self): return iter(self.odb) diff --git a/src/packbuilder.c b/src/packbuilder.c new file mode 100755 index 000000000..4151ecd44 --- /dev/null +++ b/src/packbuilder.c @@ -0,0 +1,216 @@ +/* + * Copyright 2010-2020 The pygit2 contributors + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#include +#include "error.h" +#include "types.h" +#include "utils.h" + +extern PyTypeObject RepositoryType; + +/* forward-declaration for PackBuilder._from_c() */ +PyTypeObject PackBuilderType; + +PyObject * +wrap_packbuilder(git_packbuilder *c_packbuilder, Repository *repo) +{ + PackBuilder *py_packbuilder = PyObject_GC_New(PackBuilder, &PackBuilderType); + + if (py_packbuilder) { + py_packbuilder->packbuilder = c_packbuilder; + py_packbuilder->owned = 1; + py_packbuilder->repo = repo; + } + + return (PyObject *)py_packbuilder; +} + +int +PackBuilder_init(PackBuilder *self, PyObject *args, PyObject *kwds) +{ + Repository *repo = NULL; + + if (kwds && PyDict_Size(kwds) > 0) { + PyErr_SetString(PyExc_TypeError, + "PackBuilder takes no keyword arguments"); + return -1; + } + if (!PyArg_ParseTuple(args, "O!", &RepositoryType, &repo)) { + return -1; + } + /* Create packbuilder */ + int err = git_packbuilder_new(&self->packbuilder, repo->repo); + if (err != 0) { + Error_set(err); + return -1; + } + self->owned = 1; + self->repo = repo; + return 0; +} + +PyDoc_STRVAR(PackBuilder__from_c__doc__, "Init a PackBuilder from a pointer. For internal use only."); +PyObject * +PackBuilder__from_c(PackBuilder *py_packbuilder, PyObject *args) +{ + PyObject *py_pointer, *py_free; + char *buffer; + Py_ssize_t len; + int err; + + py_packbuilder->packbuilder = NULL; + Repository *repo = NULL; + + if (!PyArg_ParseTuple(args, "OO!", &py_pointer, &RepositoryType, &repo)) + return NULL; + + err = PyBytes_AsStringAndSize(py_pointer, &buffer, &len); + if (err < 0) + return NULL; + + if (len != sizeof(git_packbuilder *)) { + PyErr_SetString(PyExc_TypeError, "invalid pointer length"); + return NULL; + } + + py_packbuilder->packbuilder = *((git_packbuilder **) buffer); + py_packbuilder->owned = py_free == Py_True; + py_packbuilder->repo = repo; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(PackBuilder__disown__doc__, "Mark the object as not-owned by us. For internal use only."); +PyObject * +PackBuilder__disown(PackBuilder *py_packbuilder) +{ + py_packbuilder->owned = 0; + Py_RETURN_NONE; +} + +void +PackBuilder_dealloc(PackBuilder *self) +{ + PyObject_GC_UnTrack(self); + + if (self->owned) + git_packbuilder_free(self->packbuilder); + + Py_TYPE(self)->tp_free(self); +} + +int +PackBuilder_traverse(PackBuilder *self, visitproc visit, void *arg) +{ + return 0; +} + +int +PackBuilder_clear(PackBuilder *self) +{ + return 0; +} + + +PyDoc_STRVAR(PackBuilder__pointer__doc__, "Get the packbuilder's pointer. For internal use only."); +PyObject * +PackBuilder__pointer__get__(PackBuilder *self) +{ + /* Bytes means a raw buffer */ + return PyBytes_FromStringAndSize((char *) &self->packbuilder, sizeof(git_packbuilder *)); +} + +PyDoc_STRVAR(PackBuilder__repo__doc__, "Get the packbuilder's repository pointer. For internal use only."); +PyObject * +PackBuilder__repo__get__(PackBuilder *self) +{ + /* Bytes means a raw buffer */ + return PyBytes_FromStringAndSize((char *) &self->repo, sizeof(Repository *)); +} + + + +PyMethodDef PackBuilder_methods[] = { + METHOD(PackBuilder, _disown, METH_NOARGS), + METHOD(PackBuilder, _from_c, METH_VARARGS), + {NULL} +}; + +PyGetSetDef PackBuilder_getseters[] = { + GETTER(PackBuilder, _pointer), + GETTER(PackBuilder, _repo), + {NULL} +}; + + +PyDoc_STRVAR(PackBuilder__doc__, + "PackBuilder(backend) -> PackBuilder\n" + "\n" + "Git packbuilder."); + +PyTypeObject PackBuilderType = { + PyVarObject_HEAD_INIT(NULL, 0) + "_pygit2.PackBuilder", /* tp_name */ + sizeof(PackBuilder), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)PackBuilder_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | + Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC, /* tp_flags */ + PackBuilder__doc__, /* tp_doc */ + (traverseproc)PackBuilder_traverse, /* tp_traverse */ + (inquiry)PackBuilder_clear, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + PackBuilder_methods, /* tp_methods */ + 0, /* tp_members */ + PackBuilder_getseters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)PackBuilder_init, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; diff --git a/src/packbuilder.h b/src/packbuilder.h new file mode 100755 index 000000000..cb94a1fe4 --- /dev/null +++ b/src/packbuilder.h @@ -0,0 +1,42 @@ +/* + * Copyright 2010-2020 The pygit2 contributors + * + * This file is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, + * as published by the Free Software Foundation. + * + * In addition to the permissions in the GNU General Public License, + * the authors give you unlimited permission to link the compiled + * version of this file into combinations with other programs, + * and to distribute those combinations without any restriction + * coming from the use of this file. (The General Public License + * restrictions do apply in other respects; for example, they cover + * modification of the file, and distribution when not linked into + * a combined executable.) + * + * This file is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; see the file COPYING. If not, write to + * the Free Software Foundation, 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef INCLUDE_pygit2_packbuilder_h +#define INCLUDE_pygit2_packbuilder_h + +#define PY_SSIZE_T_CLEAN +#include +#include +#include "types.h" + +PyObject *wrap_packbuilder(git_packbuilder *c_packbuilder, Repository *repo); + +int PackBuilder_init(PackBuilder *self, PyObject *args, PyObject *kwds); +int PackBuilder_traverse(PackBuilder *self, visitproc visit, void *arg); +int PackBuilder_clear(PackBuilder *self); + +#endif diff --git a/src/pygit2.c b/src/pygit2.c index a90f9767e..19adf38b2 100644 --- a/src/pygit2.c +++ b/src/pygit2.c @@ -33,6 +33,7 @@ #include "types.h" #include "utils.h" #include "repository.h" +#include "packbuilder.h" #include "oid.h" #include "options.h" @@ -41,6 +42,7 @@ PyObject *AlreadyExistsError; PyObject *InvalidSpecError; extern PyTypeObject RepositoryType; +extern PyTypeObject PackBuilderType; extern PyTypeObject OdbType; extern PyTypeObject OdbBackendType; extern PyTypeObject OdbBackendPackType; @@ -335,6 +337,10 @@ PyInit__pygit2(void) INIT_TYPE(RepositoryType, NULL, PyType_GenericNew) ADD_TYPE(m, Repository) + /* PackBuilder */ + INIT_TYPE(PackBuilderType, NULL, PyType_GenericNew) + ADD_TYPE(m, PackBuilder) + /* Odb */ INIT_TYPE(OdbType, NULL, PyType_GenericNew) ADD_TYPE(m, Odb) diff --git a/src/types.h b/src/types.h index 8ea01d380..3ccbc4001 100644 --- a/src/types.h +++ b/src/types.h @@ -50,6 +50,14 @@ typedef struct { int owned; /* _from_c() sometimes means we don't own the C pointer */ } Repository; +/* git_packbuilder */ +typedef struct { + PyObject_HEAD + git_packbuilder *packbuilder; + int owned; /* _from_c() sometimes means we don't own the C pointer */ + Repository *repo; +} PackBuilder; + typedef struct { PyObject_HEAD From 9baf405aa063060e113c8c813bdee2e218268911 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 16 Nov 2020 11:12:02 +0200 Subject: [PATCH 2/9] update Packbuilder to just use cffi --- pygit2/decl/pack.h | 10 +- pygit2/packbuilder.py | 50 ++++++---- src/packbuilder.c | 216 ------------------------------------------ src/packbuilder.h | 42 -------- src/pygit2.c | 6 -- src/types.h | 9 -- 6 files changed, 36 insertions(+), 297 deletions(-) delete mode 100755 src/packbuilder.c delete mode 100755 src/packbuilder.h diff --git a/pygit2/decl/pack.h b/pygit2/decl/pack.h index 42408670e..0e6973045 100644 --- a/pygit2/decl/pack.h +++ b/pygit2/decl/pack.h @@ -4,13 +4,15 @@ typedef int (*git_packbuilder_progress)( uint32_t total, void *payload); - int git_packbuilder_new(git_packbuilder **out, git_repository *repo); +void git_packbuilder_free(git_packbuilder *pb); + int git_packbuilder_insert(git_packbuilder *pb, const git_oid *id, const char *name); +int git_packbuilder_insert_recur(git_packbuilder *pb, const git_oid *id, const char *name); + size_t git_packbuilder_object_count(git_packbuilder *pb); -unsigned int git_packbuilder_set_threads(git_packbuilder *pb, unsigned int n); + int git_packbuilder_write(git_packbuilder *pb, const char *path, unsigned int mode, git_indexer_progress_cb progress_cb, void *progress_cb_payload); -int git_packbuilder_insert_recur(git_packbuilder *pb, const git_oid *id, const char *name); uint32_t git_packbuilder_written(git_packbuilder *pb); -const git_oid * git_packbuilder_hash(git_packbuilder *pb); \ No newline at end of file +unsigned int git_packbuilder_set_threads(git_packbuilder *pb, unsigned int n); \ No newline at end of file diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index 82852d9c6..d158662c0 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -25,20 +25,42 @@ # Import from pygit2 -from ._pygit2 import PackBuilder as _PackBuilder from .errors import check_error from .ffi import ffi, C from .utils import to_bytes -class PackBuilder(_PackBuilder): +class PackBuilder: - def __init__(self, *args, **kwargs): - self.repo = args[0] - super().__init__(args[0]) - packbuilder_cptr = ffi.new('git_packbuilder **') - ffi.buffer(packbuilder_cptr)[:] = self._pointer[:] - self._packbuilder = packbuilder_cptr[0] + def __init__(self, repo): + + cpackbuilder = ffi.new('git_packbuilder **') + err = C.git_packbuilder_new(cpackbuilder, repo._repo) + check_error(err) + + self._repo = repo + self._packbuilder = cpackbuilder[0] + self._cpackbuilder = cpackbuilder + + + @classmethod + def _from_c(cls, repo, ptr): + packbuilder = cls.__new__(cls) + packbuilder._repo = repo + packbuilder._packbuilder = ptr[0] + packbuilder._cpackbuilder = ptr + + return packbuilder + + @property + def _pointer(self): + return bytes(ffi.buffer(self._packbuilder)[:]) + + def __del__(self): + C.git_packbuilder_free(self._packbuilder) + + def __len__(self): + return C.git_packbuilder_object_count(self._packbuilder) @staticmethod def convert_object_to_oid(git_object): @@ -56,9 +78,6 @@ def add_recursive(self, git_object): err = C.git_packbuilder_insert_recur(self._packbuilder, oid, ffi.NULL) check_error(err) - def object_count(self): - return C.git_packbuilder_object_count(self._packbuilder) - def set_max_threads(self, n_threads): return C.git_packbuilder_set_threads(self._packbuilder, n_threads) @@ -69,12 +88,3 @@ def write(self, path): @property def written_objects_count(self): return C.git_packbuilder_written(self._packbuilder) - - @classmethod - def _from_c(cls, ptr, repo): - cptr = ffi.new('git_packbuilder **') - cptr[0] = ptr - packbuilder = cls.__new__(cls) - packbuilder.repo = repo - super(cls, packbuilder)._from_c(bytes(ffi.buffer(cptr)[:]), repo) - return packbuilder diff --git a/src/packbuilder.c b/src/packbuilder.c deleted file mode 100755 index 4151ecd44..000000000 --- a/src/packbuilder.c +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2010-2020 The pygit2 contributors - * - * This file is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, - * as published by the Free Software Foundation. - * - * In addition to the permissions in the GNU General Public License, - * the authors give you unlimited permission to link the compiled - * version of this file into combinations with other programs, - * and to distribute those combinations without any restriction - * coming from the use of this file. (The General Public License - * restrictions do apply in other respects; for example, they cover - * modification of the file, and distribution when not linked into - * a combined executable.) - * - * This file is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; see the file COPYING. If not, write to - * the Free Software Foundation, 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#include -#include "error.h" -#include "types.h" -#include "utils.h" - -extern PyTypeObject RepositoryType; - -/* forward-declaration for PackBuilder._from_c() */ -PyTypeObject PackBuilderType; - -PyObject * -wrap_packbuilder(git_packbuilder *c_packbuilder, Repository *repo) -{ - PackBuilder *py_packbuilder = PyObject_GC_New(PackBuilder, &PackBuilderType); - - if (py_packbuilder) { - py_packbuilder->packbuilder = c_packbuilder; - py_packbuilder->owned = 1; - py_packbuilder->repo = repo; - } - - return (PyObject *)py_packbuilder; -} - -int -PackBuilder_init(PackBuilder *self, PyObject *args, PyObject *kwds) -{ - Repository *repo = NULL; - - if (kwds && PyDict_Size(kwds) > 0) { - PyErr_SetString(PyExc_TypeError, - "PackBuilder takes no keyword arguments"); - return -1; - } - if (!PyArg_ParseTuple(args, "O!", &RepositoryType, &repo)) { - return -1; - } - /* Create packbuilder */ - int err = git_packbuilder_new(&self->packbuilder, repo->repo); - if (err != 0) { - Error_set(err); - return -1; - } - self->owned = 1; - self->repo = repo; - return 0; -} - -PyDoc_STRVAR(PackBuilder__from_c__doc__, "Init a PackBuilder from a pointer. For internal use only."); -PyObject * -PackBuilder__from_c(PackBuilder *py_packbuilder, PyObject *args) -{ - PyObject *py_pointer, *py_free; - char *buffer; - Py_ssize_t len; - int err; - - py_packbuilder->packbuilder = NULL; - Repository *repo = NULL; - - if (!PyArg_ParseTuple(args, "OO!", &py_pointer, &RepositoryType, &repo)) - return NULL; - - err = PyBytes_AsStringAndSize(py_pointer, &buffer, &len); - if (err < 0) - return NULL; - - if (len != sizeof(git_packbuilder *)) { - PyErr_SetString(PyExc_TypeError, "invalid pointer length"); - return NULL; - } - - py_packbuilder->packbuilder = *((git_packbuilder **) buffer); - py_packbuilder->owned = py_free == Py_True; - py_packbuilder->repo = repo; - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(PackBuilder__disown__doc__, "Mark the object as not-owned by us. For internal use only."); -PyObject * -PackBuilder__disown(PackBuilder *py_packbuilder) -{ - py_packbuilder->owned = 0; - Py_RETURN_NONE; -} - -void -PackBuilder_dealloc(PackBuilder *self) -{ - PyObject_GC_UnTrack(self); - - if (self->owned) - git_packbuilder_free(self->packbuilder); - - Py_TYPE(self)->tp_free(self); -} - -int -PackBuilder_traverse(PackBuilder *self, visitproc visit, void *arg) -{ - return 0; -} - -int -PackBuilder_clear(PackBuilder *self) -{ - return 0; -} - - -PyDoc_STRVAR(PackBuilder__pointer__doc__, "Get the packbuilder's pointer. For internal use only."); -PyObject * -PackBuilder__pointer__get__(PackBuilder *self) -{ - /* Bytes means a raw buffer */ - return PyBytes_FromStringAndSize((char *) &self->packbuilder, sizeof(git_packbuilder *)); -} - -PyDoc_STRVAR(PackBuilder__repo__doc__, "Get the packbuilder's repository pointer. For internal use only."); -PyObject * -PackBuilder__repo__get__(PackBuilder *self) -{ - /* Bytes means a raw buffer */ - return PyBytes_FromStringAndSize((char *) &self->repo, sizeof(Repository *)); -} - - - -PyMethodDef PackBuilder_methods[] = { - METHOD(PackBuilder, _disown, METH_NOARGS), - METHOD(PackBuilder, _from_c, METH_VARARGS), - {NULL} -}; - -PyGetSetDef PackBuilder_getseters[] = { - GETTER(PackBuilder, _pointer), - GETTER(PackBuilder, _repo), - {NULL} -}; - - -PyDoc_STRVAR(PackBuilder__doc__, - "PackBuilder(backend) -> PackBuilder\n" - "\n" - "Git packbuilder."); - -PyTypeObject PackBuilderType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pygit2.PackBuilder", /* tp_name */ - sizeof(PackBuilder), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)PackBuilder_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC, /* tp_flags */ - PackBuilder__doc__, /* tp_doc */ - (traverseproc)PackBuilder_traverse, /* tp_traverse */ - (inquiry)PackBuilder_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - PackBuilder_methods, /* tp_methods */ - 0, /* tp_members */ - PackBuilder_getseters, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)PackBuilder_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; diff --git a/src/packbuilder.h b/src/packbuilder.h deleted file mode 100755 index cb94a1fe4..000000000 --- a/src/packbuilder.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2010-2020 The pygit2 contributors - * - * This file is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License, version 2, - * as published by the Free Software Foundation. - * - * In addition to the permissions in the GNU General Public License, - * the authors give you unlimited permission to link the compiled - * version of this file into combinations with other programs, - * and to distribute those combinations without any restriction - * coming from the use of this file. (The General Public License - * restrictions do apply in other respects; for example, they cover - * modification of the file, and distribution when not linked into - * a combined executable.) - * - * This file is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; see the file COPYING. If not, write to - * the Free Software Foundation, 51 Franklin Street, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#ifndef INCLUDE_pygit2_packbuilder_h -#define INCLUDE_pygit2_packbuilder_h - -#define PY_SSIZE_T_CLEAN -#include -#include -#include "types.h" - -PyObject *wrap_packbuilder(git_packbuilder *c_packbuilder, Repository *repo); - -int PackBuilder_init(PackBuilder *self, PyObject *args, PyObject *kwds); -int PackBuilder_traverse(PackBuilder *self, visitproc visit, void *arg); -int PackBuilder_clear(PackBuilder *self); - -#endif diff --git a/src/pygit2.c b/src/pygit2.c index 19adf38b2..a90f9767e 100644 --- a/src/pygit2.c +++ b/src/pygit2.c @@ -33,7 +33,6 @@ #include "types.h" #include "utils.h" #include "repository.h" -#include "packbuilder.h" #include "oid.h" #include "options.h" @@ -42,7 +41,6 @@ PyObject *AlreadyExistsError; PyObject *InvalidSpecError; extern PyTypeObject RepositoryType; -extern PyTypeObject PackBuilderType; extern PyTypeObject OdbType; extern PyTypeObject OdbBackendType; extern PyTypeObject OdbBackendPackType; @@ -337,10 +335,6 @@ PyInit__pygit2(void) INIT_TYPE(RepositoryType, NULL, PyType_GenericNew) ADD_TYPE(m, Repository) - /* PackBuilder */ - INIT_TYPE(PackBuilderType, NULL, PyType_GenericNew) - ADD_TYPE(m, PackBuilder) - /* Odb */ INIT_TYPE(OdbType, NULL, PyType_GenericNew) ADD_TYPE(m, Odb) diff --git a/src/types.h b/src/types.h index 3ccbc4001..b79e75e95 100644 --- a/src/types.h +++ b/src/types.h @@ -50,15 +50,6 @@ typedef struct { int owned; /* _from_c() sometimes means we don't own the C pointer */ } Repository; -/* git_packbuilder */ -typedef struct { - PyObject_HEAD - git_packbuilder *packbuilder; - int owned; /* _from_c() sometimes means we don't own the C pointer */ - Repository *repo; -} PackBuilder; - - typedef struct { PyObject_HEAD git_oid oid; From c415819ff1e7c18f6730afd74c5341e83146401d Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 16 Nov 2020 14:59:06 +0200 Subject: [PATCH 3/9] remove unused _from_c method --- pygit2/packbuilder.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index d158662c0..82d188933 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -42,16 +42,6 @@ def __init__(self, repo): self._packbuilder = cpackbuilder[0] self._cpackbuilder = cpackbuilder - - @classmethod - def _from_c(cls, repo, ptr): - packbuilder = cls.__new__(cls) - packbuilder._repo = repo - packbuilder._packbuilder = ptr[0] - packbuilder._cpackbuilder = ptr - - return packbuilder - @property def _pointer(self): return bytes(ffi.buffer(self._packbuilder)[:]) From 402fa4e4c3c5f1c4b5de4fa80f979bee38cf9791 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 16 Nov 2020 15:01:15 +0200 Subject: [PATCH 4/9] Add tests for packbuilder --- pygit2/packbuilder.py | 2 +- src/types.h | 1 + test/test_packbuilder.py | 112 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 test/test_packbuilder.py diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index 82d188933..de57e1f58 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -63,7 +63,7 @@ def add(self, git_object): err = C.git_packbuilder_insert(self._packbuilder, oid, ffi.NULL) check_error(err) - def add_recursive(self, git_object): + def add_recur(self, git_object): oid = self.convert_object_to_oid(git_object) err = C.git_packbuilder_insert_recur(self._packbuilder, oid, ffi.NULL) check_error(err) diff --git a/src/types.h b/src/types.h index b79e75e95..8ea01d380 100644 --- a/src/types.h +++ b/src/types.h @@ -50,6 +50,7 @@ typedef struct { int owned; /* _from_c() sometimes means we don't own the C pointer */ } Repository; + typedef struct { PyObject_HEAD git_oid oid; diff --git a/test/test_packbuilder.py b/test/test_packbuilder.py new file mode 100644 index 000000000..eaae1ccbb --- /dev/null +++ b/test/test_packbuilder.py @@ -0,0 +1,112 @@ +# Copyright 2010-2020 The pygit2 contributors +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License, version 2, +# as published by the Free Software Foundation. +# +# In addition to the permissions in the GNU General Public License, +# the authors give you unlimited permission to link the compiled +# version of this file into combinations with other programs, +# and to distribute those combinations without any restriction +# coming from the use of this file. (The General Public License +# restrictions do apply in other respects; for example, they cover +# modification of the file, and distribution when not linked into +# a combined executable.) +# +# This file is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +"""Tests for Index files.""" + +import os +import shutil + +import pytest + +import pygit2 +from pygit2 import PackBuilder +from . import utils + + +def test_create_packbuilder(testrepo): + # simple test of PackBuilder creation + packbuilder = PackBuilder(testrepo) + assert len(packbuilder) == 0 + + +def test_add(testrepo): + # Add a few objects and confirm that the count is correct + packbuilder = PackBuilder(testrepo) + objects_to_add = [obj for obj in testrepo] + packbuilder.add(objects_to_add[0]) + assert len(packbuilder) == 1 + packbuilder.add(objects_to_add[1]) + assert len(packbuilder) == 2 + + +def test_add_recursively(testrepo): + # Add the head object and referenced objects recursively and confirm that the count is correct + packbuilder = PackBuilder(testrepo) + packbuilder.add_recur(testrepo.head.target) + + #expect a count of 4 made up of the following referenced objects: + # Commit + # Tree + # Blob: hello.txt + # Blob: .gitignore + + assert len(packbuilder) == 4 + + +def test_repo_pack(testrepo, tmp_path): + # pack the repo with the default strategy + confirm_same_repo_after_packing(testrepo, tmp_path, None) + + +def test_pack_with_delegate(testrepo, tmp_path): + # loop through all branches and add each commit to the packbuilder + def pack_delegate(pb): + for branch in pb._repo.branches: + br = pb._repo.branches.get(branch) + for commit in br.log(): + pb.add_recur(commit.oid_new) + confirm_same_repo_after_packing(testrepo, tmp_path, pack_delegate) + + +def setup_second_repo(tmp_path): + # helper method to set up a second repo for comparison + tmp_path_2 = os.path.join(tmp_path, 'test_repo2') + with utils.TemporaryRepository('testrepo.tar', tmp_path_2) as path: + testrepo = pygit2.Repository(path) + return testrepo + +def confirm_same_repo_after_packing(testrepo, tmp_path, pack_delegate): + # Helper method to confirm the contents of two repos before and after packing + pack_repo = setup_second_repo(tmp_path) + + objects_dir = os.path.join(pack_repo.path, 'objects') + shutil.rmtree(objects_dir) + pack_path = os.path.join(pack_repo.path, 'objects', 'pack') + os.makedirs(pack_path) + + # assert that the number of written objects is the same as the number of objects in the repo + written_objects = testrepo.pack(pack_path, pack_delegate=pack_delegate) + assert written_objects == len([obj for obj in testrepo]) + + + # assert that the number of objects in the pack repo is the same as the original repo + orig_objects = [obj for obj in testrepo.odb] + packed_objects = [obj for obj in pack_repo.odb] + assert len(packed_objects) == len(orig_objects) + + # assert that the objects in the packed repo are the same objects as the original repo + for i, obj in enumerate(orig_objects): + assert pack_repo[obj].type == testrepo[obj].type + assert pack_repo[obj].read_raw() == testrepo[obj].read_raw() From 127c2c9ef6e0d5bcd789f644e8854766d9f384df Mon Sep 17 00:00:00 2001 From: Jeremy Date: Mon, 16 Nov 2020 15:44:13 +0200 Subject: [PATCH 5/9] use the utils rmtree rather than shutil --- test/test_packbuilder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_packbuilder.py b/test/test_packbuilder.py index eaae1ccbb..01dc126e8 100644 --- a/test/test_packbuilder.py +++ b/test/test_packbuilder.py @@ -26,13 +26,13 @@ """Tests for Index files.""" import os -import shutil import pytest import pygit2 from pygit2 import PackBuilder from . import utils +from .utils import rmtree def test_create_packbuilder(testrepo): @@ -92,7 +92,7 @@ def confirm_same_repo_after_packing(testrepo, tmp_path, pack_delegate): pack_repo = setup_second_repo(tmp_path) objects_dir = os.path.join(pack_repo.path, 'objects') - shutil.rmtree(objects_dir) + rmtree(objects_dir) pack_path = os.path.join(pack_repo.path, 'objects', 'pack') os.makedirs(pack_path) From ecdafb822ee89857a4526971061fc9953f5e352b Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 17 Nov 2020 20:17:20 +0200 Subject: [PATCH 6/9] Allow None as a default value for the path, which will write to the default location --- pygit2/packbuilder.py | 5 +++-- pygit2/repository.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index de57e1f58..0e141573d 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -71,8 +71,9 @@ def add_recur(self, git_object): def set_max_threads(self, n_threads): return C.git_packbuilder_set_threads(self._packbuilder, n_threads) - def write(self, path): - err = C.git_packbuilder_write(self._packbuilder, to_bytes(path), 0, ffi.NULL, ffi.NULL) + def write(self, path=None): + path = ffi.NULL if path is None else to_bytes(path) + err = C.git_packbuilder_write(self._packbuilder, path, 0, ffi.NULL, ffi.NULL) check_error(err) @property diff --git a/pygit2/repository.py b/pygit2/repository.py index 771260b83..0a34bcb02 100644 --- a/pygit2/repository.py +++ b/pygit2/repository.py @@ -84,7 +84,7 @@ def write(self, *args, **kwargs): object.""" return self.odb.write(*args, **kwargs) - def pack(self, path, pack_delegate=None, max_threads=None): + def pack(self, path=None, pack_delegate=None, max_threads=None): """Pack the objects in the odb chosen by the pack_delegate function and write .pack and .idx files for them. @@ -93,7 +93,7 @@ def pack(self, path, pack_delegate=None, max_threads=None): Parameters: path - The path to which the .pack and .idx files should be written. + The path to which the .pack and .idx files should be written. None will write to the default location pack_delegate The method which will provide add the objects to the pack builder. Defaults to all objects. @@ -112,7 +112,7 @@ def pack_all_objects(pack_builder): if max_threads: builder.set_max_threads(max_threads) pack_delegate(builder) - builder.write(path) + builder.write(path=path) return builder.written_objects_count From 902b6bef02dcb42997873cd5c962a6a3b1c13afd Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 17 Nov 2020 20:20:19 +0200 Subject: [PATCH 7/9] Rename set_max_threads method to set_threads --- pygit2/packbuilder.py | 2 +- pygit2/repository.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index 0e141573d..5c5459062 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -68,7 +68,7 @@ def add_recur(self, git_object): err = C.git_packbuilder_insert_recur(self._packbuilder, oid, ffi.NULL) check_error(err) - def set_max_threads(self, n_threads): + def set_threads(self, n_threads): return C.git_packbuilder_set_threads(self._packbuilder, n_threads) def write(self, path=None): diff --git a/pygit2/repository.py b/pygit2/repository.py index 0a34bcb02..a7079b5d8 100644 --- a/pygit2/repository.py +++ b/pygit2/repository.py @@ -84,7 +84,7 @@ def write(self, *args, **kwargs): object.""" return self.odb.write(*args, **kwargs) - def pack(self, path=None, pack_delegate=None, max_threads=None): + def pack(self, path=None, pack_delegate=None, num_threads=None): """Pack the objects in the odb chosen by the pack_delegate function and write .pack and .idx files for them. @@ -93,13 +93,13 @@ def pack(self, path=None, pack_delegate=None, max_threads=None): Parameters: path - The path to which the .pack and .idx files should be written. None will write to the default location + The path to which the .pack and .idx files should be written. None will write to the default location. pack_delegate The method which will provide add the objects to the pack builder. Defaults to all objects. - max_threads - The maximum number of threads the PackBuilder will spawn. + num_threads + The number of threads the PackBuilder will spawn. If set to 0 libgit2 will autodetect the number of CPUs. """ def pack_all_objects(pack_builder): @@ -109,8 +109,8 @@ def pack_all_objects(pack_builder): pack_delegate = pack_delegate or pack_all_objects builder = PackBuilder(self) - if max_threads: - builder.set_max_threads(max_threads) + if num_threads is not None: + builder.set_threads(num_threads) pack_delegate(builder) builder.write(path=path) From 1b16f76f2751dddbe5ab31ac5641517a16319628 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 17 Nov 2020 20:28:23 +0200 Subject: [PATCH 8/9] update git_object to be the more correct oid --- pygit2/packbuilder.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pygit2/packbuilder.py b/pygit2/packbuilder.py index 5c5459062..b3f54e6be 100644 --- a/pygit2/packbuilder.py +++ b/pygit2/packbuilder.py @@ -53,19 +53,19 @@ def __len__(self): return C.git_packbuilder_object_count(self._packbuilder) @staticmethod - def convert_object_to_oid(git_object): - oid = ffi.new('git_oid *') - ffi.buffer(oid)[:] = git_object.raw[:] - return oid - - def add(self, git_object): - oid = self.convert_object_to_oid(git_object) - err = C.git_packbuilder_insert(self._packbuilder, oid, ffi.NULL) + def convert_object_to_oid(oid): + git_oid = ffi.new('git_oid *') + ffi.buffer(git_oid)[:] = oid.raw[:] + return git_oid + + def add(self, oid): + git_oid = self.convert_object_to_oid(oid) + err = C.git_packbuilder_insert(self._packbuilder, git_oid, ffi.NULL) check_error(err) - def add_recur(self, git_object): - oid = self.convert_object_to_oid(git_object) - err = C.git_packbuilder_insert_recur(self._packbuilder, oid, ffi.NULL) + def add_recur(self, oid): + git_oid = self.convert_object_to_oid(oid) + err = C.git_packbuilder_insert_recur(self._packbuilder, git_oid, ffi.NULL) check_error(err) def set_threads(self, n_threads): From 8b8a0b56282fc1e71f2055c57e103aa431919652 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Tue, 17 Nov 2020 20:31:05 +0200 Subject: [PATCH 9/9] rename argument for consistency --- pygit2/repository.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygit2/repository.py b/pygit2/repository.py index a7079b5d8..98dcf8f76 100644 --- a/pygit2/repository.py +++ b/pygit2/repository.py @@ -84,7 +84,7 @@ def write(self, *args, **kwargs): object.""" return self.odb.write(*args, **kwargs) - def pack(self, path=None, pack_delegate=None, num_threads=None): + def pack(self, path=None, pack_delegate=None, n_threads=None): """Pack the objects in the odb chosen by the pack_delegate function and write .pack and .idx files for them. @@ -98,7 +98,7 @@ def pack(self, path=None, pack_delegate=None, num_threads=None): pack_delegate The method which will provide add the objects to the pack builder. Defaults to all objects. - num_threads + n_threads The number of threads the PackBuilder will spawn. If set to 0 libgit2 will autodetect the number of CPUs. """ @@ -109,8 +109,8 @@ def pack_all_objects(pack_builder): pack_delegate = pack_delegate or pack_all_objects builder = PackBuilder(self) - if num_threads is not None: - builder.set_threads(num_threads) + if n_threads is not None: + builder.set_threads(n_threads) pack_delegate(builder) builder.write(path=path)