Description
Combining GitPython with multiprocessing seems to have some issues:
Traceback (most recent call last):
File "/home//pycharm-2017.1.4/helpers/pydev/pydevd.py", line 1596, in
globals = debugger.run(setup['file'], None, None, is_module)
File "/home//pycharm-2017.1.4/helpers/pydev/pydevd.py", line 1023, in run
pydev_imports.execfile(file, globals, locals) # execute the script
File "/home//pycharm-2017.1.4/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
exec(compile(contents+"\n", file, 'exec'), glob, loc)
File "/home//git/git_repo_analysis/stash/get_repos.py", line 45, in
commits += q.get()
File "/usr/lib/python3.5/multiprocessing/queues.py", line 113, in get
return ForkingPickler.loads(res)
TypeError: init() missing 1 required positional argument: 'secs_west_of_utc'
In order to reproduce you need some git repositories and this code snippet:
import subprocess
from multiprocessing import Pool, Queue
from git import Repo
def f_init(q):
parse_repo.q = q
def parse_repo(repository_directory):
commits = []
repo = Repo(repository_directory)
if not repo.branches:
return
for commit in repo.iter_commits():
commits.append({"Timestamp": commit.authored_datetime,
"CommitHash": commit.hexsha})
return parse_repo.q.put(commits)
process = subprocess.run("find repos -name .git -type d -exec dirname {} \;", shell=True, stdout=subprocess.PIPE)
repository_directories = list(filter(None, str(process.stdout, "utf-8").split("\n")))
q = Queue()
p = Pool(None, f_init, [q])
results = p.map(parse_repo, repository_directories[:5])
p.close()
commits = []
for i in range(len(repository_directories)):
commits += q.get()Running the same without multiprocessing in a simple for-loop works fine.
I am using GitPython via debian package (python3-git 2.1.5-1) on Linux 4.9.0-1-amd64 SMP Debian 4.9.6-3 (2017-01-28) x86_64 GNU/Linux