Problem sampling with cores >1

[fccoelho]

Description of your problem

I am getting this exception whenever I try to sample with more than one core.

Please provide a minimal, self-contained, and reproducible example.
I am trying to run the code from this example.

Please provide the full traceback.

ConnectionResetError                      Traceback (most recent call last)
<ipython-input-4-b9d2bf1ba15b> in <module>()
     11     Y = pm.Lognormal('Y', mu=pm.math.log(sir_curves), sigma=sigma, observed=yobs)
     12 
---> 13     trace = pm.sample(2000, tune=1000, target_accept=0.9, cores=8)
     14     data = az.from_pymc3(trace=trace)

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/site-packages/pymc3/sampling.py in sample(draws, step, init, n_init, start, trace, chain_idx, chains, cores, tune, progressbar, model, random_seed, discard_tuned_samples, compute_convergence_checks, callback, return_inferencedata, idata_kwargs, mp_ctx, pickle_backend, **kwargs)
    543         _print_step_hierarchy(step)
    544         try:
--> 545             trace = _mp_sample(**sample_args, **parallel_args)
    546         except pickle.PickleError:
    547             _log.warning("Could not pickle model, sampling singlethreaded.")

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/site-packages/pymc3/sampling.py in _mp_sample(draws, tune, step, chains, cores, chain, random_seed, start, progressbar, trace, model, callback, discard_tuned_samples, mp_ctx, pickle_backend, **kwargs)
   1484         try:
   1485             with sampler:
-> 1486                 for draw in sampler:
   1487                     trace = traces[draw.chain - chain]
   1488                     if trace.supports_sampler_stats and draw.stats is not None:

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/site-packages/pymc3/parallel_sampling.py in __iter__(self)
    490 
    491         while self._active:
--> 492             draw = ProcessAdapter.recv_draw(self._active)
    493             proc, is_last, draw, tuning, stats, warns = draw
    494             self._total_draws += 1

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/site-packages/pymc3/parallel_sampling.py in recv_draw(processes, timeout)
    355         idxs = {id(proc._msg_pipe): proc for proc in processes}
    356         proc = idxs[id(ready[0])]
--> 357         msg = ready[0].recv()
    358 
    359         if msg[0] == "error":

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/multiprocessing/connection.py in recv(self)
    248         self._check_closed()
    249         self._check_readable()
--> 250         buf = self._recv_bytes()
    251         return _ForkingPickler.loads(buf.getbuffer())
    252 

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/multiprocessing/connection.py in _recv_bytes(self, maxsize)
    405 
    406     def _recv_bytes(self, maxsize=None):
--> 407         buf = self._recv(4)
    408         size, = struct.unpack("!i", buf.getvalue())
    409         if maxsize is not None and size > maxsize:

/home/fccoelho/Downloads/SageMath/local/lib/python3.7/multiprocessing/connection.py in _recv(self, size, read)
    377         remaining = size
    378         while remaining > 0:
--> 379             chunk = read(handle, remaining)
    380             n = len(chunk)
    381             if n == 0:

ConnectionResetError: [Errno 104] Connection reset by peer

Please provide any additional information below.

Versions and main components

• PyMC3 Version: 3.9.3
• Theano Version: 1.0.5
• Python Version: 3.7
• Operating system: Ubuntu
• How did you install PyMC3: pip

[fonnesbeck]

I get this error when running the survival_analysis.ipynb example on Linux.

---------------------------------------------------------------------------
ConnectionResetError                      Traceback (most recent call last)
Input In [15], in <module>
      2 n_tune = 1000
      4 with model:
----> 5     idata = pm.sample(
      6         n_samples,
      7         tune=n_tune,
      8         target_accept=0.95,
      9         chains=4
     10     )

File ~/pymc/pymc/sampling.py:568, in sample(draws, step, init, n_init, initvals, trace, chain_idx, chains, cores, tune, progressbar, model, random_seed, discard_tuned_samples, compute_convergence_checks, callback, jitter_max_retries, return_inferencedata, idata_kwargs, mp_ctx, **kwargs)
    566 _print_step_hierarchy(step)
    567 try:
--> 568     trace = _mp_sample(**sample_args, **parallel_args)
    569 except pickle.PickleError:
    570     _log.warning("Could not pickle model, sampling singlethreaded.")

File ~/pymc/pymc/sampling.py:1485, in _mp_sample(draws, tune, step, chains, cores, chain, random_seed, start, progressbar, trace, model, callback, discard_tuned_samples, mp_ctx, **kwargs)
   1483 try:
   1484     with sampler:
-> 1485         for draw in sampler:
   1486             trace = traces[draw.chain - chain]
   1487             if trace.supports_sampler_stats and draw.stats is not None:

File ~/pymc/pymc/parallel_sampling.py:460, in ParallelSampler.__iter__(self)
    457     self._progress.update(self._total_draws)
    459 while self._active:
--> 460     draw = ProcessAdapter.recv_draw(self._active)
    461     proc, is_last, draw, tuning, stats, warns = draw
    462     self._total_draws += 1

File ~/pymc/pymc/parallel_sampling.py:341, in ProcessAdapter.recv_draw(processes, timeout)
    339 idxs = {id(proc._msg_pipe): proc for proc in processes}
    340 proc = idxs[id(ready[0])]
--> 341 msg = ready[0].recv()
    343 if msg[0] == "error":
    344     warns, old_error = msg[1:]

File ~/miniforge3/envs/pymc-dev-py39/lib/python3.9/multiprocessing/connection.py:255, in _ConnectionBase.recv(self)
    253 self._check_closed()
    254 self._check_readable()
--> 255 buf = self._recv_bytes()
    256 return _ForkingPickler.loads(buf.getbuffer())

File ~/miniforge3/envs/pymc-dev-py39/lib/python3.9/multiprocessing/connection.py:419, in Connection._recv_bytes(self, maxsize)
    418 def _recv_bytes(self, maxsize=None):
--> 419     buf = self._recv(4)
    420     size, = struct.unpack("!i", buf.getvalue())
    421     if size == -1:

File ~/miniforge3/envs/pymc-dev-py39/lib/python3.9/multiprocessing/connection.py:384, in Connection._recv(self, size, read)
    382 remaining = size
    383 while remaining > 0:
--> 384     chunk = read(handle, remaining)
    385     n = len(chunk)
    386     if n == 0:

ConnectionResetError: [Errno 104] Connection reset by peer

Interestingly, it does not happen with all models; not sure what distinguishes them from those that work.

This is running PyMC 4.0.0b2 on Aesara 2.3.6

[fonnesbeck]

Still occurs with Aesara 2.3.8

[kpsanjeet]

I have the same issue.

[ricardoV94]

Can someone still reproduce this?

[fonnesbeck]

I can no longer replicate this on Linux. Closing.