Skip to content

Commit 4074f25

Browse files
AvihaiSamjaidisido
andauthored
feat: modify_refresh_interval flag in opensearch index_documents (#2980)
* feature/opensearch: modify_refresh_interval flag * CR change: rename modify_refresh_interval to enable_refresh_interval --------- Co-authored-by: jaidisido <jaidisido@gmail.com>
1 parent cc77561 commit 4074f25

File tree

1 file changed

+6
-2
lines changed

1 file changed

+6
-2
lines changed

awswrangler/opensearch/_write.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -504,6 +504,7 @@ def index_documents(
504504
initial_backoff: int | None = None,
505505
max_backoff: int | None = None,
506506
use_threads: bool | int = False,
507+
enable_refresh_interval: bool = True,
507508
**kwargs: Any,
508509
) -> dict[str, Any]:
509510
"""
@@ -559,6 +560,8 @@ def index_documents(
559560
True to enable concurrent requests, False to disable multiple threads.
560561
If enabled os.cpu_count() will be used as the max number of threads.
561562
If integer is provided, specified number is used.
563+
enable_refresh_interval
564+
True (default) to enable ``refresh_interval`` modification to ``-1`` (disabled) while indexing documents
562565
**kwargs
563566
KEYWORD arguments forwarded to bulk operation
564567
elasticsearch >= 7.10.2 / opensearch: \
@@ -614,7 +617,7 @@ def index_documents(
614617
widgets=widgets, max_value=total_documents, prefix="Indexing: "
615618
).start()
616619
for i, bulk_chunk_documents in enumerate(actions):
617-
if i == 1: # second bulk iteration, in case the index didn't exist before
620+
if i == 1 and enable_refresh_interval: # second bulk iteration, in case the index didn't exist before
618621
refresh_interval = _get_refresh_interval(client, index)
619622
_disable_refresh_interval(client, index)
620623
_logger.debug("running bulk index of %s documents", len(bulk_chunk_documents))
@@ -655,6 +658,7 @@ def index_documents(
655658
raise e
656659

657660
finally:
658-
_set_refresh_interval(client, index, refresh_interval)
661+
if enable_refresh_interval:
662+
_set_refresh_interval(client, index, refresh_interval)
659663

660664
return {"success": success, "errors": errors}

0 commit comments

Comments
 (0)