From e59653e76be7149e08be3af1e2e0a854bd90062f Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Wed, 5 May 2021 11:30:01 -0700 Subject: [PATCH 1/7] add assertion to check for initialization, use ray 1.2.0 register_serializer --- btrdb/utils/ray.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 8df9527..d6e00e0 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -22,7 +22,8 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ - ray.register_custom_serializer( + assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" + ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) def btrdb_serializer(_): From 2fbff728ad45daa4d539f1ec3797543d5bdbeaec Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Wed, 5 May 2021 11:34:19 -0700 Subject: [PATCH 2/7] is this check being run? --- btrdb/utils/ray.py | 1 + 1 file changed, 1 insertion(+) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index d6e00e0..9b1f611 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -22,6 +22,7 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ + print("check init", ray.is_initialized()) assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) From bbaa1696a6ef1566855db70327032f5540df5154 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Wed, 5 May 2021 11:41:17 -0700 Subject: [PATCH 3/7] remove print --- btrdb/utils/ray.py | 1 - 1 file changed, 1 deletion(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 9b1f611..d6e00e0 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -22,7 +22,6 @@ def register_serializer(conn_str=None, apikey=None, profile=None): found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. """ - print("check init", ray.is_initialized()) assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) From e2bb4f91b5581c299c90dde2a52b5e4853d760d0 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:20:49 -0700 Subject: [PATCH 4/7] trying out custom serialize with a flag on connect --- btrdb/__init__.py | 15 ++++++++++++++- btrdb/utils/ray.py | 10 ++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index a7019c7..b5485a5 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -21,6 +21,8 @@ from btrdb.version import get_version from btrdb.utils.credentials import credentials_by_profile, credentials from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME +from btrdb.util import register_serializer +from warnings import warn ########################################################################## ## Module Variables @@ -40,7 +42,7 @@ def _connect(endpoints=None, apikey=None): return BTrDB(Endpoint(Connection(endpoints, apikey=apikey).channel)) -def connect(conn_str=None, apikey=None, profile=None): +def connect(conn_str=None, apikey=None, profile=None, shareable=False): """ Connect to a BTrDB server. @@ -57,6 +59,12 @@ def connect(conn_str=None, apikey=None, profile=None): The name of a profile containing the required connection information as found in the user's predictive grid credentials file `~/.predictivegrid/credentials.yaml`. + shareable: bool, default=False + Whether or not the connection can be "shared" in a distributed setting such + as Ray workers. If set to True, the connection can be serialized and sent + to other workers so that data can be retrieved in parallel; **however**, this + is less secure because it is possible for other users of the Ray cluster to + use your API key to fetch data. Returns ------- @@ -68,6 +76,11 @@ def connect(conn_str=None, apikey=None, profile=None): if conn_str and profile: raise ValueError("Received both conn_str and profile arguments.") + # check shareable flag and register custom serializer if necessary + if shareable: + warn("a shareable connection is potentially insecure; other users of the same cluster may be able to access your API key") + register_serializer(conn_str=conn_str, apikey=apikey, profile=profile) + # use specific profile if requested if profile: return _connect(**credentials_by_profile(profile)) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index d6e00e0..0ab5f65 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -23,8 +23,14 @@ def register_serializer(conn_str=None, apikey=None, profile=None): `~/.predictivegrid/credentials.yaml`. """ assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" - ray.util.register_serializer( - BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + # TODO: check the version using the 'semver' package? + if ray.__version__ == "0.8.4": + ray.register_custom_serializer( + BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + elif ray.__version__ in ["1.2.0", "1.3.0"]: + # TODO: check different versions of ray? + ray.util.register_serializer( + BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) def btrdb_serializer(_): """ From 74f5623aaef6b4ba0eff8c661febec51a62f400f Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:23:17 -0700 Subject: [PATCH 5/7] fix import --- btrdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index b5485a5..95847e7 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -21,7 +21,7 @@ from btrdb.version import get_version from btrdb.utils.credentials import credentials_by_profile, credentials from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.util import register_serializer +from btrdb.utils import register_serializer from warnings import warn ########################################################################## From c122166a775d8670c63884435c3be267735c283b Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:24:49 -0700 Subject: [PATCH 6/7] fix import --- btrdb/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/btrdb/__init__.py b/btrdb/__init__.py index 95847e7..cd14662 100644 --- a/btrdb/__init__.py +++ b/btrdb/__init__.py @@ -20,8 +20,8 @@ from btrdb.exceptions import ConnectionError from btrdb.version import get_version from btrdb.utils.credentials import credentials_by_profile, credentials +from btrdb.utils.ray import register_serializer from btrdb.stream import MINIMUM_TIME, MAXIMUM_TIME -from btrdb.utils import register_serializer from warnings import warn ########################################################################## From cd27a0b5b217fdc2fa0a3496de3271823599da37 Mon Sep 17 00:00:00 2001 From: Gabe Fierro Date: Thu, 20 May 2021 12:29:01 -0700 Subject: [PATCH 7/7] try semver --- btrdb/utils/ray.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/btrdb/utils/ray.py b/btrdb/utils/ray.py index 0ab5f65..64b2990 100644 --- a/btrdb/utils/ray.py +++ b/btrdb/utils/ray.py @@ -2,6 +2,8 @@ import ray +import semver + import btrdb from btrdb.conn import BTrDB @@ -24,13 +26,16 @@ def register_serializer(conn_str=None, apikey=None, profile=None): """ assert ray.is_initialized(), "Need to call ray.init() before registering custom serializer" # TODO: check the version using the 'semver' package? - if ray.__version__ == "0.8.4": + ver = semver.VersionInfo.parse(ray.__version__) + if ver.major == 0: ray.register_custom_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) - elif ray.__version__ in ["1.2.0", "1.3.0"]: + elif ver.major == 1 and ver.minor in range(2, 4): # TODO: check different versions of ray? ray.util.register_serializer( BTrDB, serializer=btrdb_serializer, deserializer=partial(btrdb_deserializer, conn_str=conn_str, apikey=apikey, profile=profile)) + else: + raise Exception("Ray version %s does not have custom serialization. Please upgrade to >= 1.2.0" % ray.__version__) def btrdb_serializer(_): """