Add modin format; fix for faiss ivf_pq compatibility

Author: Alexsandruss

envs/conda-env-sklearn.yml

@@ -9,8 +9,8 @@ dependencies:
   - catboost
   - lightgbm
   - faiss-cpu
-  - intel::scikit-learn-intelex
-  - intel::daal4py
+  - scikit-learn-intelex
+  - modin-all
   # sklbench dependencies
   - scikit-learn
   - pandas

sklbench/datasets/transformer.py

@@ -40,6 +40,12 @@ def convert_data(data, dformat: str, order: str, dtype: str):
         if data.ndim == 1:
             return pd.Series(data)
         return pd.DataFrame(data)
+    elif dformat == "modin":
+        import modin.pandas as modin_pd
+
+        if data.ndim == 1:
+            return modin_pd.Series(data)
+        return modin_pd.DataFrame(data)
     elif dformat == "cudf":
         import cudf
 

sklbench/emulators/common/neighbors.py

@@ -42,7 +42,7 @@ def get_m_subvectors(self, percentile, d):
         """Method to get `m_subvectors` closest to specific percentile and
         compatible with RAFT and FAISS"""
         raft_comp = np.arange(1, d // 16) * 16
-        faiss_comp = np.arange(1, d)
+        faiss_comp = np.array([1, 2, 3, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48])
         faiss_comp = faiss_comp[d % faiss_comp == 0]
         intersection = np.intersect1d(raft_comp, faiss_comp)
         if len(intersection) == 0: