From f49f6b4f34ddd3a2313e1df00c487bd7f47df845 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 24 Oct 2023 14:54:42 +0200 Subject: [PATCH 01/36] new file for lr sinkhorn --- ot/lowrank.py | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 ot/lowrank.py diff --git a/ot/lowrank.py b/ot/lowrank.py new file mode 100644 index 000000000..ba46cd1ed --- /dev/null +++ b/ot/lowrank.py @@ -0,0 +1,171 @@ +################################################################################################################# +############################################## WORK IN PROGRESS ################################################# +################################################################################################################# + + +from ot.utils import unif, list_to_array +from ot.backend import get_backend +from ot.datasets import make_1D_gauss as gauss + + + +################################## LR-DYSKTRA ALGORITHM ########################################## + +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_w): + """ + Implementation of the Dykstra algorithm for low rank Sinkhorn + """ + + # get dykstra parameters + q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_w + + # POT backend + eps1, eps2, eps3, p1, p2 = list_to_array(eps1, eps2, eps3, p1, p2) + q3_1, q3_2, v1_, v2_, q1, q2 = list_to_array(q3_1, q3_2, v1_, v2_, q1, q2) + + nx = get_backend(eps1, eps2, eps3, p1, p2, q3_1, q3_2, v1_, v2_, q1, q2) + + # ------- Dykstra algorithm ------ + g_ = eps3 + + u1 = p1 / nx.dot(eps1, v1_) + u2 = p2 / nx.dot(eps2, v2_) + + g = nx.maximum(alpha, g_ * q3_1) + q3_1 = (g_ * q3_1) / g + g_ = g + + prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) + prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) + g = (g_ * q3_2 * prod1 * prod2)**(1/3) + + v1 = g / nx.dot(eps1.T,u1) + v2 = g / nx.dot(eps2.T,u2) + + q1 = (v1_ * q1) / v1 + q2 = (v2_ * q2) / v2 + q3_2 = (g_ * q3_2) / g + + v1_, v2_ = v1, v2 + g_ = g + + # Compute error + err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) + err2 = nx.sum(nx.abs(u2 * (eps2 @ v2) - p2)) + err = err1 + err2 + + # Compute low rank matrices Q, R + Q = u1[:,None] * eps1 * v1[None,:] + R = u2[:,None] * eps2 * v2[None,:] + + dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + + return Q, R, g, err, dykstra_w + + + +#################################### LOW RANK SINKHORN ALGORITHM ######################################### + + +def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): + r''' + Solve the entropic regularization optimal transport problem under low-nonnegative low rank constraints + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + reg : float + Regularization term >0 + a : array-like, shape (n_samples_a,) + samples weights in the source domain + b : array-like, shape (n_samples_b,) + samples weights in the target domain + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshold on error (>0) + + Returns + ------- + Q : array-like, shape (n_samples_a, r) + First low-rank matrix decomposition of the OT plan + R: array-like, shape (n_samples_b, r) + Second low-rank matrix decomposition of the OT plan + g : array-like, shape (r, ) + ... + + ''' + + X_s, X_t = list_to_array(X_s, X_t) + nx = get_backend(X_s, X_t) + + ns, nt = X_s.shape[0], X_t.shape[0] + if a is None: + a = nx.from_numpy(unif(ns), type_as=X_s) + if b is None: + b = nx.from_numpy(unif(nt), type_as=X_s) + + M = ot.dist(X_s,X_t, metric=metric) + + # Compute rank + r = min(ns, nt, r) + + # Compute gamma + L = nx.sqrt((2/(alpha**4))*nx.norm(M)**2 + (reg + (2/(alpha**3))*nx.norm(M))**2) + gamma = 1/(2*L) + + # Initialisation + Q, R, g = nx.ones((ns,r)), nx.ones((nt,r)), nx.ones(r) + q3_1, q3_2 = nx.ones(r), nx.ones(r) + v1_, v2_ = nx.ones(r), nx.ones(r) + q1, q2 = nx.ones(r), nx.ones(r) + dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + n_iter = 0 + err = 1 + + while n_iter < numIterMax: + if err > stopThr: + n_iter = n_iter + 1 + + CR = nx.dot(M,R) + C_t_Q = nx.dot(M.T,Q) + diag_g = (1/g)[:,None] + + eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) + eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) + omega = nx.diag(nx.dot(Q.T, CR)) + eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) + + Q, R, g, err, dykstra_w = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_w) + else: + break + + return Q, R, g + + + + + +############################################################################ +## Test with X_s, X_t from ot.datasets +############################################################################# + +import numpy as np +import ot + +Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) +Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) + + +Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) +M = ot.dist(Xs,Xt) +P = np.dot(Q,np.dot(np.diag(1/g),R.T)) + +print(np.sum(P)) + + + + From 3c4b50fdb660f27cc080618edb664d17086d93a9 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 24 Oct 2023 16:47:21 +0200 Subject: [PATCH 02/36] lr sinkhorn, solve_sample, OTResultLazy --- ot/lowrank.py | 40 +++++++------ ot/solvers.py | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++ ot/utils.py | 90 ++++++++++++++++++++++++++++ 3 files changed, 272 insertions(+), 19 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index ba46cd1ed..a1c73bdf3 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -2,8 +2,10 @@ ############################################## WORK IN PROGRESS ################################################# ################################################################################################################# +## Implementation of the LR-Dykstra algorithm and low rank sinkhorn algorithms -from ot.utils import unif, list_to_array + +from ot.utils import unif, list_to_array, dist from ot.backend import get_backend from ot.datasets import make_1D_gauss as gauss @@ -11,13 +13,13 @@ ################################## LR-DYSKTRA ALGORITHM ########################################## -def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_w): +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): """ Implementation of the Dykstra algorithm for low rank Sinkhorn """ # get dykstra parameters - q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_w + q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_p # POT backend eps1, eps2, eps3, p1, p2 = list_to_array(eps1, eps2, eps3, p1, p2) @@ -58,18 +60,18 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_w): Q = u1[:,None] * eps1 * v1[None,:] R = u2[:,None] * eps2 * v2[None,:] - dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] - return Q, R, g, err, dykstra_w + return Q, R, g, err, dykstra_p #################################### LOW RANK SINKHORN ALGORITHM ######################################### -def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): +def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): r''' - Solve the entropic regularization optimal transport problem under low-nonnegative low rank constraints + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints on the feasible couplings. Parameters ---------- @@ -95,7 +97,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) - ... + Third low-rank matrix decomposition of the OT plan ''' @@ -108,7 +110,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', if b is None: b = nx.from_numpy(unif(nt), type_as=X_s) - M = ot.dist(X_s,X_t, metric=metric) + M = dist(X_s,X_t, metric=metric) # Compute rank r = min(ns, nt, r) @@ -122,7 +124,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', q3_1, q3_2 = nx.ones(r), nx.ones(r) v1_, v2_ = nx.ones(r), nx.ones(r) q1, q2 = nx.ones(r), nx.ones(r) - dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] n_iter = 0 err = 1 @@ -139,7 +141,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', omega = nx.diag(nx.dot(Q.T, CR)) eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - Q, R, g, err, dykstra_w = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_w) + Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) else: break @@ -153,18 +155,18 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', ## Test with X_s, X_t from ot.datasets ############################################################################# -import numpy as np -import ot +# import numpy as np +# import ot -Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) -Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) +# Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) +# Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) -Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) -M = ot.dist(Xs,Xt) -P = np.dot(Q,np.dot(np.diag(1/g),R.T)) +# Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) +# M = ot.dist(Xs,Xt) +# P = np.dot(Q,np.dot(np.diag(1/g),R.T)) -print(np.sum(P)) +# print(np.sum(P)) diff --git a/ot/solvers.py b/ot/solvers.py index 0313cf588..9c2746c25 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -848,3 +848,164 @@ def solve_gromov(Ca, Cb, M=None, a=None, b=None, loss='L2', symmetric=None, value_linear=value_linear, value_quad=value_quad, plan=plan, status=status, backend=nx) return res + + + + + + +################################## WORK IN PROGRESS ##################################### + +## Implementation of the ot.solve_sample function +## Function isn't complete, still work in progress for reg == 0 / reg is None case (and unbalanced cases) + + +from .utils import unif, list_to_array, dist, OTResultLazy +from .bregman import empirical_sinkhorn + + +def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_type="KL", unbalanced=None, + unbalanced_type='KL', is_Lazy=False, batch_size=None, n_threads=1, max_iter=None, plan_init=None, + potentials_init=None, tol=None, verbose=False): + + r"""Solve the discrete optimal transport problem using the samples in the source and target domains. + It returns either a :any:`OTResult` or :any:`OTResultLazy` object. + + The function solves the following general optimal transport problem + + .. math:: + \min_{\mathbf{T}\geq 0} \quad \sum_{i,j} T_{i,j}M_{i,j} + \lambda_r R(\mathbf{T}) + + \lambda_u U(\mathbf{T}\mathbf{1},\mathbf{a}) + + \lambda_u U(\mathbf{T}^T\mathbf{1},\mathbf{b}) + + The regularization is selected with `reg` (:math:`\lambda_r`) and `reg_type`. By + default ``reg=None`` and there is no regularization. The unbalanced marginal + penalization can be selected with `unbalanced` (:math:`\lambda_u`) and + `unbalanced_type`. By default ``unbalanced=None`` and the function + solves the exact optimal transport problem (respecting the marginals). + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + a : array-like, shape (dim_a,), optional + Samples weights in the source domain (default is uniform) + b : array-like, shape (dim_b,), optional + Samples weights in the source domain (default is uniform) + reg : float, optional + Regularization weight :math:`\lambda_r`, by default None (no reg., exact + OT) + reg_type : str, optional + Type of regularization :math:`R` either "KL", "L2", "entropy", by default "KL" + unbalanced : float, optional + Unbalanced penalization weight :math:`\lambda_u`, by default None + (balanced OT) + unbalanced_type : str, optional + Type of unbalanced penalization function :math:`U` either "KL", "L2", "TV", by default "KL" + is_Lazy : bool, optional + Return :any:`OTResultlazy` object to reduce memory cost when True, by default False + n_threads : int, optional + Number of OMP threads for exact OT solver, by default 1 + max_iter : int, optional + Maximum number of iteration, by default None (default values in each solvers) + plan_init : array_like, shape (dim_a, dim_b), optional + Initialization of the OT plan for iterative methods, by default None + potentials_init : (array_like(dim_a,),array_like(dim_b,)), optional + Initialization of the OT dual potentials for iterative methods, by default None + tol : _type_, optional + Tolerance for solution precision, by default None (default values in each solvers) + verbose : bool, optional + Print information in the solver, by default False + + Returns + ------- + + res_lazy : OTResultLazy() + Result of the optimization problem. This class only returns a partial OT plan and the OT dual potentials to reduce memory costs. + The information can be obtained as follows: + + - res.lazy_plan : OT plan computed on a subsample of X_s and X_t :math:`\mathbf{T}` + - res.potentials : OT dual potentials + + See :any:`OTResultLazy` for more information. + + res : OTResult() + Result of the optimization problem. The information can be obtained as follows: + + - res.plan : OT plan :math:`\mathbf{T}` + - res.potentials : OT dual potentials + - res.value : Optimal value of the optimization problem + - res.value_linear : Linear OT loss with the optimal OT plan + + See :any:`OTResult` for more information. + + + """ + + X_s, X_t = list_to_array(X_s,X_t) + + # detect backend + arr = [X_s,X_t] + if a is not None: + arr.append(a) + if b is not None: + arr.append(b) + nx = get_backend(*arr) + + # create uniform weights if not given + ns, nt = X_s.shape[0], X_t.shape[0] + if a is None: + a = nx.from_numpy(unif(ns), type_as=X_s) + if b is None: + b = nx.from_numpy(unif(nt), type_as=X_s) + + # default values for solutions + potentials = None + lazy_plan = None + + if max_iter is None: + max_iter = 1000 + if tol is None: + tol = 1e-9 + if batch_size is None: + batch_size = 100 + + if is_Lazy: + ################# WIP #################### + if reg is None or reg == 0: # EMD solver for isLazy ? + if unbalanced is None: # not sure "unbalanced" parameter is needed here ? (since we won't compute value) + pass + elif unbalanced_type.lower() in ['kl', 'l2']: + pass + elif unbalanced_type.lower() == 'tv': + pass + pass + ############################################# + + else: + # compute potentials + u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, + isLazy=True, batchSize=batch_size, verbose=verbose, log=True) + potentials = (log["u"], log["v"]) + + # compute lazy_plan + ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) + M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) + K = nx.exp(M / (-reg)) + lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) + + res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) + return res_lazy + + else: + # compute cost matrix M and use solve function + M = dist(X_s, X_t, metric) + + res = solve(M, a, b, reg, reg_type, unbalanced, unbalanced_type, n_threads, max_iter, plan_init, potentials_init, tol, verbose) + return res + + + + diff --git a/ot/utils.py b/ot/utils.py index 8cbb0db25..d570b9f30 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -938,3 +938,93 @@ def citation(self): url = {http://jmlr.org/papers/v22/20-451.html} } """ + + + +############################## WORK IN PROGRESS #################################### + +## Implementation of the OTResultLazy class for ot.solve_sample() with potentials and lazy_plan as attributes + +class OTResultLazy: + def __init__(self, potentials=None, lazy_plan=None, backend=None): + + self._potentials = potentials + self._lazy_plan = lazy_plan + self._backend = backend if backend is not None else NumpyBackend() + + + # Dual potentials -------------------------------------------- + + def __repr__(self): + s = 'OTResultLazy(' + if self._lazy_plan is not None: + s += 'lazy_plan={}(shape={}),'.format(self._lazy_plan.__class__.__name__, self._lazy_plan.shape) + + if s[-1] != '(': + s = s[:-1] + ')' + else: + s = s + ')' + return s + + @property + def potentials(self): + """Dual potentials, i.e. Lagrange multipliers for the marginal constraints. + + This pair of arrays has the same shape, numerical type + and properties as the input weights "a" and "b". + """ + if self._potentials is not None: + return self._potentials + else: + raise NotImplementedError() + + @property + def potential_a(self): + """First dual potential, associated to the "source" measure "a".""" + if self._potentials is not None: + return self._potentials[0] + else: + raise NotImplementedError() + + @property + def potential_b(self): + """Second dual potential, associated to the "target" measure "b".""" + if self._potentials is not None: + return self._potentials[1] + else: + raise NotImplementedError() + + # Transport plan ------------------------------------------- + @property + def lazy_plan(self): + """A subset of the Transport plan, encoded as a dense array.""" + + if self._lazy_plan is not None: + return self._lazy_plan + else: + raise NotImplementedError() + + @property + def citation(self): + """Appropriate citation(s) for this result, in plain text and BibTex formats.""" + + # The string below refers to the POT library: + # successor methods may concatenate the relevant references + # to the original definitions, solvers and underlying numerical backends. + return """POT library: + + POT Python Optimal Transport library, Journal of Machine Learning Research, 22(78):1−8, 2021. + Website: https://pythonot.github.io/ + Rémi Flamary, Nicolas Courty, Alexandre Gramfort, Mokhtar Z. Alaya, Aurélie Boisbunon, Stanislas Chambon, Laetitia Chapel, Adrien Corenflos, Kilian Fatras, Nemo Fournier, Léo Gautheron, Nathalie T.H. Gayraud, Hicham Janati, Alain Rakotomamonjy, Ievgen Redko, Antoine Rolet, Antony Schutz, Vivien Seguy, Danica J. Sutherland, Romain Tavenard, Alexander Tong, Titouan Vayer; + + @article{flamary2021pot, + author = {R{\'e}mi Flamary and Nicolas Courty and Alexandre Gramfort and Mokhtar Z. Alaya and Aur{\'e}lie Boisbunon and Stanislas Chambon and Laetitia Chapel and Adrien Corenflos and Kilian Fatras and Nemo Fournier and L{\'e}o Gautheron and Nathalie T.H. Gayraud and Hicham Janati and Alain Rakotomamonjy and Ievgen Redko and Antoine Rolet and Antony Schutz and Vivien Seguy and Danica J. Sutherland and Romain Tavenard and Alexander Tong and Titouan Vayer}, + title = {{POT}: {Python} {Optimal} {Transport}}, + journal = {Journal of Machine Learning Research}, + year = {2021}, + volume = {22}, + number = {78}, + pages = {1-8}, + url = {http://jmlr.org/papers/v22/20-451.html} + } + """ \ No newline at end of file From 3034e575c55d2ce56499be6849e1906fe52f0573 Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 25 Oct 2023 17:39:08 +0200 Subject: [PATCH 03/36] add test functions + small modif lr_sin/solve_sample --- ot/lowrank.py | 97 ++++++++++++++++++++++++++++------------- ot/solvers.py | 47 +++++++++++--------- test/test_lowrank.py | 84 ++++++++++++++++++++++++++++++++++++ test/test_solvers.py | 100 +++++++++++++++++++++++++++++++++++++++++++ test/test_utils.py | 27 ++++++++++++ 5 files changed, 304 insertions(+), 51 deletions(-) create mode 100644 test/test_lowrank.py diff --git a/ot/lowrank.py b/ot/lowrank.py index a1c73bdf3..22ff8b754 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -4,10 +4,9 @@ ## Implementation of the LR-Dykstra algorithm and low rank sinkhorn algorithms - -from ot.utils import unif, list_to_array, dist -from ot.backend import get_backend -from ot.datasets import make_1D_gauss as gauss +import warnings +from .utils import unif, list_to_array, dist +from .backend import get_backend @@ -15,7 +14,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): """ - Implementation of the Dykstra algorithm for low rank Sinkhorn + Implementation of the Dykstra algorithm for low rank sinkhorn """ # get dykstra parameters @@ -69,9 +68,12 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): #################################### LOW RANK SINKHORN ALGORITHM ######################################### -def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, metric='sqeuclidean', alpha="auto", + numItermax=10000, stopThr=1e-9, warn=True, verbose=False): r''' - Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints on the feasible couplings. + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. + + This function returns the two low-rank matrix decomposition of the OT plan (Q,R), as well as the weight vector g. Parameters ---------- @@ -79,17 +81,22 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', samples in the source domain X_t : array-like, shape (n_samples_b, dim) samples in the target domain - reg : float - Regularization term >0 a : array-like, shape (n_samples_a,) samples weights in the source domain b : array-like, shape (n_samples_b,) samples weights in the target domain + reg : float, optional + Regularization term >0 + rank: int, optional + Nonnegative rank of the OT plan + alpha: int, optional + Lower bound for the weight vector g (>0 and <1/r) numItermax : int, optional Max number of iterations stopThr : float, optional Stop threshold on error (>0) + Returns ------- Q : array-like, shape (n_samples_a, r) @@ -97,7 +104,14 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) - Third low-rank matrix decomposition of the OT plan + Weight vector for the low-rank decomposition of the OT plan + + + References + ---------- + + .. Scetbon, M., Cuturi, M., & Peyré, G (2021). + Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. ''' @@ -110,13 +124,22 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', if b is None: b = nx.from_numpy(unif(nt), type_as=X_s) + # Compute cost matrix M = dist(X_s,X_t, metric=metric) - + # Compute rank - r = min(ns, nt, r) + rank = min(ns, nt, rank) + r = rank + + if alpha == 'auto': + alpha = 1.0 / (r + 1) + + if (1/r < alpha) or (alpha < 0): + warnings.warn("The provided alpha value might lead to instabilities.") + # Compute gamma - L = nx.sqrt((2/(alpha**4))*nx.norm(M)**2 + (reg + (2/(alpha**3))*nx.norm(M))**2) + L = nx.sqrt((2/(alpha**4))*(nx.norm(M)**2) + (reg + (2/(alpha**3))*(nx.norm(M))**2)) gamma = 1/(2*L) # Initialisation @@ -125,25 +148,34 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', v1_, v2_ = nx.ones(r), nx.ones(r) q1, q2 = nx.ones(r), nx.ones(r) dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] - n_iter = 0 err = 1 - while n_iter < numIterMax: - if err > stopThr: - n_iter = n_iter + 1 - - CR = nx.dot(M,R) - C_t_Q = nx.dot(M.T,Q) - diag_g = (1/g)[:,None] - - eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) - eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) - omega = nx.diag(nx.dot(Q.T, CR)) - eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - - Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) - else: + for ii in range(numItermax): + CR = nx.dot(M,R) + C_t_Q = nx.dot(M.T,Q) + diag_g = (1/g)[:,None] + + eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) + eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) + omega = nx.diag(nx.dot(Q.T, CR)) + eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) + + Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) + + if err < stopThr: break + + if verbose: + if ii % 200 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(ii, err)) + + else: + if warn: + warnings.warn("Sinkhorn did not converge. You might want to " + "increase the number of iterations `numItermax` " + "or the regularization parameter `reg`.") return Q, R, g @@ -161,8 +193,13 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', # Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) # Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) +# ns = Xs.shape[0] +# nt = Xt.shape[0] + +# a = unif(ns) +# b = unif(nt) -# Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) +# Q, R, g = lowrank_sinkhorn(Xs, Xt, reg=0.1, metric='euclidean', verbose=True, numItermax=100) # M = ot.dist(Xs,Xt) # P = np.dot(Q,np.dot(np.diag(1/g),R.T)) diff --git a/ot/solvers.py b/ot/solvers.py index 9c2746c25..c176969ca 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -926,7 +926,7 @@ def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_t Result of the optimization problem. This class only returns a partial OT plan and the OT dual potentials to reduce memory costs. The information can be obtained as follows: - - res.lazy_plan : OT plan computed on a subsample of X_s and X_t :math:`\mathbf{T}` + - res.lazy_plan : OT plan computed on a subsample of X_s and X_t - res.potentials : OT dual potentials See :any:`OTResultLazy` for more information. @@ -975,29 +975,34 @@ def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_t if is_Lazy: ################# WIP #################### if reg is None or reg == 0: # EMD solver for isLazy ? - if unbalanced is None: # not sure "unbalanced" parameter is needed here ? (since we won't compute value) - pass - elif unbalanced_type.lower() in ['kl', 'l2']: - pass - elif unbalanced_type.lower() == 'tv': - pass - pass + + if unbalanced is None: # balanced EMD solver for isLazy ? + raise (NotImplementedError('Not implemented balanced with no regularization')) + + else: + raise (NotImplementedError('Not implemented unbalanced_type="{}" with no regularization'.format(unbalanced_type))) + + ############################################# else: - # compute potentials - u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, - isLazy=True, batchSize=batch_size, verbose=verbose, log=True) - potentials = (log["u"], log["v"]) - - # compute lazy_plan - ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) - M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) - K = nx.exp(M / (-reg)) - lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) - - res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) - return res_lazy + if unbalanced is None: + u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, + isLazy=True, batchSize=batch_size, verbose=verbose, log=True) + # compute potentials + potentials = (log["u"], log["v"]) + + # compute lazy_plan + ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) + M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) + K = nx.exp(M / (-reg)) + lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) + + res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) + return res_lazy + + else: + raise (NotImplementedError('Not implemented unbalanced_type="{}" with regularization'.format(unbalanced_type))) else: # compute cost matrix M and use solve function diff --git a/test/test_lowrank.py b/test/test_lowrank.py new file mode 100644 index 000000000..6e1f24067 --- /dev/null +++ b/test/test_lowrank.py @@ -0,0 +1,84 @@ +##################################################################################################### +####################################### WORK IN PROGRESS ############################################ +##################################################################################################### + + +""" Test for low rank sinkhorn solvers """ + +import ot +import numpy as np +import pytest +from itertools import product + + +def test_LR_Dykstra(): + # test for LR_Dykstra algorithm ? catch nan values ? + pass + + +@pytest.mark.parametrize("verbose, warn", product([True, False], [True, False])) +def test_lowrank_sinkhorn(verbose, warn): + # test low rank sinkhorn + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + Q_sqe, R_sqe, g_sqe = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) + P_sqe = np.dot(Q_sqe,np.dot(np.diag(1/g_sqe),R_sqe.T)) + + Q_m, R_m, g_m = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1, metric='euclidean') + P_m = np.dot(Q_m,np.dot(np.diag(1/g_m),R_m.T)) + + # check constraints + np.testing.assert_allclose( + a, P_sqe.sum(1), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + b, P_sqe.sum(0), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + a, P_m.sum(1), atol=1e-05) # metric euclidian + np.testing.assert_allclose( + b, P_m.sum(0), atol=1e-05) # metric euclidian + + with pytest.warns(UserWarning): + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) + + + +@pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,4))) +def test_lowrank_sinkhorn_alpha_warning(alpha,rank): + # test warning for value of alpha + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + with pytest.warns(UserWarning): + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, r=rank, alpha=alpha, warn=False) + + + +def test_lowrank_sinkhorn_backends(nx): + # test low rank sinkhorn for different backends + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) + + Q, R, g = nx.to_numpy(ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, 0.1)) + P = np.dot(Q,np.dot(np.diag(1/g),R.T)) + + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + + + diff --git a/test/test_solvers.py b/test/test_solvers.py index f0f5b638f..5a05d54cf 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -255,3 +255,103 @@ def test_solve_gromov_not_implemented(nx): ot.solve_gromov(Ca, Cb, reg=1, unbalanced_type='partial', unbalanced=1.5) with pytest.raises(NotImplementedError): ot.solve_gromov(Ca, Cb, reg=1, unbalanced_type='partial', unbalanced=0.5, symmetric=False) + + + + +########################################################################################################### +############################################ WORK IN PROGRESS ############################################# +########################################################################################################### + +def assert_allclose_sol_sample(sol1, sol2): + # test attributes of OTResultLazy class + lst_attr = ['potentials','potential_a', 'potential_b', 'lazy_plan'] + + nx1 = sol1._backend if sol1._backend is not None else ot.backend.NumpyBackend() + nx2 = sol2._backend if sol2._backend is not None else ot.backend.NumpyBackend() + + for attr in lst_attr: + try: + np.allclose(nx1.to_numpy(getattr(sol1, attr)), nx2.to_numpy(getattr(sol2, attr))) + except NotImplementedError: + pass + + +@pytest.mark.parametrize("reg,reg_type,unbalanced,unbalanced_type", itertools.product(lst_reg, lst_reg_type, lst_unbalanced, lst_unbalanced_type)) +def test_solve_sample(nx): + # test solve_sample when is_Lazy = False + n = 100 + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + a = ot.utils.unif(X_s.shape[0]) + b = ot.utils.unif(X_t.shape[0]) + + # solve unif weights + sol0 = ot.solve_sample(X_s, X_t) + + # solve signe weights + sol = ot.solve_sample(X_s, X_t, a, b) + + # check some attributes + sol.potentials + sol.sparse_plan + sol.marginals + sol.status + + assert_allclose_sol(sol0, sol) + + # solve in backend + X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) + solb = ot.solve_sample(X_sb, X_tb, ab, bb) + + assert_allclose_sol(sol, solb) + + # test not implemented unbalanced and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, unbalanced=1, unbalanced_type='cryptic divergence') + + # test not implemented reg_type and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, reg=1, reg_type='cryptic divergence') + + + +def test_lazy_solve_sample(nx): + # test solve_sample when is_Lazy = True + n = 100 + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + a = ot.utils.unif(X_s.shape[0]) + b = ot.utils.unif(X_t.shape[0]) + + # solve unif weights + sol0 = ot.solve_sample(X_s, X_t, reg=0.1, is_Lazy=True) # reg != 0 or None since no implementation yet for is_Lazy=True + + # solve signe weights + sol = ot.solve_sample(X_s, X_t, a, b, reg=0.1, is_Lazy=True) + + # check some attributes + sol.potentials + sol.lazy_plan + + assert_allclose_sol_sample(sol0, sol) + + # solve in backend + X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) + solb = ot.solve_sample(X_sb, X_tb, ab, bb, reg=0.1, is_Lazy=True) + + assert_allclose_sol_sample(sol, solb) + + # test not implemented reg==0 (or None) + balanced and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, is_Lazy=True) # reg == 0 (or None) + unbalanced= None are default + + # test not implemented reg==0 (or None) + unbalanced_type and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, unbalanced_type="kl", is_Lazy=True) # reg == 0 (or None) is default + + # test not implemented reg != 0 + unbalanced_type and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, reg=0.1, unbalanced_type="kl", is_Lazy=True) \ No newline at end of file diff --git a/test/test_utils.py b/test/test_utils.py index 40324518e..a14be460e 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -401,3 +401,30 @@ def test_get_coordinate_circle(): x_p = ot.utils.get_coordinate_circle(x) np.testing.assert_allclose(u[0], x_p) + + + +############################################################################################## +##################################### WORK IN PROGRESS ####################################### +############################################################################################## + +# test function for OTResultLazy + +def test_OTResultLazy(): + + res_lazy = ot.utils.OTResultLazy() + + # test print + print(res_lazy) + + # tets get citation + print(res_lazy.citation) + + lst_attributes = ['lazy_plan', + 'potential_a', + 'potential_b', + 'potentials'] + + for at in lst_attributes: + with pytest.raises(NotImplementedError): + getattr(res_lazy, at) \ No newline at end of file From 085863aef96f0d19e740879dfae158a762275a67 Mon Sep 17 00:00:00 2001 From: laudavid Date: Thu, 26 Oct 2023 10:49:23 +0200 Subject: [PATCH 04/36] add import to __init__ --- ot/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ot/__init__.py b/ot/__init__.py index f16b6fcfc..cb00f4553 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -35,6 +35,7 @@ from . import factored from . import solvers from . import gaussian +from . import lowrank # OT functions from .lp import (emd, emd2, emd_1d, emd2_1d, wasserstein_1d, @@ -50,7 +51,8 @@ gromov_barycenters, fused_gromov_wasserstein, fused_gromov_wasserstein2) from .weak import weak_optimal_transport from .factored import factored_optimal_transport -from .solvers import solve, solve_gromov +from .solvers import solve, solve_gromov, solve_sample +from .lowrank import lowrank_sinkhorn # utils functions from .utils import dist, unif, tic, toc, toq From 9becafc305fd6b2cc5390b0de16bae015bd41121 Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 3 Nov 2023 11:38:40 +0100 Subject: [PATCH 05/36] modify low rank, remove solve_sample,OTResultLazy --- ot/__init__.py | 4 +- ot/lowrank.py | 200 ++++++++++++++++++++++++------------------- ot/solvers.py | 160 ---------------------------------- ot/utils.py | 89 ------------------- test/test_lowrank.py | 54 ++++++------ test/test_solvers.py | 97 --------------------- test/test_utils.py | 25 ------ 7 files changed, 142 insertions(+), 487 deletions(-) diff --git a/ot/__init__.py b/ot/__init__.py index cb00f4553..4aba450af 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -51,7 +51,7 @@ gromov_barycenters, fused_gromov_wasserstein, fused_gromov_wasserstein2) from .weak import weak_optimal_transport from .factored import factored_optimal_transport -from .solvers import solve, solve_gromov, solve_sample +from .solvers import solve, solve_gromov from .lowrank import lowrank_sinkhorn # utils functions @@ -70,4 +70,4 @@ 'factored_optimal_transport', 'solve', 'solve_gromov', 'smooth', 'stochastic', 'unbalanced', 'partial', 'regpath', 'solvers', 'binary_search_circle', 'wasserstein_circle', - 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif'] + 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif', 'lowrank_sinkhorn'] diff --git a/ot/lowrank.py b/ot/lowrank.py index 22ff8b754..b3fce8de0 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -1,78 +1,88 @@ +""" +Low rank OT solvers +""" + +# Author: Laurène David +# +# License: MIT License + + + ################################################################################################################# ############################################## WORK IN PROGRESS ################################################# ################################################################################################################# -## Implementation of the LR-Dykstra algorithm and low rank sinkhorn algorithms import warnings -from .utils import unif, list_to_array, dist -from .backend import get_backend +from ot.utils import unif +from ot.backend import get_backend ################################## LR-DYSKTRA ALGORITHM ########################################## -def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, nx=None): """ - Implementation of the Dykstra algorithm for low rank sinkhorn + Implementation of the Dykstra algorithm for the Low rank sinkhorn solver + """ + # Get dykstra parameters + g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2 = dykstra_p + g_ = eps3.copy() + err = 1 - # get dykstra parameters - q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_p + # POT backend if needed + if nx is None: + nx = get_backend(eps1, eps2, eps3, p1, p2, + g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2) - # POT backend - eps1, eps2, eps3, p1, p2 = list_to_array(eps1, eps2, eps3, p1, p2) - q3_1, q3_2, v1_, v2_, q1, q2 = list_to_array(q3_1, q3_2, v1_, v2_, q1, q2) - - nx = get_backend(eps1, eps2, eps3, p1, p2, q3_1, q3_2, v1_, v2_, q1, q2) - - # ------- Dykstra algorithm ------ - g_ = eps3 - u1 = p1 / nx.dot(eps1, v1_) - u2 = p2 / nx.dot(eps2, v2_) + # ------- Dykstra algorithm ------ + while err > stopThr : + u1 = p1 / nx.dot(eps1, v1_) + u2 = p2 / nx.dot(eps2, v2_) - g = nx.maximum(alpha, g_ * q3_1) - q3_1 = (g_ * q3_1) / g - g_ = g + g = nx.maximum(alpha, g_ * q3_1) + q3_1 = (g_ * q3_1) / g + g_ = g.copy() - prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) - prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) - g = (g_ * q3_2 * prod1 * prod2)**(1/3) + prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) + prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) + g = (g_ * q3_2 * prod1 * prod2)**(1/3) - v1 = g / nx.dot(eps1.T,u1) - v2 = g / nx.dot(eps2.T,u2) + v1 = g / nx.dot(eps1.T,u1) + v2 = g / nx.dot(eps2.T,u2) + q1 = (v1_ * q1) / v1 - q1 = (v1_ * q1) / v1 - q2 = (v2_ * q2) / v2 - q3_2 = (g_ * q3_2) / g - - v1_, v2_ = v1, v2 - g_ = g + q2 = (v2_ * q2) / v2 + q3_2 = (g_ * q3_2) / g + + v1_, v2_ = v1.copy(), v2.copy() + g_ = g.copy() - # Compute error - err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) - err2 = nx.sum(nx.abs(u2 * (eps2 @ v2) - p2)) - err = err1 + err2 + # Compute error + err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) + err2 = nx.sum(nx.abs(u2 * (eps2 @ v2) - p2)) + err = err1 + err2 # Compute low rank matrices Q, R Q = u1[:,None] * eps1 * v1[None,:] R = u2[:,None] * eps2 * v2[None,:] - dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] + dykstra_p = [g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2] - return Q, R, g, err, dykstra_p + return Q, R, dykstra_p #################################### LOW RANK SINKHORN ALGORITHM ######################################### -def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, metric='sqeuclidean', alpha="auto", - numItermax=10000, stopThr=1e-9, warn=True, verbose=False): +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", + numItermax=1000, stopThr=1e-9, warn=True, verbose=False): #stopThr = 1e-9 + r''' Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. - This function returns the two low-rank matrix decomposition of the OT plan (Q,R), as well as the weight vector g. Parameters @@ -95,6 +105,9 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, metric='sqeuclidea Max number of iterations stopThr : float, optional Stop threshold on error (>0) + warn: + + verbose: Returns @@ -109,73 +122,87 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, metric='sqeuclidea References ---------- - .. Scetbon, M., Cuturi, M., & Peyré, G (2021). Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. ''' - X_s, X_t = list_to_array(X_s, X_t) nx = get_backend(X_s, X_t) - ns, nt = X_s.shape[0], X_t.shape[0] if a is None: - a = nx.from_numpy(unif(ns), type_as=X_s) + a = unif(ns, type_as=X_s) if b is None: - b = nx.from_numpy(unif(nt), type_as=X_s) + b = unif(nt, type_as=X_t) - # Compute cost matrix - M = dist(X_s,X_t, metric=metric) + d = X_s.shape[1] + + # First low rank decomposition of the cost matrix (A) + M1 = nx.zeros((ns,(d+2))) + M1[:,0] = [nx.norm(X_s[i,:])**2 for i in range(ns)] + M1[:,1] = nx.ones(ns) + M1[:,2:] = -2*X_s + + # Second low rank decomposition of the cost matrix (B) + M2 = nx.zeros((nt,(d+2))) + M2[:,0] = nx.ones(nt) + M2[:,1] = [nx.norm(X_t[i,:])**2 for i in range(nt)] + M2[:,2:] = X_t # Compute rank rank = min(ns, nt, rank) r = rank + # Alpha: lower bound for 1/rank if alpha == 'auto': - alpha = 1.0 / (r + 1) + alpha = 1e-3 # no convergence with alpha = 1 / (r+1) if (1/r < alpha) or (alpha < 0): - warnings.warn("The provided alpha value might lead to instabilities.") + warnings.warn("The provided alpha value might lead to instabilities.") - # Compute gamma - L = nx.sqrt((2/(alpha**4))*(nx.norm(M)**2) + (reg + (2/(alpha**3))*(nx.norm(M))**2)) + L = nx.sqrt(3*(2/(alpha**4))*((nx.norm(M1)*nx.norm(M2))**2) + (reg + (2/(alpha**3))*(nx.norm(M1)*nx.norm(M2)))**2) gamma = 1/(2*L) - # Initialisation + # Initialize the low rank matrices Q, R, g Q, R, g = nx.ones((ns,r)), nx.ones((nt,r)), nx.ones(r) + + # Initialize parameters for Dykstra algorithm q3_1, q3_2 = nx.ones(r), nx.ones(r) + u1, u2 = nx.ones(ns), nx.ones(nt) + v1, v2 = nx.ones(r), nx.ones(r) v1_, v2_ = nx.ones(r), nx.ones(r) q1, q2 = nx.ones(r), nx.ones(r) - dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] - err = 1 + dykstra_p = [g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2] + - for ii in range(numItermax): - CR = nx.dot(M,R) - C_t_Q = nx.dot(M.T,Q) + for ii in range(numItermax): + CR_ = nx.dot(M2.T, R) + CR = nx.dot(M1, CR_) + + CQ_ = nx.dot(M1.T, Q) + CQ = nx.dot(M2, CQ_) + diag_g = (1/g)[:,None] eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) - eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) + eps2 = nx.exp(-gamma*(nx.dot(CQ,diag_g)) - ((gamma*reg)-1)*nx.log(R)) omega = nx.diag(nx.dot(Q.T, CR)) eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) - - if err < stopThr: - break - - if verbose: - if ii % 200 == 0: - print( - '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) - print('{:5d}|{:8e}|'.format(ii, err)) - - else: - if warn: - warnings.warn("Sinkhorn did not converge. You might want to " - "increase the number of iterations `numItermax` " - "or the regularization parameter `reg`.") + Q, R, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p, stopThr, nx) + g = dykstra_p[0] + + # if verbose: + # if ii % 200 == 0: + # print( + # '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + # print('{:5d}|{:8e}|'.format(ii, err)) + + # else: + # if warn: + # warnings.warn("Sinkhorn did not converge. You might want to " + # "increase the number of iterations `numItermax` " + # "or the regularization parameter `reg`.") return Q, R, g @@ -187,24 +214,23 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, metric='sqeuclidea ## Test with X_s, X_t from ot.datasets ############################################################################# -# import numpy as np -# import ot - -# Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) -# Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) +import numpy as np +import ot -# ns = Xs.shape[0] -# nt = Xt.shape[0] +Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) +Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) -# a = unif(ns) -# b = unif(nt) +ns = Xs.shape[0] +nt = Xt.shape[0] -# Q, R, g = lowrank_sinkhorn(Xs, Xt, reg=0.1, metric='euclidean', verbose=True, numItermax=100) -# M = ot.dist(Xs,Xt) -# P = np.dot(Q,np.dot(np.diag(1/g),R.T)) +a = unif(ns) +b = unif(nt) -# print(np.sum(P)) +Q, R, g = lowrank_sinkhorn(Xs, Xt, reg=0.1, verbose=True, numItermax=20) +M = ot.dist(Xs,Xt) +P = np.dot(Q,np.dot(np.diag(1/g),R.T)) +print(np.sum(P)) diff --git a/ot/solvers.py b/ot/solvers.py index c176969ca..8d6e10a5f 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -854,163 +854,3 @@ def solve_gromov(Ca, Cb, M=None, a=None, b=None, loss='L2', symmetric=None, -################################## WORK IN PROGRESS ##################################### - -## Implementation of the ot.solve_sample function -## Function isn't complete, still work in progress for reg == 0 / reg is None case (and unbalanced cases) - - -from .utils import unif, list_to_array, dist, OTResultLazy -from .bregman import empirical_sinkhorn - - -def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_type="KL", unbalanced=None, - unbalanced_type='KL', is_Lazy=False, batch_size=None, n_threads=1, max_iter=None, plan_init=None, - potentials_init=None, tol=None, verbose=False): - - r"""Solve the discrete optimal transport problem using the samples in the source and target domains. - It returns either a :any:`OTResult` or :any:`OTResultLazy` object. - - The function solves the following general optimal transport problem - - .. math:: - \min_{\mathbf{T}\geq 0} \quad \sum_{i,j} T_{i,j}M_{i,j} + \lambda_r R(\mathbf{T}) + - \lambda_u U(\mathbf{T}\mathbf{1},\mathbf{a}) + - \lambda_u U(\mathbf{T}^T\mathbf{1},\mathbf{b}) - - The regularization is selected with `reg` (:math:`\lambda_r`) and `reg_type`. By - default ``reg=None`` and there is no regularization. The unbalanced marginal - penalization can be selected with `unbalanced` (:math:`\lambda_u`) and - `unbalanced_type`. By default ``unbalanced=None`` and the function - solves the exact optimal transport problem (respecting the marginals). - - Parameters - ---------- - X_s : array-like, shape (n_samples_a, dim) - samples in the source domain - X_t : array-like, shape (n_samples_b, dim) - samples in the target domain - a : array-like, shape (dim_a,), optional - Samples weights in the source domain (default is uniform) - b : array-like, shape (dim_b,), optional - Samples weights in the source domain (default is uniform) - reg : float, optional - Regularization weight :math:`\lambda_r`, by default None (no reg., exact - OT) - reg_type : str, optional - Type of regularization :math:`R` either "KL", "L2", "entropy", by default "KL" - unbalanced : float, optional - Unbalanced penalization weight :math:`\lambda_u`, by default None - (balanced OT) - unbalanced_type : str, optional - Type of unbalanced penalization function :math:`U` either "KL", "L2", "TV", by default "KL" - is_Lazy : bool, optional - Return :any:`OTResultlazy` object to reduce memory cost when True, by default False - n_threads : int, optional - Number of OMP threads for exact OT solver, by default 1 - max_iter : int, optional - Maximum number of iteration, by default None (default values in each solvers) - plan_init : array_like, shape (dim_a, dim_b), optional - Initialization of the OT plan for iterative methods, by default None - potentials_init : (array_like(dim_a,),array_like(dim_b,)), optional - Initialization of the OT dual potentials for iterative methods, by default None - tol : _type_, optional - Tolerance for solution precision, by default None (default values in each solvers) - verbose : bool, optional - Print information in the solver, by default False - - Returns - ------- - - res_lazy : OTResultLazy() - Result of the optimization problem. This class only returns a partial OT plan and the OT dual potentials to reduce memory costs. - The information can be obtained as follows: - - - res.lazy_plan : OT plan computed on a subsample of X_s and X_t - - res.potentials : OT dual potentials - - See :any:`OTResultLazy` for more information. - - res : OTResult() - Result of the optimization problem. The information can be obtained as follows: - - - res.plan : OT plan :math:`\mathbf{T}` - - res.potentials : OT dual potentials - - res.value : Optimal value of the optimization problem - - res.value_linear : Linear OT loss with the optimal OT plan - - See :any:`OTResult` for more information. - - - """ - - X_s, X_t = list_to_array(X_s,X_t) - - # detect backend - arr = [X_s,X_t] - if a is not None: - arr.append(a) - if b is not None: - arr.append(b) - nx = get_backend(*arr) - - # create uniform weights if not given - ns, nt = X_s.shape[0], X_t.shape[0] - if a is None: - a = nx.from_numpy(unif(ns), type_as=X_s) - if b is None: - b = nx.from_numpy(unif(nt), type_as=X_s) - - # default values for solutions - potentials = None - lazy_plan = None - - if max_iter is None: - max_iter = 1000 - if tol is None: - tol = 1e-9 - if batch_size is None: - batch_size = 100 - - if is_Lazy: - ################# WIP #################### - if reg is None or reg == 0: # EMD solver for isLazy ? - - if unbalanced is None: # balanced EMD solver for isLazy ? - raise (NotImplementedError('Not implemented balanced with no regularization')) - - else: - raise (NotImplementedError('Not implemented unbalanced_type="{}" with no regularization'.format(unbalanced_type))) - - - ############################################# - - else: - if unbalanced is None: - u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, - isLazy=True, batchSize=batch_size, verbose=verbose, log=True) - # compute potentials - potentials = (log["u"], log["v"]) - - # compute lazy_plan - ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) - M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) - K = nx.exp(M / (-reg)) - lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) - - res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) - return res_lazy - - else: - raise (NotImplementedError('Not implemented unbalanced_type="{}" with regularization'.format(unbalanced_type))) - - else: - # compute cost matrix M and use solve function - M = dist(X_s, X_t, metric) - - res = solve(M, a, b, reg, reg_type, unbalanced, unbalanced_type, n_threads, max_iter, plan_init, potentials_init, tol, verbose) - return res - - - - diff --git a/ot/utils.py b/ot/utils.py index d570b9f30..01944f56b 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -939,92 +939,3 @@ def citation(self): } """ - - -############################## WORK IN PROGRESS #################################### - -## Implementation of the OTResultLazy class for ot.solve_sample() with potentials and lazy_plan as attributes - -class OTResultLazy: - def __init__(self, potentials=None, lazy_plan=None, backend=None): - - self._potentials = potentials - self._lazy_plan = lazy_plan - self._backend = backend if backend is not None else NumpyBackend() - - - # Dual potentials -------------------------------------------- - - def __repr__(self): - s = 'OTResultLazy(' - if self._lazy_plan is not None: - s += 'lazy_plan={}(shape={}),'.format(self._lazy_plan.__class__.__name__, self._lazy_plan.shape) - - if s[-1] != '(': - s = s[:-1] + ')' - else: - s = s + ')' - return s - - @property - def potentials(self): - """Dual potentials, i.e. Lagrange multipliers for the marginal constraints. - - This pair of arrays has the same shape, numerical type - and properties as the input weights "a" and "b". - """ - if self._potentials is not None: - return self._potentials - else: - raise NotImplementedError() - - @property - def potential_a(self): - """First dual potential, associated to the "source" measure "a".""" - if self._potentials is not None: - return self._potentials[0] - else: - raise NotImplementedError() - - @property - def potential_b(self): - """Second dual potential, associated to the "target" measure "b".""" - if self._potentials is not None: - return self._potentials[1] - else: - raise NotImplementedError() - - # Transport plan ------------------------------------------- - @property - def lazy_plan(self): - """A subset of the Transport plan, encoded as a dense array.""" - - if self._lazy_plan is not None: - return self._lazy_plan - else: - raise NotImplementedError() - - @property - def citation(self): - """Appropriate citation(s) for this result, in plain text and BibTex formats.""" - - # The string below refers to the POT library: - # successor methods may concatenate the relevant references - # to the original definitions, solvers and underlying numerical backends. - return """POT library: - - POT Python Optimal Transport library, Journal of Machine Learning Research, 22(78):1−8, 2021. - Website: https://pythonot.github.io/ - Rémi Flamary, Nicolas Courty, Alexandre Gramfort, Mokhtar Z. Alaya, Aurélie Boisbunon, Stanislas Chambon, Laetitia Chapel, Adrien Corenflos, Kilian Fatras, Nemo Fournier, Léo Gautheron, Nathalie T.H. Gayraud, Hicham Janati, Alain Rakotomamonjy, Ievgen Redko, Antoine Rolet, Antony Schutz, Vivien Seguy, Danica J. Sutherland, Romain Tavenard, Alexander Tong, Titouan Vayer; - - @article{flamary2021pot, - author = {R{\'e}mi Flamary and Nicolas Courty and Alexandre Gramfort and Mokhtar Z. Alaya and Aur{\'e}lie Boisbunon and Stanislas Chambon and Laetitia Chapel and Adrien Corenflos and Kilian Fatras and Nemo Fournier and L{\'e}o Gautheron and Nathalie T.H. Gayraud and Hicham Janati and Alain Rakotomamonjy and Ievgen Redko and Antoine Rolet and Antony Schutz and Vivien Seguy and Danica J. Sutherland and Romain Tavenard and Alexander Tong and Titouan Vayer}, - title = {{POT}: {Python} {Optimal} {Transport}}, - journal = {Journal of Machine Learning Research}, - year = {2021}, - volume = {22}, - number = {78}, - pages = {1-8}, - url = {http://jmlr.org/papers/v22/20-451.html} - } - """ \ No newline at end of file diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 6e1f24067..7d90ce9ef 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -16,34 +16,34 @@ def test_LR_Dykstra(): pass -@pytest.mark.parametrize("verbose, warn", product([True, False], [True, False])) -def test_lowrank_sinkhorn(verbose, warn): - # test low rank sinkhorn - n = 100 - a = ot.unif(n) - b = ot.unif(n) - - X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - - Q_sqe, R_sqe, g_sqe = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) - P_sqe = np.dot(Q_sqe,np.dot(np.diag(1/g_sqe),R_sqe.T)) - - Q_m, R_m, g_m = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1, metric='euclidean') - P_m = np.dot(Q_m,np.dot(np.diag(1/g_m),R_m.T)) - - # check constraints - np.testing.assert_allclose( - a, P_sqe.sum(1), atol=1e-05) # metric sqeuclidian - np.testing.assert_allclose( - b, P_sqe.sum(0), atol=1e-05) # metric sqeuclidian - np.testing.assert_allclose( - a, P_m.sum(1), atol=1e-05) # metric euclidian - np.testing.assert_allclose( - b, P_m.sum(0), atol=1e-05) # metric euclidian +# @pytest.mark.parametrize("verbose, warn", product([True, False], [True, False])) +# def test_lowrank_sinkhorn(verbose, warn): +# # test low rank sinkhorn +# n = 100 +# a = ot.unif(n) +# b = ot.unif(n) + +# X_s = np.reshape(1.0 * np.arange(n), (n, 1)) +# X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + +# Q_sqe, R_sqe, g_sqe = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) +# P_sqe = np.dot(Q_sqe,np.dot(np.diag(1/g_sqe),R_sqe.T)) + +# Q_m, R_m, g_m = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1, metric='euclidean') +# P_m = np.dot(Q_m,np.dot(np.diag(1/g_m),R_m.T)) + +# # check constraints +# np.testing.assert_allclose( +# a, P_sqe.sum(1), atol=1e-05) # metric sqeuclidian +# np.testing.assert_allclose( +# b, P_sqe.sum(0), atol=1e-05) # metric sqeuclidian +# np.testing.assert_allclose( +# a, P_m.sum(1), atol=1e-05) # metric euclidian +# np.testing.assert_allclose( +# b, P_m.sum(0), atol=1e-05) # metric euclidian - with pytest.warns(UserWarning): - ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) +# with pytest.warns(UserWarning): +# ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) diff --git a/test/test_solvers.py b/test/test_solvers.py index 5a05d54cf..e845ac7c2 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -258,100 +258,3 @@ def test_solve_gromov_not_implemented(nx): - -########################################################################################################### -############################################ WORK IN PROGRESS ############################################# -########################################################################################################### - -def assert_allclose_sol_sample(sol1, sol2): - # test attributes of OTResultLazy class - lst_attr = ['potentials','potential_a', 'potential_b', 'lazy_plan'] - - nx1 = sol1._backend if sol1._backend is not None else ot.backend.NumpyBackend() - nx2 = sol2._backend if sol2._backend is not None else ot.backend.NumpyBackend() - - for attr in lst_attr: - try: - np.allclose(nx1.to_numpy(getattr(sol1, attr)), nx2.to_numpy(getattr(sol2, attr))) - except NotImplementedError: - pass - - -@pytest.mark.parametrize("reg,reg_type,unbalanced,unbalanced_type", itertools.product(lst_reg, lst_reg_type, lst_unbalanced, lst_unbalanced_type)) -def test_solve_sample(nx): - # test solve_sample when is_Lazy = False - n = 100 - X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - - a = ot.utils.unif(X_s.shape[0]) - b = ot.utils.unif(X_t.shape[0]) - - # solve unif weights - sol0 = ot.solve_sample(X_s, X_t) - - # solve signe weights - sol = ot.solve_sample(X_s, X_t, a, b) - - # check some attributes - sol.potentials - sol.sparse_plan - sol.marginals - sol.status - - assert_allclose_sol(sol0, sol) - - # solve in backend - X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) - solb = ot.solve_sample(X_sb, X_tb, ab, bb) - - assert_allclose_sol(sol, solb) - - # test not implemented unbalanced and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, unbalanced=1, unbalanced_type='cryptic divergence') - - # test not implemented reg_type and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, reg=1, reg_type='cryptic divergence') - - - -def test_lazy_solve_sample(nx): - # test solve_sample when is_Lazy = True - n = 100 - X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - - a = ot.utils.unif(X_s.shape[0]) - b = ot.utils.unif(X_t.shape[0]) - - # solve unif weights - sol0 = ot.solve_sample(X_s, X_t, reg=0.1, is_Lazy=True) # reg != 0 or None since no implementation yet for is_Lazy=True - - # solve signe weights - sol = ot.solve_sample(X_s, X_t, a, b, reg=0.1, is_Lazy=True) - - # check some attributes - sol.potentials - sol.lazy_plan - - assert_allclose_sol_sample(sol0, sol) - - # solve in backend - X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) - solb = ot.solve_sample(X_sb, X_tb, ab, bb, reg=0.1, is_Lazy=True) - - assert_allclose_sol_sample(sol, solb) - - # test not implemented reg==0 (or None) + balanced and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, is_Lazy=True) # reg == 0 (or None) + unbalanced= None are default - - # test not implemented reg==0 (or None) + unbalanced_type and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, unbalanced_type="kl", is_Lazy=True) # reg == 0 (or None) is default - - # test not implemented reg != 0 + unbalanced_type and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, reg=0.1, unbalanced_type="kl", is_Lazy=True) \ No newline at end of file diff --git a/test/test_utils.py b/test/test_utils.py index a14be460e..bbadec65a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -403,28 +403,3 @@ def test_get_coordinate_circle(): np.testing.assert_allclose(u[0], x_p) - -############################################################################################## -##################################### WORK IN PROGRESS ####################################### -############################################################################################## - -# test function for OTResultLazy - -def test_OTResultLazy(): - - res_lazy = ot.utils.OTResultLazy() - - # test print - print(res_lazy) - - # tets get citation - print(res_lazy.citation) - - lst_attributes = ['lazy_plan', - 'potential_a', - 'potential_b', - 'potentials'] - - for at in lst_attributes: - with pytest.raises(NotImplementedError): - getattr(res_lazy, at) \ No newline at end of file From 6ea251c89ecf52603eb81c798a0769e9a2cb9f54 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 24 Oct 2023 14:54:42 +0200 Subject: [PATCH 06/36] new file for lr sinkhorn --- ot/lowrank.py | 171 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 ot/lowrank.py diff --git a/ot/lowrank.py b/ot/lowrank.py new file mode 100644 index 000000000..ba46cd1ed --- /dev/null +++ b/ot/lowrank.py @@ -0,0 +1,171 @@ +################################################################################################################# +############################################## WORK IN PROGRESS ################################################# +################################################################################################################# + + +from ot.utils import unif, list_to_array +from ot.backend import get_backend +from ot.datasets import make_1D_gauss as gauss + + + +################################## LR-DYSKTRA ALGORITHM ########################################## + +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_w): + """ + Implementation of the Dykstra algorithm for low rank Sinkhorn + """ + + # get dykstra parameters + q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_w + + # POT backend + eps1, eps2, eps3, p1, p2 = list_to_array(eps1, eps2, eps3, p1, p2) + q3_1, q3_2, v1_, v2_, q1, q2 = list_to_array(q3_1, q3_2, v1_, v2_, q1, q2) + + nx = get_backend(eps1, eps2, eps3, p1, p2, q3_1, q3_2, v1_, v2_, q1, q2) + + # ------- Dykstra algorithm ------ + g_ = eps3 + + u1 = p1 / nx.dot(eps1, v1_) + u2 = p2 / nx.dot(eps2, v2_) + + g = nx.maximum(alpha, g_ * q3_1) + q3_1 = (g_ * q3_1) / g + g_ = g + + prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) + prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) + g = (g_ * q3_2 * prod1 * prod2)**(1/3) + + v1 = g / nx.dot(eps1.T,u1) + v2 = g / nx.dot(eps2.T,u2) + + q1 = (v1_ * q1) / v1 + q2 = (v2_ * q2) / v2 + q3_2 = (g_ * q3_2) / g + + v1_, v2_ = v1, v2 + g_ = g + + # Compute error + err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) + err2 = nx.sum(nx.abs(u2 * (eps2 @ v2) - p2)) + err = err1 + err2 + + # Compute low rank matrices Q, R + Q = u1[:,None] * eps1 * v1[None,:] + R = u2[:,None] * eps2 * v2[None,:] + + dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + + return Q, R, g, err, dykstra_w + + + +#################################### LOW RANK SINKHORN ALGORITHM ######################################### + + +def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): + r''' + Solve the entropic regularization optimal transport problem under low-nonnegative low rank constraints + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + reg : float + Regularization term >0 + a : array-like, shape (n_samples_a,) + samples weights in the source domain + b : array-like, shape (n_samples_b,) + samples weights in the target domain + numItermax : int, optional + Max number of iterations + stopThr : float, optional + Stop threshold on error (>0) + + Returns + ------- + Q : array-like, shape (n_samples_a, r) + First low-rank matrix decomposition of the OT plan + R: array-like, shape (n_samples_b, r) + Second low-rank matrix decomposition of the OT plan + g : array-like, shape (r, ) + ... + + ''' + + X_s, X_t = list_to_array(X_s, X_t) + nx = get_backend(X_s, X_t) + + ns, nt = X_s.shape[0], X_t.shape[0] + if a is None: + a = nx.from_numpy(unif(ns), type_as=X_s) + if b is None: + b = nx.from_numpy(unif(nt), type_as=X_s) + + M = ot.dist(X_s,X_t, metric=metric) + + # Compute rank + r = min(ns, nt, r) + + # Compute gamma + L = nx.sqrt((2/(alpha**4))*nx.norm(M)**2 + (reg + (2/(alpha**3))*nx.norm(M))**2) + gamma = 1/(2*L) + + # Initialisation + Q, R, g = nx.ones((ns,r)), nx.ones((nt,r)), nx.ones(r) + q3_1, q3_2 = nx.ones(r), nx.ones(r) + v1_, v2_ = nx.ones(r), nx.ones(r) + q1, q2 = nx.ones(r), nx.ones(r) + dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + n_iter = 0 + err = 1 + + while n_iter < numIterMax: + if err > stopThr: + n_iter = n_iter + 1 + + CR = nx.dot(M,R) + C_t_Q = nx.dot(M.T,Q) + diag_g = (1/g)[:,None] + + eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) + eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) + omega = nx.diag(nx.dot(Q.T, CR)) + eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) + + Q, R, g, err, dykstra_w = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_w) + else: + break + + return Q, R, g + + + + + +############################################################################ +## Test with X_s, X_t from ot.datasets +############################################################################# + +import numpy as np +import ot + +Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) +Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) + + +Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) +M = ot.dist(Xs,Xt) +P = np.dot(Q,np.dot(np.diag(1/g),R.T)) + +print(np.sum(P)) + + + + From 965e4d69113f6fe8eab106412b652dabdbc05712 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 24 Oct 2023 16:47:21 +0200 Subject: [PATCH 07/36] lr sinkhorn, solve_sample, OTResultLazy --- ot/lowrank.py | 40 +++++++------ ot/solvers.py | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++ ot/utils.py | 3 +- 3 files changed, 183 insertions(+), 21 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index ba46cd1ed..a1c73bdf3 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -2,8 +2,10 @@ ############################################## WORK IN PROGRESS ################################################# ################################################################################################################# +## Implementation of the LR-Dykstra algorithm and low rank sinkhorn algorithms -from ot.utils import unif, list_to_array + +from ot.utils import unif, list_to_array, dist from ot.backend import get_backend from ot.datasets import make_1D_gauss as gauss @@ -11,13 +13,13 @@ ################################## LR-DYSKTRA ALGORITHM ########################################## -def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_w): +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): """ Implementation of the Dykstra algorithm for low rank Sinkhorn """ # get dykstra parameters - q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_w + q3_1, q3_2, v1_, v2_, q1, q2 = dykstra_p # POT backend eps1, eps2, eps3, p1, p2 = list_to_array(eps1, eps2, eps3, p1, p2) @@ -58,18 +60,18 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_w): Q = u1[:,None] * eps1 * v1[None,:] R = u2[:,None] * eps2 * v2[None,:] - dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] - return Q, R, g, err, dykstra_w + return Q, R, g, err, dykstra_p #################################### LOW RANK SINKHORN ALGORITHM ######################################### -def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): +def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): r''' - Solve the entropic regularization optimal transport problem under low-nonnegative low rank constraints + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints on the feasible couplings. Parameters ---------- @@ -95,7 +97,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) - ... + Third low-rank matrix decomposition of the OT plan ''' @@ -108,7 +110,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', if b is None: b = nx.from_numpy(unif(nt), type_as=X_s) - M = ot.dist(X_s,X_t, metric=metric) + M = dist(X_s,X_t, metric=metric) # Compute rank r = min(ns, nt, r) @@ -122,7 +124,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', q3_1, q3_2 = nx.ones(r), nx.ones(r) v1_, v2_ = nx.ones(r), nx.ones(r) q1, q2 = nx.ones(r), nx.ones(r) - dykstra_w = [q3_1, q3_2, v1_, v2_, q1, q2] + dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] n_iter = 0 err = 1 @@ -139,7 +141,7 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', omega = nx.diag(nx.dot(Q.T, CR)) eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - Q, R, g, err, dykstra_w = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_w) + Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) else: break @@ -153,18 +155,18 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=4, metric='sqeuclidean', ## Test with X_s, X_t from ot.datasets ############################################################################# -import numpy as np -import ot +# import numpy as np +# import ot -Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) -Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) +# Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) +# Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) -Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) -M = ot.dist(Xs,Xt) -P = np.dot(Q,np.dot(np.diag(1/g),R.T)) +# Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) +# M = ot.dist(Xs,Xt) +# P = np.dot(Q,np.dot(np.diag(1/g),R.T)) -print(np.sum(P)) +# print(np.sum(P)) diff --git a/ot/solvers.py b/ot/solvers.py index 0313cf588..9c2746c25 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -848,3 +848,164 @@ def solve_gromov(Ca, Cb, M=None, a=None, b=None, loss='L2', symmetric=None, value_linear=value_linear, value_quad=value_quad, plan=plan, status=status, backend=nx) return res + + + + + + +################################## WORK IN PROGRESS ##################################### + +## Implementation of the ot.solve_sample function +## Function isn't complete, still work in progress for reg == 0 / reg is None case (and unbalanced cases) + + +from .utils import unif, list_to_array, dist, OTResultLazy +from .bregman import empirical_sinkhorn + + +def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_type="KL", unbalanced=None, + unbalanced_type='KL', is_Lazy=False, batch_size=None, n_threads=1, max_iter=None, plan_init=None, + potentials_init=None, tol=None, verbose=False): + + r"""Solve the discrete optimal transport problem using the samples in the source and target domains. + It returns either a :any:`OTResult` or :any:`OTResultLazy` object. + + The function solves the following general optimal transport problem + + .. math:: + \min_{\mathbf{T}\geq 0} \quad \sum_{i,j} T_{i,j}M_{i,j} + \lambda_r R(\mathbf{T}) + + \lambda_u U(\mathbf{T}\mathbf{1},\mathbf{a}) + + \lambda_u U(\mathbf{T}^T\mathbf{1},\mathbf{b}) + + The regularization is selected with `reg` (:math:`\lambda_r`) and `reg_type`. By + default ``reg=None`` and there is no regularization. The unbalanced marginal + penalization can be selected with `unbalanced` (:math:`\lambda_u`) and + `unbalanced_type`. By default ``unbalanced=None`` and the function + solves the exact optimal transport problem (respecting the marginals). + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + a : array-like, shape (dim_a,), optional + Samples weights in the source domain (default is uniform) + b : array-like, shape (dim_b,), optional + Samples weights in the source domain (default is uniform) + reg : float, optional + Regularization weight :math:`\lambda_r`, by default None (no reg., exact + OT) + reg_type : str, optional + Type of regularization :math:`R` either "KL", "L2", "entropy", by default "KL" + unbalanced : float, optional + Unbalanced penalization weight :math:`\lambda_u`, by default None + (balanced OT) + unbalanced_type : str, optional + Type of unbalanced penalization function :math:`U` either "KL", "L2", "TV", by default "KL" + is_Lazy : bool, optional + Return :any:`OTResultlazy` object to reduce memory cost when True, by default False + n_threads : int, optional + Number of OMP threads for exact OT solver, by default 1 + max_iter : int, optional + Maximum number of iteration, by default None (default values in each solvers) + plan_init : array_like, shape (dim_a, dim_b), optional + Initialization of the OT plan for iterative methods, by default None + potentials_init : (array_like(dim_a,),array_like(dim_b,)), optional + Initialization of the OT dual potentials for iterative methods, by default None + tol : _type_, optional + Tolerance for solution precision, by default None (default values in each solvers) + verbose : bool, optional + Print information in the solver, by default False + + Returns + ------- + + res_lazy : OTResultLazy() + Result of the optimization problem. This class only returns a partial OT plan and the OT dual potentials to reduce memory costs. + The information can be obtained as follows: + + - res.lazy_plan : OT plan computed on a subsample of X_s and X_t :math:`\mathbf{T}` + - res.potentials : OT dual potentials + + See :any:`OTResultLazy` for more information. + + res : OTResult() + Result of the optimization problem. The information can be obtained as follows: + + - res.plan : OT plan :math:`\mathbf{T}` + - res.potentials : OT dual potentials + - res.value : Optimal value of the optimization problem + - res.value_linear : Linear OT loss with the optimal OT plan + + See :any:`OTResult` for more information. + + + """ + + X_s, X_t = list_to_array(X_s,X_t) + + # detect backend + arr = [X_s,X_t] + if a is not None: + arr.append(a) + if b is not None: + arr.append(b) + nx = get_backend(*arr) + + # create uniform weights if not given + ns, nt = X_s.shape[0], X_t.shape[0] + if a is None: + a = nx.from_numpy(unif(ns), type_as=X_s) + if b is None: + b = nx.from_numpy(unif(nt), type_as=X_s) + + # default values for solutions + potentials = None + lazy_plan = None + + if max_iter is None: + max_iter = 1000 + if tol is None: + tol = 1e-9 + if batch_size is None: + batch_size = 100 + + if is_Lazy: + ################# WIP #################### + if reg is None or reg == 0: # EMD solver for isLazy ? + if unbalanced is None: # not sure "unbalanced" parameter is needed here ? (since we won't compute value) + pass + elif unbalanced_type.lower() in ['kl', 'l2']: + pass + elif unbalanced_type.lower() == 'tv': + pass + pass + ############################################# + + else: + # compute potentials + u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, + isLazy=True, batchSize=batch_size, verbose=verbose, log=True) + potentials = (log["u"], log["v"]) + + # compute lazy_plan + ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) + M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) + K = nx.exp(M / (-reg)) + lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) + + res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) + return res_lazy + + else: + # compute cost matrix M and use solve function + M = dist(X_s, X_t, metric) + + res = solve(M, a, b, reg, reg_type, unbalanced, unbalanced_type, n_threads, max_iter, plan_init, potentials_init, tol, verbose) + return res + + + + diff --git a/ot/utils.py b/ot/utils.py index 0936648ca..2f4cfc9e7 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -1168,7 +1168,6 @@ def citation(self): } """ - class LazyTensor(object): """ A lazy tensor is a tensor that is not stored in memory. Instead, it is defined by a function that computes its values on the fly from slices. @@ -1233,4 +1232,4 @@ def __getitem__(self, key): return self._getitem(*k, **self.kwargs) def __repr__(self): - return "LazyTensor(shape={},attributes=({}))".format(self.shape, ','.join(self.kwargs.keys())) + return "LazyTensor(shape={},attributes=({}))".format(self.shape, ','.join(self.kwargs.keys())) \ No newline at end of file From fd5e26d86e484f310f55a792bca89de13bd7340f Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 25 Oct 2023 17:39:08 +0200 Subject: [PATCH 08/36] add test functions + small modif lr_sin/solve_sample --- ot/lowrank.py | 97 ++++++++++++++++++++++++++++------------- ot/solvers.py | 47 +++++++++++--------- test/test_lowrank.py | 84 ++++++++++++++++++++++++++++++++++++ test/test_solvers.py | 100 +++++++++++++++++++++++++++++++++++++++++++ test/test_utils.py | 1 + 5 files changed, 278 insertions(+), 51 deletions(-) create mode 100644 test/test_lowrank.py diff --git a/ot/lowrank.py b/ot/lowrank.py index a1c73bdf3..22ff8b754 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -4,10 +4,9 @@ ## Implementation of the LR-Dykstra algorithm and low rank sinkhorn algorithms - -from ot.utils import unif, list_to_array, dist -from ot.backend import get_backend -from ot.datasets import make_1D_gauss as gauss +import warnings +from .utils import unif, list_to_array, dist +from .backend import get_backend @@ -15,7 +14,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): """ - Implementation of the Dykstra algorithm for low rank Sinkhorn + Implementation of the Dykstra algorithm for low rank sinkhorn """ # get dykstra parameters @@ -69,9 +68,12 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p): #################################### LOW RANK SINKHORN ALGORITHM ######################################### -def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', alpha=1e-10, numIterMax=10000, stopThr=1e-20): +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, metric='sqeuclidean', alpha="auto", + numItermax=10000, stopThr=1e-9, warn=True, verbose=False): r''' - Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints on the feasible couplings. + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. + + This function returns the two low-rank matrix decomposition of the OT plan (Q,R), as well as the weight vector g. Parameters ---------- @@ -79,17 +81,22 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', samples in the source domain X_t : array-like, shape (n_samples_b, dim) samples in the target domain - reg : float - Regularization term >0 a : array-like, shape (n_samples_a,) samples weights in the source domain b : array-like, shape (n_samples_b,) samples weights in the target domain + reg : float, optional + Regularization term >0 + rank: int, optional + Nonnegative rank of the OT plan + alpha: int, optional + Lower bound for the weight vector g (>0 and <1/r) numItermax : int, optional Max number of iterations stopThr : float, optional Stop threshold on error (>0) + Returns ------- Q : array-like, shape (n_samples_a, r) @@ -97,7 +104,14 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) - Third low-rank matrix decomposition of the OT plan + Weight vector for the low-rank decomposition of the OT plan + + + References + ---------- + + .. Scetbon, M., Cuturi, M., & Peyré, G (2021). + Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. ''' @@ -110,13 +124,22 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', if b is None: b = nx.from_numpy(unif(nt), type_as=X_s) + # Compute cost matrix M = dist(X_s,X_t, metric=metric) - + # Compute rank - r = min(ns, nt, r) + rank = min(ns, nt, rank) + r = rank + + if alpha == 'auto': + alpha = 1.0 / (r + 1) + + if (1/r < alpha) or (alpha < 0): + warnings.warn("The provided alpha value might lead to instabilities.") + # Compute gamma - L = nx.sqrt((2/(alpha**4))*nx.norm(M)**2 + (reg + (2/(alpha**3))*nx.norm(M))**2) + L = nx.sqrt((2/(alpha**4))*(nx.norm(M)**2) + (reg + (2/(alpha**3))*(nx.norm(M))**2)) gamma = 1/(2*L) # Initialisation @@ -125,25 +148,34 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', v1_, v2_ = nx.ones(r), nx.ones(r) q1, q2 = nx.ones(r), nx.ones(r) dykstra_p = [q3_1, q3_2, v1_, v2_, q1, q2] - n_iter = 0 err = 1 - while n_iter < numIterMax: - if err > stopThr: - n_iter = n_iter + 1 - - CR = nx.dot(M,R) - C_t_Q = nx.dot(M.T,Q) - diag_g = (1/g)[:,None] - - eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) - eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) - omega = nx.diag(nx.dot(Q.T, CR)) - eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - - Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) - else: + for ii in range(numItermax): + CR = nx.dot(M,R) + C_t_Q = nx.dot(M.T,Q) + diag_g = (1/g)[:,None] + + eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) + eps2 = nx.exp(-gamma*(nx.dot(C_t_Q,diag_g)) - ((gamma*reg)-1)*nx.log(R)) + omega = nx.diag(nx.dot(Q.T, CR)) + eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) + + Q, R, g, err, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p) + + if err < stopThr: break + + if verbose: + if ii % 200 == 0: + print( + '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) + print('{:5d}|{:8e}|'.format(ii, err)) + + else: + if warn: + warnings.warn("Sinkhorn did not converge. You might want to " + "increase the number of iterations `numItermax` " + "or the regularization parameter `reg`.") return Q, R, g @@ -161,8 +193,13 @@ def lowrank_sinkhorn(X_s, X_t, reg=0, a=None, b=None, r=2, metric='sqeuclidean', # Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) # Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) +# ns = Xs.shape[0] +# nt = Xt.shape[0] + +# a = unif(ns) +# b = unif(nt) -# Q, R, g = lowrank_sinkhorn(Xs,Xt,reg=0.1) +# Q, R, g = lowrank_sinkhorn(Xs, Xt, reg=0.1, metric='euclidean', verbose=True, numItermax=100) # M = ot.dist(Xs,Xt) # P = np.dot(Q,np.dot(np.diag(1/g),R.T)) diff --git a/ot/solvers.py b/ot/solvers.py index 9c2746c25..c176969ca 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -926,7 +926,7 @@ def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_t Result of the optimization problem. This class only returns a partial OT plan and the OT dual potentials to reduce memory costs. The information can be obtained as follows: - - res.lazy_plan : OT plan computed on a subsample of X_s and X_t :math:`\mathbf{T}` + - res.lazy_plan : OT plan computed on a subsample of X_s and X_t - res.potentials : OT dual potentials See :any:`OTResultLazy` for more information. @@ -975,29 +975,34 @@ def solve_sample(X_s, X_t, a=None, b=None, metric='sqeuclidean', reg=None, reg_t if is_Lazy: ################# WIP #################### if reg is None or reg == 0: # EMD solver for isLazy ? - if unbalanced is None: # not sure "unbalanced" parameter is needed here ? (since we won't compute value) - pass - elif unbalanced_type.lower() in ['kl', 'l2']: - pass - elif unbalanced_type.lower() == 'tv': - pass - pass + + if unbalanced is None: # balanced EMD solver for isLazy ? + raise (NotImplementedError('Not implemented balanced with no regularization')) + + else: + raise (NotImplementedError('Not implemented unbalanced_type="{}" with no regularization'.format(unbalanced_type))) + + ############################################# else: - # compute potentials - u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, - isLazy=True, batchSize=batch_size, verbose=verbose, log=True) - potentials = (log["u"], log["v"]) - - # compute lazy_plan - ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) - M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) - K = nx.exp(M / (-reg)) - lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) - - res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) - return res_lazy + if unbalanced is None: + u, v, log = empirical_sinkhorn(X_s, X_t, reg, a, b, metric='sqeuclidean', numIterMax=max_iter, stopThr=tol, + isLazy=True, batchSize=batch_size, verbose=verbose, log=True) + # compute potentials + potentials = (log["u"], log["v"]) + + # compute lazy_plan + ns_lazy, nt_lazy = 100, 100 # size of the lazy_plan (subplan) + M = dist(X_s[:ns_lazy,:], X_t[:nt_lazy,:], metric) + K = nx.exp(M / (-reg)) + lazy_plan = u[:ns_lazy].reshape((-1, 1)) * K * v[:nt_lazy].reshape((1, -1)) + + res_lazy = OTResultLazy(potentials=potentials, lazy_plan=lazy_plan, backend=nx) + return res_lazy + + else: + raise (NotImplementedError('Not implemented unbalanced_type="{}" with regularization'.format(unbalanced_type))) else: # compute cost matrix M and use solve function diff --git a/test/test_lowrank.py b/test/test_lowrank.py new file mode 100644 index 000000000..6e1f24067 --- /dev/null +++ b/test/test_lowrank.py @@ -0,0 +1,84 @@ +##################################################################################################### +####################################### WORK IN PROGRESS ############################################ +##################################################################################################### + + +""" Test for low rank sinkhorn solvers """ + +import ot +import numpy as np +import pytest +from itertools import product + + +def test_LR_Dykstra(): + # test for LR_Dykstra algorithm ? catch nan values ? + pass + + +@pytest.mark.parametrize("verbose, warn", product([True, False], [True, False])) +def test_lowrank_sinkhorn(verbose, warn): + # test low rank sinkhorn + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + Q_sqe, R_sqe, g_sqe = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) + P_sqe = np.dot(Q_sqe,np.dot(np.diag(1/g_sqe),R_sqe.T)) + + Q_m, R_m, g_m = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1, metric='euclidean') + P_m = np.dot(Q_m,np.dot(np.diag(1/g_m),R_m.T)) + + # check constraints + np.testing.assert_allclose( + a, P_sqe.sum(1), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + b, P_sqe.sum(0), atol=1e-05) # metric sqeuclidian + np.testing.assert_allclose( + a, P_m.sum(1), atol=1e-05) # metric euclidian + np.testing.assert_allclose( + b, P_m.sum(0), atol=1e-05) # metric euclidian + + with pytest.warns(UserWarning): + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) + + + +@pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,4))) +def test_lowrank_sinkhorn_alpha_warning(alpha,rank): + # test warning for value of alpha + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + with pytest.warns(UserWarning): + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, r=rank, alpha=alpha, warn=False) + + + +def test_lowrank_sinkhorn_backends(nx): + # test low rank sinkhorn for different backends + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) + + Q, R, g = nx.to_numpy(ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, 0.1)) + P = np.dot(Q,np.dot(np.diag(1/g),R.T)) + + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + + + diff --git a/test/test_solvers.py b/test/test_solvers.py index f0f5b638f..5a05d54cf 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -255,3 +255,103 @@ def test_solve_gromov_not_implemented(nx): ot.solve_gromov(Ca, Cb, reg=1, unbalanced_type='partial', unbalanced=1.5) with pytest.raises(NotImplementedError): ot.solve_gromov(Ca, Cb, reg=1, unbalanced_type='partial', unbalanced=0.5, symmetric=False) + + + + +########################################################################################################### +############################################ WORK IN PROGRESS ############################################# +########################################################################################################### + +def assert_allclose_sol_sample(sol1, sol2): + # test attributes of OTResultLazy class + lst_attr = ['potentials','potential_a', 'potential_b', 'lazy_plan'] + + nx1 = sol1._backend if sol1._backend is not None else ot.backend.NumpyBackend() + nx2 = sol2._backend if sol2._backend is not None else ot.backend.NumpyBackend() + + for attr in lst_attr: + try: + np.allclose(nx1.to_numpy(getattr(sol1, attr)), nx2.to_numpy(getattr(sol2, attr))) + except NotImplementedError: + pass + + +@pytest.mark.parametrize("reg,reg_type,unbalanced,unbalanced_type", itertools.product(lst_reg, lst_reg_type, lst_unbalanced, lst_unbalanced_type)) +def test_solve_sample(nx): + # test solve_sample when is_Lazy = False + n = 100 + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + a = ot.utils.unif(X_s.shape[0]) + b = ot.utils.unif(X_t.shape[0]) + + # solve unif weights + sol0 = ot.solve_sample(X_s, X_t) + + # solve signe weights + sol = ot.solve_sample(X_s, X_t, a, b) + + # check some attributes + sol.potentials + sol.sparse_plan + sol.marginals + sol.status + + assert_allclose_sol(sol0, sol) + + # solve in backend + X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) + solb = ot.solve_sample(X_sb, X_tb, ab, bb) + + assert_allclose_sol(sol, solb) + + # test not implemented unbalanced and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, unbalanced=1, unbalanced_type='cryptic divergence') + + # test not implemented reg_type and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, reg=1, reg_type='cryptic divergence') + + + +def test_lazy_solve_sample(nx): + # test solve_sample when is_Lazy = True + n = 100 + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + a = ot.utils.unif(X_s.shape[0]) + b = ot.utils.unif(X_t.shape[0]) + + # solve unif weights + sol0 = ot.solve_sample(X_s, X_t, reg=0.1, is_Lazy=True) # reg != 0 or None since no implementation yet for is_Lazy=True + + # solve signe weights + sol = ot.solve_sample(X_s, X_t, a, b, reg=0.1, is_Lazy=True) + + # check some attributes + sol.potentials + sol.lazy_plan + + assert_allclose_sol_sample(sol0, sol) + + # solve in backend + X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) + solb = ot.solve_sample(X_sb, X_tb, ab, bb, reg=0.1, is_Lazy=True) + + assert_allclose_sol_sample(sol, solb) + + # test not implemented reg==0 (or None) + balanced and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, is_Lazy=True) # reg == 0 (or None) + unbalanced= None are default + + # test not implemented reg==0 (or None) + unbalanced_type and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, unbalanced_type="kl", is_Lazy=True) # reg == 0 (or None) is default + + # test not implemented reg != 0 + unbalanced_type and check raise + with pytest.raises(NotImplementedError): + sol0 = ot.solve_sample(X_s, X_t, reg=0.1, unbalanced_type="kl", is_Lazy=True) \ No newline at end of file diff --git a/test/test_utils.py b/test/test_utils.py index 3a9d590ab..942f403ce 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -569,3 +569,4 @@ def test_lowrank_LazyTensor(nx): T = ot.utils.get_lowrank_lazytensor(X1, X2, diag_d, nx=nx) np.testing.assert_allclose(nx.to_numpy(T[:]), nx.to_numpy(T0)) + \ No newline at end of file From 3df3b77de2605d233224e0ccefa6ee127af9f040 Mon Sep 17 00:00:00 2001 From: laudavid Date: Thu, 26 Oct 2023 10:49:23 +0200 Subject: [PATCH 09/36] add import to __init__ --- ot/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ot/__init__.py b/ot/__init__.py index f16b6fcfc..cb00f4553 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -35,6 +35,7 @@ from . import factored from . import solvers from . import gaussian +from . import lowrank # OT functions from .lp import (emd, emd2, emd_1d, emd2_1d, wasserstein_1d, @@ -50,7 +51,8 @@ gromov_barycenters, fused_gromov_wasserstein, fused_gromov_wasserstein2) from .weak import weak_optimal_transport from .factored import factored_optimal_transport -from .solvers import solve, solve_gromov +from .solvers import solve, solve_gromov, solve_sample +from .lowrank import lowrank_sinkhorn # utils functions from .utils import dist, unif, tic, toc, toq From ab5475b894207f66e017beb07e97ff8da0d381aa Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 3 Nov 2023 15:30:51 +0100 Subject: [PATCH 10/36] remove test solve_sample --- test/test_solvers.py | 105 ------------------------------------------- 1 file changed, 105 deletions(-) diff --git a/test/test_solvers.py b/test/test_solvers.py index ff5719251..5e398d732 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -256,108 +256,3 @@ def test_solve_gromov_not_implemented(nx): with pytest.raises(NotImplementedError): ot.solve_gromov(Ca, Cb, reg=1, unbalanced_type='partial', unbalanced=0.5, symmetric=False) - - - - - - - - - -########################################################################################################### -############################################ WORK IN PROGRESS ############################################# -########################################################################################################### - -def assert_allclose_sol_sample(sol1, sol2): - # test attributes of OTResultLazy class - lst_attr = ['potentials','potential_a', 'potential_b', 'lazy_plan'] - - nx1 = sol1._backend if sol1._backend is not None else ot.backend.NumpyBackend() - nx2 = sol2._backend if sol2._backend is not None else ot.backend.NumpyBackend() - - for attr in lst_attr: - try: - np.allclose(nx1.to_numpy(getattr(sol1, attr)), nx2.to_numpy(getattr(sol2, attr))) - except NotImplementedError: - pass - - -@pytest.mark.parametrize("reg,reg_type,unbalanced,unbalanced_type", itertools.product(lst_reg, lst_reg_type, lst_unbalanced, lst_unbalanced_type)) -def test_solve_sample(nx): - # test solve_sample when is_Lazy = False - n = 100 - X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - - a = ot.utils.unif(X_s.shape[0]) - b = ot.utils.unif(X_t.shape[0]) - - # solve unif weights - sol0 = ot.solve_sample(X_s, X_t) - - # solve signe weights - sol = ot.solve_sample(X_s, X_t, a, b) - - # check some attributes - sol.potentials - sol.sparse_plan - sol.marginals - sol.status - - assert_allclose_sol(sol0, sol) - - # solve in backend - X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) - solb = ot.solve_sample(X_sb, X_tb, ab, bb) - - assert_allclose_sol(sol, solb) - - # test not implemented unbalanced and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, unbalanced=1, unbalanced_type='cryptic divergence') - - # test not implemented reg_type and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, reg=1, reg_type='cryptic divergence') - - - -def test_lazy_solve_sample(nx): - # test solve_sample when is_Lazy = True - n = 100 - X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - - a = ot.utils.unif(X_s.shape[0]) - b = ot.utils.unif(X_t.shape[0]) - - # solve unif weights - sol0 = ot.solve_sample(X_s, X_t, reg=0.1, is_Lazy=True) # reg != 0 or None since no implementation yet for is_Lazy=True - - # solve signe weights - sol = ot.solve_sample(X_s, X_t, a, b, reg=0.1, is_Lazy=True) - - # check some attributes - sol.potentials - sol.lazy_plan - - assert_allclose_sol_sample(sol0, sol) - - # solve in backend - X_sb, X_tb, ab, bb = nx.from_numpy(X_s, X_t, a, b) - solb = ot.solve_sample(X_sb, X_tb, ab, bb, reg=0.1, is_Lazy=True) - - assert_allclose_sol_sample(sol, solb) - - # test not implemented reg==0 (or None) + balanced and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, is_Lazy=True) # reg == 0 (or None) + unbalanced= None are default - - # test not implemented reg==0 (or None) + unbalanced_type and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, unbalanced_type="kl", is_Lazy=True) # reg == 0 (or None) is default - - # test not implemented reg != 0 + unbalanced_type and check raise - with pytest.raises(NotImplementedError): - sol0 = ot.solve_sample(X_s, X_t, reg=0.1, unbalanced_type="kl", is_Lazy=True) \ No newline at end of file From f1c8cdd9ca88568b035d691a234497cf9f54fab5 Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 8 Nov 2023 11:08:25 +0100 Subject: [PATCH 11/36] add value, value_linear, lazy_plan --- ot/lowrank.py | 245 +++++++++++++++++++++++++++++--------------------- 1 file changed, 144 insertions(+), 101 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index d583f4741..cd060bdd2 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -14,17 +14,45 @@ import warnings -from ot.utils import unif +from ot.utils import unif, LazyTensor from ot.backend import get_backend +def compute_lr_cost_matrix(X_s, X_t, nx=None): + """ + Compute low rank decomposition of a sqeuclidean cost matrix. + This function won't work for other metrics. + + See Proposition 1 of the low rank sinkhorn paper + """ + + if nx is None: + nx = get_backend(X_s,X_t) + + ns = X_s.shape[0] + nt = X_t.shape[0] + d = X_s.shape[1] + + # First low rank decomposition of the cost matrix (A) + M1 = nx.zeros((ns,(d+2))) + M1[:,0] = [nx.norm(X_s[i,:])**2 for i in range(ns)] + M1[:,1] = nx.ones(ns) + M1[:,2:] = -2*X_s + + # Second low rank decomposition of the cost matrix (B) + M2 = nx.zeros((nt,(d+2))) + M2[:,0] = nx.ones(nt) + M2[:,1] = [nx.norm(X_t[i,:])**2 for i in range(nt)] + M2[:,2:] = X_t + + return M1, M2 + -################################## LR-DYSKTRA ALGORITHM ########################################## -def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, nx=None): +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, numItermax, warn, nx=None): """ - Implementation of the Dykstra algorithm for the Low rank sinkhorn solver - + Implementation of the Dykstra algorithm for the Low Rank sinkhorn OT solver. + """ # Get dykstra parameters g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2 = dykstra_p @@ -36,34 +64,44 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, nx=None): nx = get_backend(eps1, eps2, eps3, p1, p2, g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2) - - # ------- Dykstra algorithm ------ - while err > stopThr : - u1 = p1 / nx.dot(eps1, v1_) - u2 = p2 / nx.dot(eps2, v2_) - - g = nx.maximum(alpha, g_ * q3_1) - q3_1 = (g_ * q3_1) / g - g_ = g.copy() - - prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) - prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) - g = (g_ * q3_2 * prod1 * prod2)**(1/3) - - v1 = g / nx.dot(eps1.T,u1) - v2 = g / nx.dot(eps2.T,u2) - q1 = (v1_ * q1) / v1 - - q2 = (v2_ * q2) / v2 - q3_2 = (g_ * q3_2) / g - - v1_, v2_ = v1.copy(), v2.copy() - g_ = g.copy() - - # Compute error - err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) - err2 = nx.sum(nx.abs(u2 * (eps2 @ v2) - p2)) - err = err1 + err2 + # ------------- Dykstra algorithm ---------------- + # see "Algorithm 2 LR-Dykstra" in paper + for ii in range(numItermax): + if err > stopThr: + u1 = p1 / nx.dot(eps1, v1_) + u2 = p2 / nx.dot(eps2, v2_) + + g = nx.maximum(alpha, g_ * q3_1) + q3_1 = (g_ * q3_1) / g + g_ = g.copy() + + prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) + prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) + g = (g_ * q3_2 * prod1 * prod2)**(1/3) + + v1 = g / nx.dot(eps1.T,u1) + v2 = g / nx.dot(eps2.T,u2) + q1 = (v1_ * q1) / v1 + + q2 = (v2_ * q2) / v2 + q3_2 = (g_ * q3_2) / g + + v1_, v2_ = v1.copy(), v2.copy() + g_ = g.copy() + + # Compute error + err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) + err2 = nx.sum(nx.abs(u2 * (eps2 @ v2) - p2)) + err = err1 + err2 + + else: + break + + else: + if warn: + warnings.warn("Sinkhorn did not converge. You might want to " + "increase the number of iterations `numItermax` " + "or the regularization parameter `reg`.") # Compute low rank matrices Q, R Q = u1[:,None] * eps1 * v1[None,:] @@ -80,12 +118,28 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, nx=None): def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", - numItermax=1000, stopThr=1e-9, warn=True, verbose=False): #stopThr = 1e-9 + numItermax=1000, stopThr=1e-9, warn=True, shape_plan="auto"): r''' Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. - This function returns the two low-rank matrix decomposition of the OT plan (Q,R), as well as the weight vector g. + + The function solves the following optimization problem: + + .. math:: + \mathop{\inf}_{Q,R,g \in \mathcal{C(a,b,r)}} \langle C, Q\mathrm{diag}(1/g)R^T \rangle - + \mathrm{reg} \cdot H((Q,R,g)) + + where : + - :math:`C` is the (`dim_a`, `dim_b`) metric cost matrix + - :math:`H` is the entropic regularization term + - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target + weights (histograms, both sum to 1) + (add r, C(a,b,r), Q, R, g) !!! + + The entropy H is to be understood as that of the values of the three respective + entropies evaluated for each term. + Parameters ---------- X_s : array-like, shape (n_samples_a, dim) @@ -106,13 +160,21 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", Max number of iterations stopThr : float, optional Stop threshold on error (>0) - warn: - - verbose: + warn : bool, optional + if True, raises a warning if the algorithm doesn't convergence. + shape_plan : tuple + Shape of the lazy_plan Returns ------- + lazy_plan : + OT plan in a LazyTensor object + value : + Optimal value of the optimization problem, if reg=0 it will return the full value + if reg != 0, will return LazyTensor object + value_linear : + Linear OT loss with the optimal OT Q : array-like, shape (n_samples_a, r) First low-rank matrix decomposition of the OT plan R: array-like, shape (n_samples_b, r) @@ -127,44 +189,44 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. ''' + # POT backend nx = get_backend(X_s, X_t) ns, nt = X_s.shape[0], X_t.shape[0] + + # Initialize weights a, b if a is None: a = unif(ns, type_as=X_s) if b is None: b = unif(nt, type_as=X_t) - - d = X_s.shape[1] - - # First low rank decomposition of the cost matrix (A) - M1 = nx.zeros((ns,(d+2))) - M1[:,0] = [nx.norm(X_s[i,:])**2 for i in range(ns)] - M1[:,1] = nx.ones(ns) - M1[:,2:] = -2*X_s - # Second low rank decomposition of the cost matrix (B) - M2 = nx.zeros((nt,(d+2))) - M2[:,0] = nx.ones(nt) - M2[:,1] = [nx.norm(X_t[i,:])**2 for i in range(nt)] - M2[:,2:] = X_t - - # Compute rank + # Low rank decomposition of the sqeuclidean cost matrix (M1, M2) + M1, M2 = compute_lr_cost_matrix(X_s, X_t, nx=None) + + # Compute rank (not sure ?) rank = min(ns, nt, rank) r = rank - # Alpha: lower bound for 1/rank + # Check values of alpha, the lower bound for 1/rank (see ) if alpha == 'auto': - alpha = 1e-3 # no convergence with alpha = 1 / (r+1) - + alpha = 1e-10 + if (1/r < alpha) or (alpha < 0): warnings.warn("The provided alpha value might lead to instabilities.") - # Compute gamma + + # Compute gamma (see Proposition 4 of low rank sinkhorn paper) L = nx.sqrt(3*(2/(alpha**4))*((nx.norm(M1)*nx.norm(M2))**2) + (reg + (2/(alpha**3))*(nx.norm(M1)*nx.norm(M2)))**2) gamma = 1/(2*L) + + # Shape_plan default + if shape_plan == "auto": + shape_plan = (ns,nt) - # Initialize the low rank matrices Q, R, g + + # ----------- Initialisation of LR sinkhorn + Dykstra -------------- + + # Initialize the low rank matrices Q, R, g (not sure ?) Q, R, g = nx.ones((ns,r)), nx.ones((nt,r)), nx.ones(r) # Initialize parameters for Dykstra algorithm @@ -174,74 +236,55 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", v1_, v2_ = nx.ones(r), nx.ones(r) q1, q2 = nx.ones(r), nx.ones(r) dykstra_p = [g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2] + k = 100 # not specified in paper ? + + # ----------------- Low rank algorithm ------------------ - for ii in range(numItermax): + for ii in range(k): + # Compute the C*R dot matrix using the lr decomposition of C CR_ = nx.dot(M2.T, R) CR = nx.dot(M1, CR_) + # Compute the C.t * Q dot matrix using the lr decomposition of C CQ_ = nx.dot(M1.T, Q) CQ = nx.dot(M2, CQ_) - diag_g = (1/g)[:,None] + #diag_g = (1/g)[:,None] + diag_g = nx.diag(1/g) eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) eps2 = nx.exp(-gamma*(nx.dot(CQ,diag_g)) - ((gamma*reg)-1)*nx.log(R)) omega = nx.diag(nx.dot(Q.T, CR)) eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - Q, R, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p, stopThr, nx) + Q, R, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p, stopThr, numItermax, warn, nx) g = dykstra_p[0] - # if verbose: - # if ii % 200 == 0: - # print( - # '{:5s}|{:12s}'.format('It.', 'Err') + '\n' + '-' * 19) - # print('{:5d}|{:8e}|'.format(ii, err)) - # else: - # if warn: - # warnings.warn("Sinkhorn did not converge. You might want to " - # "increase the number of iterations `numItermax` " - # "or the regularization parameter `reg`.") + # ----------------- Compute lazy_plan, value and value_linear ------------------ - # Compute OT value using trace formula for scalar product + # Compute lazy plan (using LazyTensor class) + plan1 = Q + plan2 = nx.dot(nx.diag(1/g),R.T) # low memory cost since shape r*m + compute_plan = lambda i,j,P1,P2: nx.dot(P1[i,:], P2[:,j]) # function for LazyTensor + lazy_plan = LazyTensor(shape_plan, compute_plan, P1=plan1, P2=plan2) + + # Compute value_linear (using trace formula) v1 = nx.dot(Q.T,M1) v2 = nx.dot(R,nx.dot(diag_g.T,v1)) - value_linear = nx.sum(nx.diag(nx.dot(v2,M2.T))) # compute Trace - - #value = value_linear + reg * nx.sum(plan * nx.log(plan + 1e-16)) - - #value - - return value_linear, Q, R, g - - - - - -############################################################################ -## Test with X_s, X_t from ot.datasets -############################################################################# - -import numpy as np -import ot - -Xs, _ = ot.datasets.make_data_classif('3gauss', n=1000) -Xt, _ = ot.datasets.make_data_classif('3gauss2', n=1500) + value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) -ns = Xs.shape[0] -nt = Xt.shape[0] + # Compute value with entropy reg + reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) + reg_g = nx.sum(g * nx.log(g + 1e-16)) + reg_R = nx.sum(R * nx.log(R + 1e-16)) + value = value_linear + reg * (reg_Q + reg_g + reg_R) -a = unif(ns) -b = unif(nt) + return value, value_linear, lazy_plan, Q, R, g -Q, R, g = lowrank_sinkhorn(Xs, Xt, reg=0.1, verbose=True, numItermax=20) -M = ot.dist(Xs,Xt) -P = np.dot(Q,np.dot(np.diag(1/g),R.T)) -print(np.sum(P)) From df01cff10cc449ba3bde5f1a1809a335f5f1e9e8 Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 8 Nov 2023 12:43:48 +0100 Subject: [PATCH 12/36] add comments to lr algorithm --- ot/lowrank.py | 161 ++++++++++++++++++++++++++------------------------ 1 file changed, 83 insertions(+), 78 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index cd060bdd2..4c7f4c2bb 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -7,12 +7,6 @@ # License: MIT License - -################################################################################################################# -############################################## WORK IN PROGRESS ################################################# -################################################################################################################# - - import warnings from ot.utils import unif, LazyTensor from ot.backend import get_backend @@ -23,7 +17,12 @@ def compute_lr_cost_matrix(X_s, X_t, nx=None): Compute low rank decomposition of a sqeuclidean cost matrix. This function won't work for other metrics. - See Proposition 1 of the low rank sinkhorn paper + See "Section 3.5, proposition 1" of the paper + + References + ---------- + .. Scetbon, M., Cuturi, M., & Peyré, G (2021). + Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. """ if nx is None: @@ -49,43 +48,62 @@ def compute_lr_cost_matrix(X_s, X_t, nx=None): -def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, numItermax, warn, nx=None): +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=None): """ Implementation of the Dykstra algorithm for the Low Rank sinkhorn OT solver. + References + ---------- + .. Scetbon, M., Cuturi, M., & Peyré, G (2021). + Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. + """ - # Get dykstra parameters - g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2 = dykstra_p - g_ = eps3.copy() - err = 1 - # POT backend if needed + # POT backend if None if nx is None: - nx = get_backend(eps1, eps2, eps3, p1, p2, - g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2) + nx = get_backend(eps1, eps2, eps3, p1, p2) + + + # ----------------- Initialisation of Dykstra algorithm ----------------- + r = len(eps3) # rank + g_ = eps3.copy() # \tilde{g} + q3_1, q3_2 = nx.ones(r), nx.ones(r) # q^{(3)}_1, q^{(3)}_2 + v1_, v2_ = nx.ones(r), nx.ones(r) # \tilde{v}^{(1)}, \tilde{v}^{(2)} + q1, q2 = nx.ones(r), nx.ones(r) # q^{(1)}, q^{(2)} + err = 1 # initial error + - # ------------- Dykstra algorithm ---------------- - # see "Algorithm 2 LR-Dykstra" in paper + # --------------------- Dykstra algorithm ------------------------- + + # See Section 3.3 - "Algorithm 2 LR-Dykstra" in paper + for ii in range(numItermax): if err > stopThr: + + # Compute u^{(1)} and u^{(2)} u1 = p1 / nx.dot(eps1, v1_) u2 = p2 / nx.dot(eps2, v2_) + # Compute g, g^{(3)}_1 and update \tilde{g} g = nx.maximum(alpha, g_ * q3_1) q3_1 = (g_ * q3_1) / g g_ = g.copy() + # Compute new value of g with \prod prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) g = (g_ * q3_2 * prod1 * prod2)**(1/3) + # Compute v^{(1)} and v^{(2)} v1 = g / nx.dot(eps1.T,u1) v2 = g / nx.dot(eps2.T,u2) - q1 = (v1_ * q1) / v1 + # Compute q^{(1)}, q^{(2)} and q^{(3)}_2 + q1 = (v1_ * q1) / v1 q2 = (v2_ * q2) / v2 q3_2 = (g_ * q3_2) / g + # Update values of \tilde{v}^{(1)}, \tilde{v}^{(2)} and \tilde{g} v1_, v2_ = v1.copy(), v2.copy() g_ = g.copy() @@ -100,16 +118,13 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, numItermax, else: if warn: warnings.warn("Sinkhorn did not converge. You might want to " - "increase the number of iterations `numItermax` " - "or the regularization parameter `reg`.") + "increase the number of iterations `numItermax` ") # Compute low rank matrices Q, R Q = u1[:,None] * eps1 * v1[None,:] R = u2[:,None] * eps2 * v2[None,:] - dykstra_p = [g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2] - - return Q, R, dykstra_p + return Q, R, g @@ -118,7 +133,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, dykstra_p, stopThr, numItermax, def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", - numItermax=1000, stopThr=1e-9, warn=True, shape_plan="auto"): + numItermax=10000, stopThr=1e-9, warn=True, shape_plan="auto"): r''' Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. @@ -126,18 +141,20 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", The function solves the following optimization problem: .. math:: - \mathop{\inf}_{Q,R,g \in \mathcal{C(a,b,r)}} \langle C, Q\mathrm{diag}(1/g)R^T \rangle - + \mathop{\inf_{(Q,R,g) \in \mathcal{C(a,b,r)}}} \langle C, Q\mathrm{diag}(1/g)R^T \rangle - \mathrm{reg} \cdot H((Q,R,g)) - + where : - :math:`C` is the (`dim_a`, `dim_b`) metric cost matrix - - :math:`H` is the entropic regularization term - - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target - weights (histograms, both sum to 1) - (add r, C(a,b,r), Q, R, g) !!! - - The entropy H is to be understood as that of the values of the three respective - entropies evaluated for each term. + - :math:`H((Q,R,g))` is the values of the three respective entropies evaluated for each term. + - :math: `Q` and `R` are the low-rank matrix decomposition of the OT plan + - :math: `g` is the weight vector for the low-rank decomposition of the OT plan + - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target weights (histograms, both sum to 1) + - :math: `r` is the rank of the OT plan + - :math: `\mathcal{C(a,b,r)}` are the low-rank couplings of the OT problem + \mathcal{C(a,b,r)} = \mathcal{C_1(a,b,r)} \cap \mathcal{C_2(r)} with + \mathcal{C_1(a,b,r)} = \{ (Q,R,g) s.t Q\mathbb{1}_r = a, R^T \mathbb{1}_m = b \} + \mathcal{C_2(r)} = \{ (Q,R,g) s.t Q\mathbb{1}_n = R^T \mathbb{1}_m = g \} Parameters @@ -152,9 +169,9 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", samples weights in the target domain reg : float, optional Regularization term >0 - rank: int, optional + rank: int, default "auto" Nonnegative rank of the OT plan - alpha: int, optional + alpha: int, default "auto" Lower bound for the weight vector g (>0 and <1/r) numItermax : int, optional Max number of iterations @@ -168,12 +185,12 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", Returns ------- - lazy_plan : - OT plan in a LazyTensor object - value : - Optimal value of the optimization problem, if reg=0 it will return the full value - if reg != 0, will return LazyTensor object - value_linear : + lazy_plan : LazyTensor() + OT plan in a LazyTensor object of shape (shape_plan) + See :any:`LazyTensor` for more information. + value : float + Optimal value of the optimization problem, + value_linear : float Linear OT loss with the optimal OT Q : array-like, shape (n_samples_a, r) First low-rank matrix decomposition of the OT plan @@ -200,46 +217,36 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", if b is None: b = unif(nt, type_as=X_t) - # Low rank decomposition of the sqeuclidean cost matrix (M1, M2) - M1, M2 = compute_lr_cost_matrix(X_s, X_t, nx=None) - - # Compute rank (not sure ?) - rank = min(ns, nt, rank) - r = rank - - # Check values of alpha, the lower bound for 1/rank (see ) + # Compute rank (see Section 3.1, def 1) + if rank == "auto": + r = min(ns, nt) + + # Check values of alpha, the lower bound for 1/rank + # (see "Section 3.2: The Low-rank OT Problem (LOT)" in the paper) if alpha == 'auto': alpha = 1e-10 if (1/r < alpha) or (alpha < 0): - warnings.warn("The provided alpha value might lead to instabilities.") + warnings.warn("The provided alpha value might lead to instabilities.") + # Default value for shape tensor parameter in LazyTensor + if shape_plan == "auto": + shape_plan = (ns,nt) - # Compute gamma (see Proposition 4 of low rank sinkhorn paper) + # Low rank decomposition of the sqeuclidean cost matrix (A, B) + M1, M2 = compute_lr_cost_matrix(X_s, X_t, nx=None) + + # Compute gamma (see "Section 3.4, proposition 4" in the paper) L = nx.sqrt(3*(2/(alpha**4))*((nx.norm(M1)*nx.norm(M2))**2) + (reg + (2/(alpha**3))*(nx.norm(M1)*nx.norm(M2)))**2) gamma = 1/(2*L) - - # Shape_plan default - if shape_plan == "auto": - shape_plan = (ns,nt) - - - # ----------- Initialisation of LR sinkhorn + Dykstra -------------- - # Initialize the low rank matrices Q, R, g (not sure ?) + # Initialize the low rank matrices Q, R, g Q, R, g = nx.ones((ns,r)), nx.ones((nt,r)), nx.ones(r) - - # Initialize parameters for Dykstra algorithm - q3_1, q3_2 = nx.ones(r), nx.ones(r) - u1, u2 = nx.ones(ns), nx.ones(nt) - v1, v2 = nx.ones(r), nx.ones(r) - v1_, v2_ = nx.ones(r), nx.ones(r) - q1, q2 = nx.ones(r), nx.ones(r) - dykstra_p = [g, q3_1, q3_2, v1_, v2_, q1, q2, u1, u2, v1, v2] k = 100 # not specified in paper ? - # ----------------- Low rank algorithm ------------------ + # -------------------------- Low rank algorithm ------------------------------ + # see "Section 3.3, Algorithm 3 LOT" in the paper for ii in range(k): # Compute the C*R dot matrix using the lr decomposition of C @@ -250,24 +257,22 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", CQ_ = nx.dot(M1.T, Q) CQ = nx.dot(M2, CQ_) - #diag_g = (1/g)[:,None] diag_g = nx.diag(1/g) - + eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) eps2 = nx.exp(-gamma*(nx.dot(CQ,diag_g)) - ((gamma*reg)-1)*nx.log(R)) omega = nx.diag(nx.dot(Q.T, CR)) eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) - Q, R, dykstra_p = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, dykstra_p, stopThr, numItermax, warn, nx) - g = dykstra_p[0] - + Q, R, g = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx) # ----------------- Compute lazy_plan, value and value_linear ------------------ + # see "Section 3.2: The Low-rank OT Problem" in the paper # Compute lazy plan (using LazyTensor class) plan1 = Q - plan2 = nx.dot(nx.diag(1/g),R.T) # low memory cost since shape r*m + plan2 = nx.dot(nx.diag(1/g),R.T) # low memory cost since shape (r*m) compute_plan = lambda i,j,P1,P2: nx.dot(P1[i,:], P2[:,j]) # function for LazyTensor lazy_plan = LazyTensor(shape_plan, compute_plan, P1=plan1, P2=plan2) @@ -276,10 +281,10 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", v2 = nx.dot(R,nx.dot(diag_g.T,v1)) value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) - # Compute value with entropy reg - reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) - reg_g = nx.sum(g * nx.log(g + 1e-16)) - reg_R = nx.sum(R * nx.log(R + 1e-16)) + # Compute value with entropy reg (entropy of Q, R, g must be computed separatly) + reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q + reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g + reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R value = value_linear + reg * (reg_Q + reg_g + reg_R) return value, value_linear, lazy_plan, Q, R, g From 5bc9de96719de312bd41ef4df9f8df499fbe4e1b Mon Sep 17 00:00:00 2001 From: laudavid Date: Thu, 9 Nov 2023 17:06:51 +0100 Subject: [PATCH 13/36] modify test functions + add comments to lowrank --- ot/lowrank.py | 6 ++-- test/test_lowrank.py | 72 ++++++++++++++++++++------------------------ 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index 4c7f4c2bb..af9f06d62 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -229,7 +229,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", if (1/r < alpha) or (alpha < 0): warnings.warn("The provided alpha value might lead to instabilities.") - # Default value for shape tensor parameter in LazyTensor + # Default value for shape tensor parameter in LazyTensor if shape_plan == "auto": shape_plan = (ns,nt) @@ -245,6 +245,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", k = 100 # not specified in paper ? + # -------------------------- Low rank algorithm ------------------------------ # see "Section 3.3, Algorithm 3 LOT" in the paper @@ -267,6 +268,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", Q, R, g = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx) + # ----------------- Compute lazy_plan, value and value_linear ------------------ # see "Section 3.2: The Low-rank OT Problem" in the paper @@ -281,7 +283,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", v2 = nx.dot(R,nx.dot(diag_g.T,v1)) value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) - # Compute value with entropy reg (entropy of Q, R, g must be computed separatly) + # Compute value with entropy reg (entropy of Q, R, g must be computed separatly, see "Section 3.2" in the paper) reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 7d90ce9ef..7d326388b 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -11,45 +11,39 @@ from itertools import product -def test_LR_Dykstra(): - # test for LR_Dykstra algorithm ? catch nan values ? - pass - - -# @pytest.mark.parametrize("verbose, warn", product([True, False], [True, False])) -# def test_lowrank_sinkhorn(verbose, warn): -# # test low rank sinkhorn -# n = 100 -# a = ot.unif(n) -# b = ot.unif(n) - -# X_s = np.reshape(1.0 * np.arange(n), (n, 1)) -# X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - -# Q_sqe, R_sqe, g_sqe = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) -# P_sqe = np.dot(Q_sqe,np.dot(np.diag(1/g_sqe),R_sqe.T)) - -# Q_m, R_m, g_m = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1, metric='euclidean') -# P_m = np.dot(Q_m,np.dot(np.diag(1/g_m),R_m.T)) - -# # check constraints -# np.testing.assert_allclose( -# a, P_sqe.sum(1), atol=1e-05) # metric sqeuclidian -# np.testing.assert_allclose( -# b, P_sqe.sum(0), atol=1e-05) # metric sqeuclidian -# np.testing.assert_allclose( -# a, P_m.sum(1), atol=1e-05) # metric euclidian -# np.testing.assert_allclose( -# b, P_m.sum(0), atol=1e-05) # metric euclidian + +################################################## WORK IN PROGRESS ####################################################### + +# Add test functions for each function in lowrank.py file ? + +def test_lowrank_sinkhorn(verbose, warn): + # test low rank sinkhorn + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + + # what to test for value, value_linear, Q, R and g ? + value, value_linear, lazy_plan, Q, R, g = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) + P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) + + # check constraints for P + np.testing.assert_allclose( + a, P.sum(1), atol=1e-05) + np.testing.assert_allclose( + b, P.sum(0), atol=1e-05) -# with pytest.warns(UserWarning): -# ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) + # check warn parameter when Dykstra algorithm doesn't converge + with pytest.warns(UserWarning): + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) @pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,4))) def test_lowrank_sinkhorn_alpha_warning(alpha,rank): - # test warning for value of alpha + # Test warning for value of alpha n = 100 a = ot.unif(n) b = ot.unif(n) @@ -58,12 +52,12 @@ def test_lowrank_sinkhorn_alpha_warning(alpha,rank): X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) with pytest.warns(UserWarning): - ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, r=rank, alpha=alpha, warn=False) + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, r=rank, alpha=alpha, warn=False) # remove warning for lack of convergence def test_lowrank_sinkhorn_backends(nx): - # test low rank sinkhorn for different backends + # Test low rank sinkhorn for different backends n = 100 a = ot.unif(n) b = ot.unif(n) @@ -73,11 +67,11 @@ def test_lowrank_sinkhorn_backends(nx): ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) - Q, R, g = nx.to_numpy(ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, 0.1)) - P = np.dot(Q,np.dot(np.diag(1/g),R.T)) + value, value_linear, lazy_plan, Q, R, g = ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, 0.1) + P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) - np.testing.assert_allclose(a, P.sum(1), atol=1e-05) - np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) + np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) From 6040e6ffacb91673b4628118c23cc5b2ab44e22d Mon Sep 17 00:00:00 2001 From: laudavid Date: Thu, 9 Nov 2023 17:09:40 +0100 Subject: [PATCH 14/36] modify __init__ with lowrank --- ot/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ot/__init__.py b/ot/__init__.py index 3a4f21083..4aba450af 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -36,7 +36,6 @@ from . import solvers from . import gaussian from . import lowrank -from . import lowrank # OT functions from .lp import (emd, emd2, emd_1d, emd2_1d, wasserstein_1d, From a7fdffd4656ecd6b5e80d37392d040734eefd462 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 14 Nov 2023 14:01:29 +0100 Subject: [PATCH 15/36] debug lowrank + test --- ot/lowrank.py | 6 +++--- test/test_lowrank.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index af9f06d62..f2160e2ce 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -8,8 +8,8 @@ import warnings -from ot.utils import unif, LazyTensor -from ot.backend import get_backend +from .utils import unif, LazyTensor +from .backend import get_backend def compute_lr_cost_matrix(X_s, X_t, nx=None): @@ -218,6 +218,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", b = unif(nt, type_as=X_t) # Compute rank (see Section 3.1, def 1) + r = rank if rank == "auto": r = min(ns, nt) @@ -294,4 +295,3 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", - diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 7d326388b..b1aa64e32 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -16,7 +16,7 @@ # Add test functions for each function in lowrank.py file ? -def test_lowrank_sinkhorn(verbose, warn): +def test_lowrank_sinkhorn(): # test low rank sinkhorn n = 100 a = ot.unif(n) From d90c186668e66d0c0ec8b496e08a589089a436f2 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 14 Nov 2023 14:13:32 +0100 Subject: [PATCH 16/36] debug test function low_rank --- test/test_lowrank.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_lowrank.py b/test/test_lowrank.py index b1aa64e32..22c6c2f88 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -23,7 +23,7 @@ def test_lowrank_sinkhorn(): b = ot.unif(n) X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(n), (n, 1)) # what to test for value, value_linear, Q, R and g ? value, value_linear, lazy_plan, Q, R, g = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) @@ -52,7 +52,7 @@ def test_lowrank_sinkhorn_alpha_warning(alpha,rank): X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) with pytest.warns(UserWarning): - ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, r=rank, alpha=alpha, warn=False) # remove warning for lack of convergence + ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, rank=rank, alpha=alpha, warn=False) # remove warning for lack of convergence From ea3a3e0948ec60c7c8b94df1ca36497aa8138408 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 14 Nov 2023 14:31:35 +0100 Subject: [PATCH 17/36] error test --- ot/lowrank.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index f2160e2ce..c6653f6c8 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -132,7 +132,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No #################################### LOW RANK SINKHORN ALGORITHM ######################################### -def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=2, alpha="auto", +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", numItermax=10000, stopThr=1e-9, warn=True, shape_plan="auto"): r''' From 165e8f5352dc8688a7f9df16e915d8c0df2d597f Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 15 Nov 2023 16:03:06 +0100 Subject: [PATCH 18/36] final debug of lowrank + add new test functions --- ot/lowrank.py | 15 +++++----- test/test_lowrank.py | 66 +++++++++++++++++++++++++++----------------- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index c6653f6c8..ed526cf1c 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -171,7 +171,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", Regularization term >0 rank: int, default "auto" Nonnegative rank of the OT plan - alpha: int, default "auto" + alpha: int, default "auto" (1e-10) Lower bound for the weight vector g (>0 and <1/r) numItermax : int, optional Max number of iterations @@ -221,14 +221,14 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", r = rank if rank == "auto": r = min(ns, nt) + + if alpha == "auto": + alpha = 1e-10 - # Check values of alpha, the lower bound for 1/rank + # Dykstra algorithm won't converge if 1/rank < alpha (alpha is the lower bound for 1/rank) # (see "Section 3.2: The Low-rank OT Problem (LOT)" in the paper) - if alpha == 'auto': - alpha = 1e-10 - - if (1/r < alpha) or (alpha < 0): - warnings.warn("The provided alpha value might lead to instabilities.") + if 1/r < alpha : + raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format(a=alpha,r=1/rank)) # Default value for shape tensor parameter in LazyTensor if shape_plan == "auto": @@ -295,3 +295,4 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", + diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 22c6c2f88..820c147bf 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -8,13 +8,23 @@ import ot import numpy as np import pytest -from itertools import product ################################################## WORK IN PROGRESS ####################################################### -# Add test functions for each function in lowrank.py file ? +def test_compute_lr_cost_matrix(): + # test computation of low rank cost matrices M1 and M2 + n = 100 + X_s = np.reshape(1.0 * np.arange(2*n), (n, 2)) + X_t = np.reshape(1.0 * np.arange(2*n), (n, 2)) + + M1, M2 = ot.lowrank.compute_lr_cost_matrix(X_s, X_t) + M = ot.dist(X_s, X_t, metric="sqeuclidean") # original cost matrix + + np.testing.assert_allclose( + np.dot(M1,M2.T), M, atol=1e-05) + def test_lowrank_sinkhorn(): # test low rank sinkhorn @@ -27,21 +37,29 @@ def test_lowrank_sinkhorn(): # what to test for value, value_linear, Q, R and g ? value, value_linear, lazy_plan, Q, R, g = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) + P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) # check constraints for P - np.testing.assert_allclose( - a, P.sum(1), atol=1e-05) - np.testing.assert_allclose( - b, P.sum(0), atol=1e-05) + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + # check if lazy_plan is equal to the fully computed plan + P_true = np.dot(Q,np.dot(np.diag(1/g),R.T)) + np.testing.assert_allclose(P, P_true, atol=1e-05) + + # check if value_linear is correct with its original formula + M = ot.dist(X_s, X_t, metric="sqeuclidean") + value_linear_true = np.sum(M * P_true) + np.testing.assert_allclose(value_linear, value_linear_true, atol=1e-05) # check warn parameter when Dykstra algorithm doesn't converge with pytest.warns(UserWarning): - ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, stopThr=0, numItermax=1) + ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1) -@pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,4))) +@pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,6))) def test_lowrank_sinkhorn_alpha_warning(alpha,rank): # Test warning for value of alpha n = 100 @@ -51,28 +69,24 @@ def test_lowrank_sinkhorn_alpha_warning(alpha,rank): X_s = np.reshape(1.0 * np.arange(n), (n, 1)) X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - with pytest.warns(UserWarning): - ot.lowrank.lowrank_sinkhorn(X_s, X_t, 0.1, a, b, rank=rank, alpha=alpha, warn=False) # remove warning for lack of convergence - - - -def test_lowrank_sinkhorn_backends(nx): - # Test low rank sinkhorn for different backends - n = 100 - a = ot.unif(n) - b = ot.unif(n) + with pytest.raises(ValueError): + ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) - X_s = np.reshape(1.0 * np.arange(n), (n, 1)) - X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) - - value, value_linear, lazy_plan, Q, R, g = ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, 0.1) - P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) - np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) - np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) +# def test_lowrank_sinkhorn_backends(nx): +# # Test low rank sinkhorn for different backends +# n = 100 +# a = ot.unif(n) +# b = ot.unif(n) +# X_s = np.reshape(1.0 * np.arange(n), (n, 1)) +# X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) +# ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) +# value, value_linear, lazy_plan, Q, R, g = lowrank_sinkhorn(X_sb, X_tb, ab, bb, reg=0.1) +# P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) +# np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) +# np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) \ No newline at end of file From 8c6ac67a715096f9d8b1cc41e0dca73dbfff8774 Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 24 Nov 2023 15:43:40 +0100 Subject: [PATCH 19/36] Debug tests + add lowrank to solve_sample --- CONTRIBUTORS.md | 1 + README.md | 2 ++ RELEASES.md | 1 + ot/__init__.py | 3 +- ot/lowrank.py | 67 ++++++++++++++++++++++---------------------- ot/solvers.py | 21 ++++++++++++++ test/test_lowrank.py | 37 ++++++++++++------------ test/test_solvers.py | 6 ++-- 8 files changed, 84 insertions(+), 54 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index c7916f50a..5cc34f38b 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -45,6 +45,7 @@ The contributors to this library are: * [Ronak Mehta](https://ronakrm.github.io) (Efficient Discrete Multi Marginal Optimal Transport Regularization) * [Xizheng Yu](https://github.com/x12hengyu) (Efficient Discrete Multi Marginal Optimal Transport Regularization) * [Sonia Mazelet](https://github.com/SoniaMaz8) (Template based GNN layers) +* [Laurène David](https://github.com/laudavid) (Low rank sinkhorn) ## Acknowledgments diff --git a/README.md b/README.md index 84b3cf0ee..94bf97043 100644 --- a/README.md +++ b/README.md @@ -343,3 +343,5 @@ distances between Gaussian distributions](https://hal.science/hal-03197398v2/fil [61] Charlier, B., Feydy, J., Glaunes, J. A., Collin, F. D., & Durif, G. (2021). [Kernel operations on the gpu, with autodiff, without memory overflows](https://www.jmlr.org/papers/volume22/20-275/20-275.pdf). The Journal of Machine Learning Research, 22(1), 3457-3462. [62] H. Van Assel, C. Vincent-Cuaz, T. Vayer, R. Flamary, N. Courty (2023). [Interpolating between Clustering and Dimensionality Reduction with Gromov-Wasserstein](https://arxiv.org/pdf/2310.03398.pdf). NeurIPS 2023 Workshop Optimal Transport and Machine Learning. + +[63] Scetbon, M., Cuturi, M., & Peyré, G. (2021). [Low-Rank Sinkhorn Factorization](https://arxiv.org/pdf/2103.04737.pdf). diff --git a/RELEASES.md b/RELEASES.md index 349c56214..befda9d30 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -20,6 +20,7 @@ + Wrapper for `geomloss`` solver on empirical samples (PR #571) + Add `stop_criterion` feature to (un)regularized (f)gw barycenter solvers (PR #578) + Add `fixed_structure` and `fixed_features` to entropic fgw barycenter solver (PR #578) ++ Added support for [Low-Rank Sinkhorn Factorization](https://arxiv.org/pdf/2103.04737.pdf) (PR #568) #### Closed issues - Fix line search evaluating cost outside of the interpolation range (Issue #502, PR #504) diff --git a/ot/__init__.py b/ot/__init__.py index bd26d96e3..99d075e5a 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -35,6 +35,7 @@ from . import factored from . import solvers from . import gaussian +from . import lowrank # OT functions @@ -51,7 +52,7 @@ gromov_barycenters, fused_gromov_wasserstein, fused_gromov_wasserstein2) from .weak import weak_optimal_transport from .factored import factored_optimal_transport -from .solvers import solve, solve_gromov +from .solvers import solve, solve_gromov, solve_sample from .lowrank import lowrank_sinkhorn # utils functions diff --git a/ot/lowrank.py b/ot/lowrank.py index ed526cf1c..b2e443b74 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -8,11 +8,11 @@ import warnings -from .utils import unif, LazyTensor +from .utils import unif, get_lowrank_lazytensor from .backend import get_backend -def compute_lr_cost_matrix(X_s, X_t, nx=None): +def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): """ Compute low rank decomposition of a sqeuclidean cost matrix. This function won't work for other metrics. @@ -21,8 +21,8 @@ def compute_lr_cost_matrix(X_s, X_t, nx=None): References ---------- - .. Scetbon, M., Cuturi, M., & Peyré, G (2021). - Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. + .. [63] Scetbon, M., Cuturi, M., & Peyré, G (2021). + "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. """ if nx is None: @@ -54,9 +54,9 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No References ---------- - .. Scetbon, M., Cuturi, M., & Peyré, G (2021). - Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. - + .. [63] Scetbon, M., Cuturi, M., & Peyré, G (2021). + "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. + """ # POT backend if None @@ -66,7 +66,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No # ----------------- Initialisation of Dykstra algorithm ----------------- r = len(eps3) # rank - g_ = eps3.copy() # \tilde{g} + g_ = nx.copy(eps3) # \tilde{g} q3_1, q3_2 = nx.ones(r), nx.ones(r) # q^{(3)}_1, q^{(3)}_2 v1_, v2_ = nx.ones(r), nx.ones(r) # \tilde{v}^{(1)}, \tilde{v}^{(2)} q1, q2 = nx.ones(r), nx.ones(r) # q^{(1)}, q^{(2)} @@ -87,7 +87,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No # Compute g, g^{(3)}_1 and update \tilde{g} g = nx.maximum(alpha, g_ * q3_1) q3_1 = (g_ * q3_1) / g - g_ = g.copy() + g_ = nx.copy(g) # Compute new value of g with \prod prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) @@ -104,8 +104,8 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No q3_2 = (g_ * q3_2) / g # Update values of \tilde{v}^{(1)}, \tilde{v}^{(2)} and \tilde{g} - v1_, v2_ = v1.copy(), v2.copy() - g_ = g.copy() + v1_, v2_ = nx.copy(v1), nx.copy(v2) + g_ = nx.copy(g) # Compute error err1 = nx.sum(nx.abs(u1 * (eps1 @ v1) - p1)) @@ -133,7 +133,7 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", - numItermax=10000, stopThr=1e-9, warn=True, shape_plan="auto"): + numItermax=10000, stopThr=1e-9, warn=True, log=False): r''' Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. @@ -179,8 +179,8 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", Stop threshold on error (>0) warn : bool, optional if True, raises a warning if the algorithm doesn't convergence. - shape_plan : tuple - Shape of the lazy_plan + log : bool, optional + record log if True Returns @@ -202,8 +202,8 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", References ---------- - .. Scetbon, M., Cuturi, M., & Peyré, G (2021). - Low-Rank Sinkhorn Factorization. arXiv preprint arXiv:2103.04737. + .. [63] Scetbon, M., Cuturi, M., & Peyré, G (2021). + "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. ''' @@ -230,12 +230,8 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", if 1/r < alpha : raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format(a=alpha,r=1/rank)) - # Default value for shape tensor parameter in LazyTensor - if shape_plan == "auto": - shape_plan = (ns,nt) - # Low rank decomposition of the sqeuclidean cost matrix (A, B) - M1, M2 = compute_lr_cost_matrix(X_s, X_t, nx=None) + M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None) # Compute gamma (see "Section 3.4, proposition 4" in the paper) L = nx.sqrt(3*(2/(alpha**4))*((nx.norm(M1)*nx.norm(M2))**2) + (reg + (2/(alpha**3))*(nx.norm(M1)*nx.norm(M2)))**2) @@ -246,7 +242,6 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", k = 100 # not specified in paper ? - # -------------------------- Low rank algorithm ------------------------------ # see "Section 3.3, Algorithm 3 LOT" in the paper @@ -259,29 +254,27 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", CQ_ = nx.dot(M1.T, Q) CQ = nx.dot(M2, CQ_) - diag_g = nx.diag(1/g) - - eps1 = nx.exp(-gamma*(nx.dot(CR,diag_g)) - ((gamma*reg)-1)*nx.log(Q)) - eps2 = nx.exp(-gamma*(nx.dot(CQ,diag_g)) - ((gamma*reg)-1)*nx.log(R)) + diag_g = (1/g)[None,:] + + eps1 = nx.exp(-gamma*(CR*diag_g) - ((gamma*reg)-1)*nx.log(Q)) + eps2 = nx.exp(-gamma*(CQ*diag_g) - ((gamma*reg)-1)*nx.log(R)) omega = nx.diag(nx.dot(Q.T, CR)) eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) Q, R, g = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx) - + Q = Q+1e-16 + R = R+1e-16 # ----------------- Compute lazy_plan, value and value_linear ------------------ # see "Section 3.2: The Low-rank OT Problem" in the paper # Compute lazy plan (using LazyTensor class) - plan1 = Q - plan2 = nx.dot(nx.diag(1/g),R.T) # low memory cost since shape (r*m) - compute_plan = lambda i,j,P1,P2: nx.dot(P1[i,:], P2[:,j]) # function for LazyTensor - lazy_plan = LazyTensor(shape_plan, compute_plan, P1=plan1, P2=plan2) + lazy_plan = get_lowrank_lazytensor(Q, R, 1/g) # Compute value_linear (using trace formula) v1 = nx.dot(Q.T,M1) - v2 = nx.dot(R,nx.dot(diag_g.T,v1)) + v2 = nx.dot(R,(v1.T*diag_g).T) value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) # Compute value with entropy reg (entropy of Q, R, g must be computed separatly, see "Section 3.2" in the paper) @@ -290,7 +283,15 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R value = value_linear + reg * (reg_Q + reg_g + reg_R) - return value, value_linear, lazy_plan, Q, R, g + if log: + dict_log = dict() + dict_log["value"] = value + dict_log["value_linear"] = value_linear + dict_log["lazy_plan"] = lazy_plan + + return Q, R, g, dict_log + + return Q, R, g diff --git a/ot/solvers.py b/ot/solvers.py index a41762a5c..958b951d1 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -22,6 +22,7 @@ from .partial import partial_gromov_wasserstein2, entropic_partial_gromov_wasserstein2 from .gaussian import empirical_bures_wasserstein_distance from .factored import factored_optimal_transport +from .lowrank import lowrank_sinkhorn lst_method_lazy = ['1d', 'gaussian', 'lowrank', 'factored', 'geomloss', 'geomloss_auto', 'geomloss_tensorized', 'geomloss_online', 'geomloss_multiscale'] @@ -1247,6 +1248,26 @@ def solve_sample(X_a, X_b, a=None, b=None, metric='sqeuclidean', reg=None, reg_t lazy_plan = log['lazy_plan'] if not lazy0: # store plan if not lazy plan = lazy_plan[:] + + elif method == "lowrank": + + if not metric.lower() in ['sqeuclidean']: + raise (NotImplementedError('Not implemented metric="{}"'.format(metric))) + + if max_iter is None: + max_iter = 1000 + if tol is None: + tol = 1e-9 + if reg is None: + reg = 0 + + Q, R, g, log = lowrank_sinkhorn(X_a, X_b, reg=reg, a=a, b=b, numItermax=max_iter, stopThr=tol, log=True) + value = log['value'] + value_linear = log['value_linear'] + lazy_plan = log['lazy_plan'] + if not lazy0: # store plan if not lazy + plan = lazy_plan[:] + elif method.startswith('geomloss'): # Geomloss solver for entropi OT diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 43c3655f4..e3ffe6df3 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -13,13 +13,13 @@ ################################################## WORK IN PROGRESS ####################################################### -def test_compute_lr_cost_matrix(): +def test_compute_lr_sqeuclidean_matrix(): # test computation of low rank cost matrices M1 and M2 n = 100 X_s = np.reshape(1.0 * np.arange(2*n), (n, 2)) X_t = np.reshape(1.0 * np.arange(2*n), (n, 2)) - M1, M2 = ot.lowrank.compute_lr_cost_matrix(X_s, X_t) + M1, M2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X_s, X_t) M = ot.dist(X_s, X_t, metric="sqeuclidean") # original cost matrix np.testing.assert_allclose( @@ -35,8 +35,9 @@ def test_lowrank_sinkhorn(): X_s = np.reshape(1.0 * np.arange(n), (n, 1)) X_t = np.reshape(1.0 * np.arange(n), (n, 1)) - value, value_linear, lazy_plan, Q, R, g = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, 0.1) - P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True) + P = log["lazy_plan"][:] + value_linear = log["value_linear"] # check constraints for P np.testing.assert_allclose(a, P.sum(1), atol=1e-05) @@ -58,7 +59,7 @@ def test_lowrank_sinkhorn(): @pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,6))) -def test_lowrank_sinkhorn_alpha_warning(alpha,rank): +def test_lowrank_sinkhorn_alpha_error(alpha,rank): # Test warning for value of alpha n = 100 a = ot.unif(n) @@ -71,20 +72,20 @@ def test_lowrank_sinkhorn_alpha_warning(alpha,rank): ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) +def test_lowrank_sinkhorn_backends(nx): + # Test low rank sinkhorn for different backends + n = 100 + a = ot.unif(n) + b = ot.unif(n) -# def test_lowrank_sinkhorn_backends(nx): -# # Test low rank sinkhorn for different backends -# n = 100 -# a = ot.unif(n) -# b = ot.unif(n) - -# X_s = np.reshape(1.0 * np.arange(n), (n, 1)) -# X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) -# ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) + ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) -# value, value_linear, lazy_plan, Q, R, g = lowrank_sinkhorn(X_sb, X_tb, ab, bb, reg=0.1) -# P = lazy_plan[:] # default shape for lazy_plan in lowrank_sinkhorn is (ns, nt) + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, reg=0.1, log=True) + lazy_plan = log["lazy_plan"] + P = lazy_plan[:] -# np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) -# np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) \ No newline at end of file + np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) + np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) \ No newline at end of file diff --git a/test/test_solvers.py b/test/test_solvers.py index bf07b7af8..7cb26a096 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -30,12 +30,14 @@ {'method': 'gaussian'}, {'method': 'gaussian', 'reg': 1}, {'method': 'factored', 'rank': 10}, + {'method': 'lowrank', 'reg':0.1} ] lst_parameters_solve_sample_NotImplemented = [ {'method': '1d', 'metric': 'any other one'}, # fail 1d on weird metrics {'method': 'gaussian', 'metric': 'euclidean'}, # fail gaussian on metric not euclidean - {'method': 'factored', 'metric': 'euclidean'}, # fail factored on metric not euclidean + {'method': 'factored', 'metric': 'euclidean'}, # fail factored on metric not euclidean + {"method": 'lowrank', 'metric':'euclidean'}, # fail lowrank on metric not euclidean {'lazy': True}, # fail lazy for non regularized {'lazy': True, 'unbalanced': 1}, # fail lazy for non regularized unbalanced {'lazy': True, 'reg': 1, 'unbalanced': 1}, # fail lazy for unbalanced and regularized @@ -413,7 +415,7 @@ def test_solve_sample_methods(nx, method_params): assert_allclose_sol(sol, solb) sol2 = ot.solve_sample(x, x, **method_params) - if method_params['method'] != 'factored': + if method_params['method'] not in ['factored','lowrank']: np.testing.assert_allclose(sol2.value, 0) From bc7af6b33faa929dc2747a4018d6a9b07a4d71ee Mon Sep 17 00:00:00 2001 From: laudavid Date: Sat, 25 Nov 2023 13:06:07 +0100 Subject: [PATCH 20/36] fix torch backend for lowrank --- ot/lowrank.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index b2e443b74..7b44ad4a0 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -8,7 +8,7 @@ import warnings -from .utils import unif, get_lowrank_lazytensor +from .utils import unif, list_to_array, get_lowrank_lazytensor from .backend import get_backend @@ -33,15 +33,17 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): d = X_s.shape[1] # First low rank decomposition of the cost matrix (A) - M1 = nx.zeros((ns,(d+2))) - M1[:,0] = [nx.norm(X_s[i,:])**2 for i in range(ns)] - M1[:,1] = nx.ones(ns) + M1 = nx.zeros((ns,(d+2)), type_as=X_s) + norm_M1 = list_to_array([nx.norm(X_s[i,:])**2 for i in range(ns)]) + M1[:,0] = nx.from_numpy(norm_M1) + M1[:,1] = nx.ones(ns, type_as=X_s) M1[:,2:] = -2*X_s # Second low rank decomposition of the cost matrix (B) - M2 = nx.zeros((nt,(d+2))) - M2[:,0] = nx.ones(nt) - M2[:,1] = [nx.norm(X_t[i,:])**2 for i in range(nt)] + M2 = nx.zeros((nt,(d+2)), type_as=X_s) + M2[:,0] = nx.ones(nt, type_as=X_s) + norm_M2 = list_to_array([nx.norm(X_t[i,:])**2 for i in range(nt)]) + M2[:,1] = nx.from_numpy(norm_M2) M2[:,2:] = X_t return M1, M2 @@ -67,9 +69,9 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No # ----------------- Initialisation of Dykstra algorithm ----------------- r = len(eps3) # rank g_ = nx.copy(eps3) # \tilde{g} - q3_1, q3_2 = nx.ones(r), nx.ones(r) # q^{(3)}_1, q^{(3)}_2 - v1_, v2_ = nx.ones(r), nx.ones(r) # \tilde{v}^{(1)}, \tilde{v}^{(2)} - q1, q2 = nx.ones(r), nx.ones(r) # q^{(1)}, q^{(2)} + q3_1, q3_2 = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # q^{(3)}_1, q^{(3)}_2 + v1_, v2_ = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # \tilde{v}^{(1)}, \tilde{v}^{(2)} + q1, q2 = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # q^{(1)}, q^{(2)} err = 1 # initial error @@ -238,7 +240,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", gamma = 1/(2*L) # Initialize the low rank matrices Q, R, g - Q, R, g = nx.ones((ns,r)), nx.ones((nt,r)), nx.ones(r) + Q, R, g = nx.ones((ns,r), type_as=a), nx.ones((nt,r), type_as=a), nx.ones(r, type_as=a) k = 100 # not specified in paper ? @@ -295,5 +297,3 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", - - From b40705c463d363777f1d79c94bebdc14e8e1c33d Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 28 Nov 2023 13:41:10 +0100 Subject: [PATCH 21/36] fix jax backend and skip tf --- ot/lowrank.py | 185 ++++++++++++++++++++----------------------- test/test_lowrank.py | 53 ++++++------- 2 files changed, 113 insertions(+), 125 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index 7b44ad4a0..ad804efd8 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -14,43 +14,37 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): """ - Compute low rank decomposition of a sqeuclidean cost matrix. - This function won't work for other metrics. + Compute low rank decomposition of a sqeuclidean cost matrix. + This function won't work for other metrics. See "Section 3.5, proposition 1" of the paper References ---------- .. [63] Scetbon, M., Cuturi, M., & Peyré, G (2021). - "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. + "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. """ if nx is None: - nx = get_backend(X_s,X_t) - + nx = get_backend(X_s, X_t) + ns = X_s.shape[0] nt = X_t.shape[0] - d = X_s.shape[1] # First low rank decomposition of the cost matrix (A) - M1 = nx.zeros((ns,(d+2)), type_as=X_s) - norm_M1 = list_to_array([nx.norm(X_s[i,:])**2 for i in range(ns)]) - M1[:,0] = nx.from_numpy(norm_M1) - M1[:,1] = nx.ones(ns, type_as=X_s) - M1[:,2:] = -2*X_s + array1 = nx.reshape(nx.sum(X_s**2, 1), (-1, 1)) + array2 = nx.reshape(nx.ones(ns, type_as=X_s), (-1, 1)) + M1 = nx.concatenate((array1, array2, -2 * X_s), axis=1) # Second low rank decomposition of the cost matrix (B) - M2 = nx.zeros((nt,(d+2)), type_as=X_s) - M2[:,0] = nx.ones(nt, type_as=X_s) - norm_M2 = list_to_array([nx.norm(X_t[i,:])**2 for i in range(nt)]) - M2[:,1] = nx.from_numpy(norm_M2) - M2[:,2:] = X_t + array1 = nx.reshape(nx.ones(nt, type_as=X_s), (-1, 1)) + array2 = nx.reshape(nx.sum(X_t**2, 1), (-1, 1)) + M2 = nx.concatenate((array1, array2, X_t), axis=1) return M1, M2 - -def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=None): +def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=None): """ Implementation of the Dykstra algorithm for the Low Rank sinkhorn OT solver. @@ -58,30 +52,27 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No ---------- .. [63] Scetbon, M., Cuturi, M., & Peyré, G (2021). "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. - + """ # POT backend if None if nx is None: nx = get_backend(eps1, eps2, eps3, p1, p2) - # ----------------- Initialisation of Dykstra algorithm ----------------- - r = len(eps3) # rank - g_ = nx.copy(eps3) # \tilde{g} - q3_1, q3_2 = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # q^{(3)}_1, q^{(3)}_2 - v1_, v2_ = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # \tilde{v}^{(1)}, \tilde{v}^{(2)} - q1, q2 = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # q^{(1)}, q^{(2)} - err = 1 # initial error - + r = len(eps3) # rank + g_ = nx.copy(eps3) # \tilde{g} + q3_1, q3_2 = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # q^{(3)}_1, q^{(3)}_2 + v1_, v2_ = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # \tilde{v}^{(1)}, \tilde{v}^{(2)} + q1, q2 = nx.ones(r, type_as=p1), nx.ones(r, type_as=p1) # q^{(1)}, q^{(2)} + err = 1 # initial error # --------------------- Dykstra algorithm ------------------------- - + # See Section 3.3 - "Algorithm 2 LR-Dykstra" in paper - - for ii in range(numItermax): - if err > stopThr: + for ii in range(numItermax): + if err > stopThr: # Compute u^{(1)} and u^{(2)} u1 = p1 / nx.dot(eps1, v1_) u2 = p2 / nx.dot(eps2, v2_) @@ -92,19 +83,19 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No g_ = nx.copy(g) # Compute new value of g with \prod - prod1 = ((v1_ * q1) * nx.dot(eps1.T, u1)) - prod2 = ((v2_ * q2) * nx.dot(eps2.T, u2)) - g = (g_ * q3_2 * prod1 * prod2)**(1/3) + prod1 = (v1_ * q1) * nx.dot(eps1.T, u1) + prod2 = (v2_ * q2) * nx.dot(eps2.T, u2) + g = (g_ * q3_2 * prod1 * prod2) ** (1 / 3) # Compute v^{(1)} and v^{(2)} - v1 = g / nx.dot(eps1.T,u1) - v2 = g / nx.dot(eps2.T,u2) + v1 = g / nx.dot(eps1.T, u1) + v2 = g / nx.dot(eps2.T, u2) # Compute q^{(1)}, q^{(2)} and q^{(3)}_2 q1 = (v1_ * q1) / v1 q2 = (v2_ * q2) / v2 q3_2 = (g_ * q3_2) / g - + # Update values of \tilde{v}^{(1)}, \tilde{v}^{(2)} and \tilde{g} v1_, v2_ = nx.copy(v1), nx.copy(v2) g_ = nx.copy(g) @@ -116,36 +107,32 @@ def LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=No else: break - - else: + + else: if warn: - warnings.warn("Sinkhorn did not converge. You might want to " - "increase the number of iterations `numItermax` ") + warnings.warn( + "Sinkhorn did not converge. You might want to " + "increase the number of iterations `numItermax` " + ) # Compute low rank matrices Q, R - Q = u1[:,None] * eps1 * v1[None,:] - R = u2[:,None] * eps2 * v2[None,:] + Q = u1[:, None] * eps1 * v1[None, :] + R = u2[:, None] * eps2 * v2[None, :] return Q, R, g - - -#################################### LOW RANK SINKHORN ALGORITHM ######################################### - - -def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", - numItermax=10000, stopThr=1e-9, warn=True, log=False): - - r''' +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", + numItermax=1000, stopThr=1e-9, warn=True, log=False): + r""" Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. The function solves the following optimization problem: .. math:: - \mathop{\inf_{(Q,R,g) \in \mathcal{C(a,b,r)}}} \langle C, Q\mathrm{diag}(1/g)R^T \rangle - + \mathop{\inf_{(Q,R,g) \in \mathcal{C(a,b,r)}}} \langle C, Q\mathrm{diag}(1/g)R^T \rangle - \mathrm{reg} \cdot H((Q,R,g)) - + where : - :math:`C` is the (`dim_a`, `dim_b`) metric cost matrix - :math:`H((Q,R,g))` is the values of the three respective entropies evaluated for each term. @@ -153,11 +140,11 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", - :math: `g` is the weight vector for the low-rank decomposition of the OT plan - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target weights (histograms, both sum to 1) - :math: `r` is the rank of the OT plan - - :math: `\mathcal{C(a,b,r)}` are the low-rank couplings of the OT problem - \mathcal{C(a,b,r)} = \mathcal{C_1(a,b,r)} \cap \mathcal{C_2(r)} with + - :math: `\mathcal{C(a,b,r)}` are the low-rank couplings of the OT problem + \mathcal{C(a,b,r)} = \mathcal{C_1(a,b,r)} \cap \mathcal{C_2(r)} with \mathcal{C_1(a,b,r)} = \{ (Q,R,g) s.t Q\mathbb{1}_r = a, R^T \mathbb{1}_m = b \} \mathcal{C_2(r)} = \{ (Q,R,g) s.t Q\mathbb{1}_n = R^T \mathbb{1}_m = g \} - + Parameters ---------- @@ -184,30 +171,30 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", log : bool, optional record log if True - + Returns ------- lazy_plan : LazyTensor() - OT plan in a LazyTensor object of shape (shape_plan) + OT plan in a LazyTensor object of shape (shape_plan) See :any:`LazyTensor` for more information. value : float - Optimal value of the optimization problem, + Optimal value of the optimization problem value_linear : float - Linear OT loss with the optimal OT + Linear OT loss with the optimal OT Q : array-like, shape (n_samples_a, r) - First low-rank matrix decomposition of the OT plan + First low-rank matrix decomposition of the OT plan R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) Weight vector for the low-rank decomposition of the OT plan - - + + References ---------- .. [63] Scetbon, M., Cuturi, M., & Peyré, G (2021). "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. - ''' + """ # POT backend nx = get_backend(X_s, X_t) @@ -223,66 +210,72 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", r = rank if rank == "auto": r = min(ns, nt) - + if alpha == "auto": alpha = 1e-10 # Dykstra algorithm won't converge if 1/rank < alpha (alpha is the lower bound for 1/rank) # (see "Section 3.2: The Low-rank OT Problem (LOT)" in the paper) - if 1/r < alpha : - raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format(a=alpha,r=1/rank)) + if 1 / r < alpha: + raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format( + a=alpha, r=1 / rank)) # Low rank decomposition of the sqeuclidean cost matrix (A, B) M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None) # Compute gamma (see "Section 3.4, proposition 4" in the paper) - L = nx.sqrt(3*(2/(alpha**4))*((nx.norm(M1)*nx.norm(M2))**2) + (reg + (2/(alpha**3))*(nx.norm(M1)*nx.norm(M2)))**2) - gamma = 1/(2*L) - - # Initialize the low rank matrices Q, R, g - Q, R, g = nx.ones((ns,r), type_as=a), nx.ones((nt,r), type_as=a), nx.ones(r, type_as=a) - k = 100 # not specified in paper ? - + L = nx.sqrt( + 3 * (2 / (alpha**4)) * ((nx.norm(M1) * nx.norm(M2)) ** 2) + + (reg + (2 / (alpha**3)) * (nx.norm(M1) * nx.norm(M2))) ** 2 + ) + gamma = 1 / (2 * L) + + # Initialize the low rank matrices Q, R, g + Q = nx.ones((ns, r), type_as=a) + R = nx.ones((nt, r), type_as=a) + g = nx.ones(r, type_as=a) + k = 100 # -------------------------- Low rank algorithm ------------------------------ # see "Section 3.3, Algorithm 3 LOT" in the paper - for ii in range(k): + for ii in range(k): # Compute the C*R dot matrix using the lr decomposition of C CR_ = nx.dot(M2.T, R) - CR = nx.dot(M1, CR_) - + CR = nx.dot(M1, CR_) + # Compute the C.t * Q dot matrix using the lr decomposition of C CQ_ = nx.dot(M1.T, Q) CQ = nx.dot(M2, CQ_) - - diag_g = (1/g)[None,:] - eps1 = nx.exp(-gamma*(CR*diag_g) - ((gamma*reg)-1)*nx.log(Q)) - eps2 = nx.exp(-gamma*(CQ*diag_g) - ((gamma*reg)-1)*nx.log(R)) - omega = nx.diag(nx.dot(Q.T, CR)) - eps3 = nx.exp(gamma*omega/(g**2) - (gamma*reg - 1)*nx.log(g)) + diag_g = (1 / g)[None, :] - Q, R, g = LR_Dysktra(eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx) - Q = Q+1e-16 - R = R+1e-16 + eps1 = nx.exp(-gamma * (CR * diag_g) - ((gamma * reg) - 1) * nx.log(Q)) + eps2 = nx.exp(-gamma * (CQ * diag_g) - ((gamma * reg) - 1) * nx.log(R)) + omega = nx.diag(nx.dot(Q.T, CR)) + eps3 = nx.exp(gamma * omega / (g**2) - (gamma * reg - 1) * nx.log(g)) + Q, R, g = LR_Dysktra( + eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx + ) + Q = Q + 1e-16 + R = R + 1e-16 # ----------------- Compute lazy_plan, value and value_linear ------------------ # see "Section 3.2: The Low-rank OT Problem" in the paper # Compute lazy plan (using LazyTensor class) - lazy_plan = get_lowrank_lazytensor(Q, R, 1/g) - + lazy_plan = get_lowrank_lazytensor(Q, R, 1 / g) + # Compute value_linear (using trace formula) - v1 = nx.dot(Q.T,M1) - v2 = nx.dot(R,(v1.T*diag_g).T) + v1 = nx.dot(Q.T, M1) + v2 = nx.dot(R, (v1.T * diag_g).T) value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) # Compute value with entropy reg (entropy of Q, R, g must be computed separatly, see "Section 3.2" in the paper) - reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q - reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g - reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R + reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q + reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g + reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R value = value_linear + reg * (reg_Q + reg_g + reg_R) if log: @@ -290,10 +283,8 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", dict_log["value"] = value dict_log["value_linear"] = value_linear dict_log["lazy_plan"] = lazy_plan - + return Q, R, g, dict_log return Q, R, g - - diff --git a/test/test_lowrank.py b/test/test_lowrank.py index e3ffe6df3..65f76a77b 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -1,29 +1,24 @@ -##################################################################################################### -####################################### WORK IN PROGRESS ############################################ -##################################################################################################### - - """ Test for low rank sinkhorn solvers """ +# Author: Laurène DAVID +# +# License: MIT License + import ot import numpy as np import pytest - -################################################## WORK IN PROGRESS ####################################################### - def test_compute_lr_sqeuclidean_matrix(): # test computation of low rank cost matrices M1 and M2 n = 100 - X_s = np.reshape(1.0 * np.arange(2*n), (n, 2)) - X_t = np.reshape(1.0 * np.arange(2*n), (n, 2)) + X_s = np.reshape(1.0 * np.arange(2 * n), (n, 2)) + X_t = np.reshape(1.0 * np.arange(2 * n), (n, 2)) M1, M2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X_s, X_t) - M = ot.dist(X_s, X_t, metric="sqeuclidean") # original cost matrix + M = ot.dist(X_s, X_t, metric="sqeuclidean") # original cost matrix - np.testing.assert_allclose( - np.dot(M1,M2.T), M, atol=1e-05) + np.testing.assert_allclose(np.dot(M1, M2.T), M, atol=1e-05) def test_lowrank_sinkhorn(): @@ -40,38 +35,40 @@ def test_lowrank_sinkhorn(): value_linear = log["value_linear"] # check constraints for P - np.testing.assert_allclose(a, P.sum(1), atol=1e-05) - np.testing.assert_allclose(b, P.sum(0), atol=1e-05) - + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + # check if lazy_plan is equal to the fully computed plan - P_true = np.dot(Q,np.dot(np.diag(1/g),R.T)) + P_true = np.dot(Q, np.dot(np.diag(1 / g), R.T)) np.testing.assert_allclose(P, P_true, atol=1e-05) # check if value_linear is correct with its original formula M = ot.dist(X_s, X_t, metric="sqeuclidean") value_linear_true = np.sum(M * P_true) np.testing.assert_allclose(value_linear, value_linear_true, atol=1e-05) - - # check warn parameter when Dykstra algorithm doesn't converge + + # check warn parameter when Dykstra algorithm doesn't converge with pytest.warns(UserWarning): ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1) - -@pytest.mark.parametrize(("alpha, rank"),((0.8,2),(0.5,3),(0.2,6))) -def test_lowrank_sinkhorn_alpha_error(alpha,rank): - # Test warning for value of alpha +@pytest.mark.parametrize(("alpha, rank"), ((0.8, 2), (0.5, 3), (0.2, 6))) +def test_lowrank_sinkhorn_alpha_error(alpha, rank): + # Test warning for value of alpha n = 100 a = ot.unif(n) b = ot.unif(n) X_s = np.reshape(1.0 * np.arange(n), (n, 1)) X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) - + with pytest.raises(ValueError): - ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) + ot.lowrank.lowrank_sinkhorn( + X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False + ) +@pytest.skip_backend('tf') def test_lowrank_sinkhorn_backends(nx): # Test low rank sinkhorn for different backends n = 100 @@ -85,7 +82,7 @@ def test_lowrank_sinkhorn_backends(nx): Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_sb, X_tb, ab, bb, reg=0.1, log=True) lazy_plan = log["lazy_plan"] - P = lazy_plan[:] + P = lazy_plan[:] - np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) - np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) \ No newline at end of file + np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) + np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) From 55c8d2bafcd01ba0fb52b65f5324b165d2f0ff35 Mon Sep 17 00:00:00 2001 From: laudavid Date: Tue, 28 Nov 2023 22:06:28 +0100 Subject: [PATCH 22/36] fix pep 8 tests --- ot/lowrank.py | 3 +-- ot/solvers.py | 7 +++---- ot/utils.py | 3 ++- test/test_solvers.py | 8 ++++---- test/test_utils.py | 1 - 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index ad804efd8..365d78ed9 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -8,7 +8,7 @@ import warnings -from .utils import unif, list_to_array, get_lowrank_lazytensor +from .utils import unif, get_lowrank_lazytensor from .backend import get_backend @@ -287,4 +287,3 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank="auto", alpha="auto", return Q, R, g, dict_log return Q, R, g - diff --git a/ot/solvers.py b/ot/solvers.py index 958b951d1..40a03e974 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -1248,19 +1248,19 @@ def solve_sample(X_a, X_b, a=None, b=None, metric='sqeuclidean', reg=None, reg_t lazy_plan = log['lazy_plan'] if not lazy0: # store plan if not lazy plan = lazy_plan[:] - + elif method == "lowrank": if not metric.lower() in ['sqeuclidean']: raise (NotImplementedError('Not implemented metric="{}"'.format(metric))) - + if max_iter is None: max_iter = 1000 if tol is None: tol = 1e-9 if reg is None: reg = 0 - + Q, R, g, log = lowrank_sinkhorn(X_a, X_b, reg=reg, a=a, b=b, numItermax=max_iter, stopThr=tol, log=True) value = log['value'] value_linear = log['value_linear'] @@ -1268,7 +1268,6 @@ def solve_sample(X_a, X_b, a=None, b=None, metric='sqeuclidean', reg=None, reg_t if not lazy0: # store plan if not lazy plan = lazy_plan[:] - elif method.startswith('geomloss'): # Geomloss solver for entropi OT split_method = method.split('_') diff --git a/ot/utils.py b/ot/utils.py index 3e67a08b4..cb29b21c9 100644 --- a/ot/utils.py +++ b/ot/utils.py @@ -1176,6 +1176,7 @@ def citation(self): } """ + class LazyTensor(object): """ A lazy tensor is a tensor that is not stored in memory. Instead, it is defined by a function that computes its values on the fly from slices. @@ -1240,4 +1241,4 @@ def __getitem__(self, key): return self._getitem(*k, **self.kwargs) def __repr__(self): - return "LazyTensor(shape={},attributes=({}))".format(self.shape, ','.join(self.kwargs.keys())) \ No newline at end of file + return "LazyTensor(shape={},attributes=({}))".format(self.shape, ','.join(self.kwargs.keys())) diff --git a/test/test_solvers.py b/test/test_solvers.py index 7cb26a096..343220c45 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -30,14 +30,14 @@ {'method': 'gaussian'}, {'method': 'gaussian', 'reg': 1}, {'method': 'factored', 'rank': 10}, - {'method': 'lowrank', 'reg':0.1} + {'method': 'lowrank', 'reg': 0.1} ] lst_parameters_solve_sample_NotImplemented = [ {'method': '1d', 'metric': 'any other one'}, # fail 1d on weird metrics {'method': 'gaussian', 'metric': 'euclidean'}, # fail gaussian on metric not euclidean - {'method': 'factored', 'metric': 'euclidean'}, # fail factored on metric not euclidean - {"method": 'lowrank', 'metric':'euclidean'}, # fail lowrank on metric not euclidean + {'method': 'factored', 'metric': 'euclidean'}, # fail factored on metric not euclidean + {"method": 'lowrank', 'metric': 'euclidean'}, # fail lowrank on metric not euclidean {'lazy': True}, # fail lazy for non regularized {'lazy': True, 'unbalanced': 1}, # fail lazy for non regularized unbalanced {'lazy': True, 'reg': 1, 'unbalanced': 1}, # fail lazy for unbalanced and regularized @@ -415,7 +415,7 @@ def test_solve_sample_methods(nx, method_params): assert_allclose_sol(sol, solb) sol2 = ot.solve_sample(x, x, **method_params) - if method_params['method'] not in ['factored','lowrank']: + if method_params['method'] not in ['factored', 'lowrank']: np.testing.assert_allclose(sol2.value, 0) diff --git a/test/test_utils.py b/test/test_utils.py index 4a6dec9cf..258a1c742 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -583,4 +583,3 @@ def test_lowrank_LazyTensor(nx): T = ot.utils.get_lowrank_lazytensor(X1, X2, diag_d, nx=nx) np.testing.assert_allclose(nx.to_numpy(T[:]), nx.to_numpy(T0)) - \ No newline at end of file From e25c91db6c846d5f28b6076932b94a048e2f9630 Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 6 Dec 2023 10:13:07 +0100 Subject: [PATCH 23/36] add lowrank init + test functions --- ot/lowrank.py | 143 ++++++++++++++++++++++++++++++++++++++++--- test/test_lowrank.py | 22 ++++++- 2 files changed, 154 insertions(+), 11 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index 5c8f673cb..d87fd913f 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -8,8 +8,126 @@ import warnings -from .utils import unif, get_lowrank_lazytensor +from .utils import unif, dist, get_lowrank_lazytensor from .backend import get_backend +from .bregman import sinkhorn +from sklearn.cluster import KMeans + +# ADD FUNCTION FOR LOW RANK INIT [WIP] + + +def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=None, nx=None): + """ + Implementation of different initialization strategies for the low rank sinkhorn solver (Q ,R, g). + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + a : array-like, shape (n_samples_a,) + samples weights in the source domain + b : array-like, shape (n_samples_b,) + samples weights in the target domain + rank : int, optional. Default is None. (>0) + Nonnegative rank of the OT plan. + init : str, default is 'kmeans' + Initialization strategy for Q, R and g. 'random', 'trivial' or 'kmeans' + reg_init : float, optional. Default is None. (>0) + Regularization term for a 'kmeans' init. If None, 1 is considered. + random_state : default None + Random state for a "random" or 'kmeans' init strategy + nx : default None + POT backend + + + Returns + --------- + Q : array-like, shape (n_samples_a, r) + Init for the first low-rank matrix decomposition of the OT plan (Q) + R: array-like, shape (n_samples_b, r) + Init for the second low-rank matrix decomposition of the OT plan (R) + g : array-like, shape (r, ) + Init for the weight vector of the low-rank decomposition of the OT plan (g) + + """ + + if nx is None: + nx = get_backend(X_s, X_t, a, b) + + if reg_init is None: + reg_init = 0.1 + + ns = X_s.shape[0] + nt = X_t.shape[0] + r = rank + + if init == "random": + nx.seed(seed=random_state) + + # Init g + g = nx.abs(nx.randn(r, type_as=X_s)) + 1 + g = g / nx.sum(g) + + # Init Q + Q = nx.abs(nx.randn(ns, r, type_as=X_s)) + 1 + Q = (Q.T * (a / nx.sum(Q, axis=1))).T + + # Init R + R = nx.abs(nx.randn(nt, rank, type_as=X_s)) + 1 + R = (R.T * (b / nx.sum(R, axis=1))).T + + if init == "trivial": + # Init g + g = nx.ones(rank) / rank + + lambda_1 = min(nx.min(a), nx.min(g), nx.min(b)) / 2 + a1 = nx.arange(start=1, stop=ns + 1, type_as=X_s) + a1 = a1 / nx.sum(a1) + a2 = (a - lambda_1 * a1) / (1 - lambda_1) + + b1 = nx.arange(start=1, stop=nt + 1, type_as=X_s) + b1 = b1 / nx.sum(b1) + b2 = (b - lambda_1 * b1) / (1 - lambda_1) + + g1 = nx.arange(start=1, stop=rank + 1, type_as=X_s) + g1 = g1 / nx.sum(g1) + g2 = (g - lambda_1 * g1) / (1 - lambda_1) + + # Init Q + Q1 = lambda_1 * nx.dot(a1[:, None], nx.reshape(g1, (1, -1))) + Q2 = (1 - lambda_1) * nx.dot(a2[:, None], nx.reshape(g2, (1, -1))) + Q = Q1 + Q2 + + # Init R + R1 = lambda_1 * nx.dot(b1[:, None], nx.reshape(g1, (1, -1))) + R2 = (1 - lambda_1) * nx.dot(b2[:, None], nx.reshape(g2, (1, -1))) + R = R1 + R2 + + if init == "kmeans": + # Init g + g = nx.ones(rank, type_as=X_s) / rank + + # Init Q + kmeans_Xs = KMeans(n_clusters=rank, random_state=random_state, n_init="auto") + kmeans_Xs.fit(X_s) + Z_Xs = nx.from_numpy(kmeans_Xs.cluster_centers_) + C_Xs = dist(X_s, Z_Xs) # shape (ns, rank) + C_Xs = C_Xs / nx.max(C_Xs) + Q = sinkhorn(a, g, C_Xs, reg=reg_init, numItermax=10000, stopThr=1e-3) + + # Init R + kmeans_Xt = KMeans(n_clusters=rank, random_state=random_state, n_init="auto") + kmeans_Xt.fit(X_t) + Z_Xt = nx.from_numpy(kmeans_Xt.cluster_centers_) + C_Xt = dist(X_t, Z_Xt) # shape (nt, rank) + C_Xt = C_Xt / nx.max(C_Xt) + R = sinkhorn(b, g, C_Xt, reg=reg_init, numItermax=10000, stopThr=1e-3) + + return Q, R, g + +################################################################################## def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): @@ -25,7 +143,8 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): samples in the source domain X_t : array-like, shape (n_samples_b, dim) samples in the target domain - nx : POT backend, default none + nx : default None + POT backend Returns @@ -175,6 +294,7 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, + init="kmeans", reg_init=None, seed_init=None, numItermax=1000, stopThr=1e-9, warn=True, log=False): r""" Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. @@ -207,10 +327,16 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, samples weights in the target domain reg : float, optional Regularization term >0 - rank: int, optional. Default is None. (>0) + rank : int, optional. Default is None. (>0) Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. - alpha: int, optional. Default is None. (>0 and <1/r) + alpha : int, optional. Default is None. (>0 and <1/r) Lower bound for the weight vector g. If None, 1e-10 is considered + init : str, optional. Default is 'kmeans' + Initialization strategy for Q, R, g. 'random', 'trivial' or 'kmeans' + reg_init : float, optional. Default is None. (>0) + Regularization term for a 'kmeans' init. If None, 1 is considered. + seed_init : int, optional. Default is None. (>0) + Random state for a 'random' or 'kmeans' init strategy. numItermax : int, optional Max number of iterations stopThr : float, optional @@ -222,7 +348,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, Returns - ------- + --------- lazy_plan : LazyTensor() OT plan in a LazyTensor object of shape (shape_plan) See :any:`LazyTensor` for more information. @@ -282,10 +408,11 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, ) gamma = 1 / (2 * L) + if reg_init is None: + reg_init = 1 + # Initialize the low rank matrices Q, R, g - Q = nx.ones((ns, r), type_as=a) - R = nx.ones((nt, r), type_as=a) - g = nx.ones(r, type_as=a) + Q, R, g = _init_lr_sinkhorn(X_s, X_t, a, b, r, init, reg_init, seed_init, nx=nx) k = 100 # -------------------------- Low rank algorithm ------------------------------ diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 65f76a77b..e6c95e3de 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -52,6 +52,24 @@ def test_lowrank_sinkhorn(): ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1) +@pytest.mark.parametrize(("init"), ("random", "trivial", "kmeans")) +def test_lowrank_sinkhorn_init(init): + # test lowrank inits + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(n), (n, 1)) + + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True, init=init, reg_init=1) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + @pytest.mark.parametrize(("alpha, rank"), ((0.8, 2), (0.5, 3), (0.2, 6))) def test_lowrank_sinkhorn_alpha_error(alpha, rank): # Test warning for value of alpha @@ -63,9 +81,7 @@ def test_lowrank_sinkhorn_alpha_error(alpha, rank): X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) with pytest.raises(ValueError): - ot.lowrank.lowrank_sinkhorn( - X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False - ) + ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) @pytest.skip_backend('tf') From 0436e956d2a41a25e4d05c5aa8e1569688b3dc15 Mon Sep 17 00:00:00 2001 From: laudavid Date: Mon, 18 Dec 2023 18:02:03 +0100 Subject: [PATCH 24/36] Add init strategies in lowrank + example (PythonOT#588) --- README.md | 3 +- docs/source/_static/images/logo_hiparis.png | Bin 0 -> 78752 bytes examples/others/plot_lowrank_sinkhorn.py | 121 ++++++++++++++++ ot/lowrank.py | 150 ++++++++++++-------- ot/solvers.py | 10 +- test/test_lowrank.py | 24 +++- test/test_solvers.py | 2 +- 7 files changed, 244 insertions(+), 66 deletions(-) create mode 100644 docs/source/_static/images/logo_hiparis.png create mode 100644 examples/others/plot_lowrank_sinkhorn.py diff --git a/README.md b/README.md index a9a94c53e..3c9b212ce 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,8 @@ The numerous contributors to this library are listed [here](CONTRIBUTORS.md). POT has benefited from the financing or manpower from the following partners: -ANRCNRS3IA +ANRCNRS3IAHi!PARIS + ## Contributions and code of conduct diff --git a/docs/source/_static/images/logo_hiparis.png b/docs/source/_static/images/logo_hiparis.png new file mode 100644 index 0000000000000000000000000000000000000000..1ce6dfb5a33d1e49b24555b89445c1985cdafcf6 GIT binary patch literal 78752 zcmeFZ_dnI|A3y#i6d9F7W|EN+vNtJ28Ie7+cV^Z>2}M?Bb|DJcWRp1dF*1*ld2sAw zALDx+z22YC?fd;3zSl3#?Rl>2dS2IK-XG(5udb>{MnX#hK@izvrH9WT=wcrP5ei%) z0#DLj)%k%xH{BoUx@$UHxxX@VwS?p>oL^YpdhBRsZTZa7%);BH!%`B$6L|3Wp`4c2 z`0AwEv)g0i7yn{BCJuK~ZmK35%jt1((DU8nGL)SmQD%QKbeA$;=T-$IZ{ICzM`&4? zx%_@+)kkL&=G~tCh;Ts-|5FF%VK#@Ia>J)vn_ZanE8qnG|M$);)hj;>tY|8?Il;y)v1EZMn16u(=-+SB>(g0J}tA z%e(-^jZW^#kzPPElXc3xJ$}=cm%$_Rk(J24N@70|_S~8f?k=DwZ4DR7A5ux!=ooms zzCqE62NpZf;)z4D9TSdBtD%D2QHd2)XNUe?q@zCg!miSlYO&$GW=9&c+-h~sXP+kE zYVr7B&FCF`cPQ}MPo%UB8#R+WZ8CL-hf>Rqy_xuB`e}Z?MiN^Q!Z6mzhca3?ku%l2 z)Q1O=aHhZ|BikZAd*4);_x!kk>^-qWOqsWtpWM!NMBe>c)Nyj8dlO6#e0odvBE*qY z@W_|67tK2P*1t4$ZNkKJ_32m9sg#b^VCCym>9OYLGdEMNLC|%1(rd8`Me3*8!xa|Q z?~osZvo@0{Svi6c%3u=YfX@P6KfLOM%}h5;W$$u&BvQikLCJc%Bp~bBLi9vvy*~k% z#)-HD?j!_QkjJV*@5p%J_auq49ZxpPK!DZjyQDC!H_xQ3s?TuOkKZpUo#ja=UKKuP&o0!#D&qzF6%?>jQ!{c&X+I0YqMbYg zCU^ewg`h$^P8P;BO8WOh&oqOdy+4~%jmPmYuk=^tON4%Mo99>j&o6A)x)7^xi86Jg z5?!X|>7wF;y$g=H#@H2=o8MyX1+*HCu;sJu*!*m^k_v2stL$Arb=<8oS4hP0rRY9U zO&ji6+3~mbzhJNotdP-mB9t?XXr`9^C#X@%f*ihq%{PQ7(Dq885) zZ|o{FsflBDW+#aVzLA$P3AMr(d1gbx8SfaOxpsZaTb@H5>Bxg=v9XsAe+B4X>wb6C$S#;`CpKG%f;L%)LS?eU>!&U761kACF$9zS{g-^>_=OpzdwPhCr8*P8Shc}ox+dEAoJ zr;9HeAKM)YNOS;E1H#N2c0DtoTteB_gxJ)h^asVS?Cr76ac|8Ntc}1W1m-#_vhi6+ zr_VV_p|Gz?cs`Ts963CW6V2wVnq$x3*ma#UksuN`f99t73IVyE`i>($^hom^bmD}S z%~N;UV=Cyq$XdL?k-l2}A!eYlrH4 z@;fY+xLHubHx9g-tKAD$XRonz&iQxcEQt|1sEI%P#oCL77r_M14G~rdNSk)uZU{g@RUI@Dzf+s|J!oe4({#K_~MMRJMk~vrc+Ma7G5=4u%*YQ-!fQ6g6jw zaBYXQ=yM#yps!!0ioAvY=u<+-l_8vQ`}t5(eB8-8gW8zf=kt ziD8rsrqw>@)x*vcYHwP41^9UR#aAXerbgTkMWt_()$Lgn)5P4tL2}fwxKAWCXpDg71 zU3#d=4o5@<3rvRni4oaiNhm##L+LzxsP1$BcxQU(x>XUE^an|iNqy7$7R1rmwKLu` zl4W8i3Vcr}@EEr?9UQ`vbWx7_2yZjW{ST_+>n|+W^gQ_yV*q6`FwZd0jvK+AW={)Qmp1=Q=syGyUz;-! zoXv+%i8&Wuv2v6*T!ZXgQ|D@Wh3~ypRt!7)G&s(vJaQyGc`EQKD=d7H^F`OG0$Qd+ zN2gl&U%4qhqadLYBoA?Ip4;9n9)%==P`DDrFI3JOXqC-8Fl zoYHJaP>gNchh;<=#oMuT-Cwm7$b3U8Ce0ti_?xK5HoVjpnI|zexINU{`yi?r=ZHCI`7UY#sA_(lLEqQ_h*OW|S z@aWy!Yd}(quj00ZpzgVi;1Tt8^&g4c?3a)INj3KqT)PGWJ#O$zMzYHvryFUX-TAG6 z7My2FowkCvMiqCHQ%!pEVhds&Rd}-Hj^M>!))5*Nt28o?PZj7RPJ8m)AyZ16kTMR% zXZoyHQrO`T5KsZ(HuEO&aJZ=;!zYHIQAE6NWNNzM%d6XT6%6ti=b)oP^xNp24Ns!3 zxrT7|i}T*MACAvTF~px*D=Fs4{4B$Rp5FnAow;K7d1B4%^Ch5Fra{@wHtgB29i+0e zqR}~1Uj=m|o4pxX%FSW3#1{kDDr$H(8^ZB?NC#5HWU8boqO#WTAWjs{{z8Ar6q4{9 zzKbSHISK_Bjyu|vzpy#g(Up{pI-@{*T!_kM3k|-EFj=ofHm`lCdNsPv00oNR3;c%122HHYQv6q+sjheOn zcv3{@tj&>=MC8M$!bvIi3lI&?EP~O=leQsAgi^d*ki#@9xlv+-{kl{s!R**Iov|fy zwgyb>cV%#D*&$m>a!B2hUy@>mO)nYDrXJYz!=y`mMIQUNp?Vgjrq$go;xU>)9R}s{wH& z&h#7orb{0`s=0f zA>qh#b#J&fngPJ8R@M+{cqnR7GDCs(3z*^I0cM2ZX53{-$j>Pv2u6|VL8jsn+74s> zB~7$1RJP57Q{}3ps;!n!0raCkx0;KGh0y?rrgQ#8;6YSNX50hl=F5|_6IMPM8{bHK-_}q~&%T%OZ-y|_KdIfjct3YK={ziQM zzH}E4k|h+>SL}eF5Dvd&s9CcQHk1{2ou1f&niQ_!-0mOEH_Ch&QjqMJkbcGdTnE1W zmE*-6WMomSfDk(aQRUCR(4*$aM~j$bs0akyQ9l-hJFu0lm#}&G*l+KLBcH9kyxJK%p-G zSa`ymcuVT;=b|f+>~kDT+bEk7QtshL7a6cZrmc}5b1M5V&F;NU2`rHh0&+CWi6&MVQW&ItoLe%fy{^f9W3aOB0h8w4$~0r5yO zMV;P{l-_mccy+u$<$Y`#R<;Y6B=_EKVLg z(k#5yLtIz$&m#^9iqjV07T!PHJd^-d^eIjg!oW8)>2$5n$QNq}Adad>$SRXiCuBzv zKyNtYu~}Kv%(3%D;E4X5#y;C6m4EdcxbFN9L4hxSg=67MZcv~rHnm0kNBmVN@DYws zUnvzo@5g_RJ{82R-Pxl)hOHs-p!Rc2NHWH;V;;|u>aV2c1SO!gi6Px&1+Ea30pLsM z1AHj@+_T34%9yLPrz>CYbn$e(P&@&b*%yEkmd(l0o07n|8RBfVl^e}~Se0HQdNmt> z%TCAsy)}x@$YCDB|9v0bQvv&W&7NBAg{W4DG7ud13~rJ>V%1ZJyQCeBOCgAiEu`F98sVUx^nJp)_CvvM(aLZAx&7@%kl)C zjnXcoR`cKfM#z>8`eEbT^VmL5~xVB%?6Z2-c#7l#)Z(PuN{W@J)TUvu_Z?ZO@k z84u4h->FI4pC&(SXP(k)#|+M3XC-jh;y?%K9nm)h-)L2+ru&38#Gprm z6PqY#*E%t3Ai6$-rzNnS`YpP3z@~LhB%QQ|imd`^@}%q?hF|5^B;(8*c=dzXJ@z%c9s zcPDWII0?@1M@P;0E!8T=)XL9@LDi3u&*5hyOjPEDo@J~dKU+*;qSV-^)$PjJ&pvAz zj?;!XlB3@LZ*9=1{;?C*l=o6U2H^C_`%U`9Y#dbz`HTZs`v8M^9^EaNF{&T0l7aH73*wg06ZgAaZKOeRU0((cTYM?^^d{8U(?_&kFfp-VF^ZmN z2CS2xfCavTb2woiy2DSUs_QPLoP3?pdlsdsNZ1sjqWBu#_d{&Xd9TIX^d7fKsOX#O z{v!is&N7)f8+Q)?(r{TU-D+Tw(IAjCR#J3ia2#EPuSpB9(3`g8G1YFPMh?%TMVo|`l z04@vjl59mOFqL(Uf3P2X45I9$WWdtQz|yPB^?;UT(UA>dW!xLPPh6z@A~kD7R4eF@Gvx$_ZfOG*Q|D%9kRxbz>>n!C7#YGxIG_g^bGkPc1NCM@m^MODAX#eP~Y{-lI>2 zj>YZSDIn)?mLR%9(Gw-fW_0&EH*No8smLc$i0}|I@zx=V@QhVc0`dROqxCkEF7Uw; zz$LHo=|i`?HFyQ};c`v-7AMh)Re@f=-covk*S-g@1u|+E=J;!HUHrP{DvdhAOiCyR z*g)P!fcd~ECJI26rzZm+8chwP68D6>N~{nS^nafeErY2lG1GMm9<8{Qz*c}^YUER0 zkkdc@9K^}nI4yvN*FvvBO>Ld|5xz!XMSIZs+acY=4Y5MFB+g&rvKU|#DEebEH@Mg! z$QvijV4&U8y7Xyzn85)qQ~viTPKRzkN|_jH`tlFfH;Ege${BH4UfrbscFu6ZGS5?f zakiZ!|G(2}U|qIS<3BGz#aGTRROAh^G6_GBkt}^!v4U?mX$yG>2_aMC^J}mHn^*Uo z5YoO=q_L!kds`juZA~>Gl>jOEksZ2A9~Q&`~$$UrdrmINmjflUm7@;k7_%j1Gq$#IF^<+J`^?e%I=VizZGD3IiwBYcm+-F5SE6bEU& z8|AJhPsGE{mc}Y5NG?EH5x9ftRsk3eu_+r=XDk$d9qp|Dmg%OXxM318CXx=v`|sYv zb|8*yX9Ylaa?f_VB(WvAS3avEKT{xlruxRgf8hqWqvhsZZs~d@($~)2q0&JhHE^h_ z1Cl7`RR$o_vnnY0DCVOtllE_y29*t_GM*g#*js~SS#dZPeDA7+nmCPXNq#~1iuuTX zKzpCM==iOz7Zj+BQ)e(1`W^Q_MumzMJbowrm^OWyg8(Xi{~wL2-H~ax9-2e~J{Km_ zfM8kc<9~!_V)i-xN`c5SA%6g6hW$5PTASekitfu;wt+?EJ%K2 zGy4QI3$?EORH{VTvFO*p9UaQ=5g{dO(=Ran3q~My0x4D~@S_hYVs2q=+)1&D;cO3^ zny#}NTAqXT58g+u|^21K4GxLjre3iz`dZH;nN(>4?Hp6|DMD*LZ;;R;(*ffpc-Qb9o{wywlYLD4gm zn9+=r8|}7#=i>dwq?fhUR|C#0iUsx4=C=)Q{Gs;97%D>=AZ#N}EX25XgfBqcIOrRl z1)R!3h(4%~g&a?1kH?L-iir3!GgQ0Ry9JEtP(hi$Dv4OC!==MS$V|~=7-W?y#N(s_ zoUHg*!hxYcNQa_WVTG73ZQ((kP^4>=*^wjOo*qxOBWmtFc}40lZPga{SVLs(JxJXH za5OL#)Y?)*I&*u&H`f zKR3(kK|a3$OtG2h>BcHXKZ(Z=rsSm3*hi(NrY)+vvdJSoO?sHNNsD$D)Maeh@p`tj zGM*};Upuq)Fw`f~J`%|ydhuUc!BlQ64aV_0-xeyXIb&jz zLliEjdsoDRom(iNVtky3Yw{}{UNg~p!MgIe9^I*K3uEQTzQ~&SabOSbP4L{Hqb8Y7 zzcnj$xXfh0oMU}ISz?fKL1vkgXnAtaY^$l12(b+dWq`Lhy1x`&obLV^uZ3qB1!Ej@$hUAE*9go z-JGPWUewnBE);!U<1X%AN}*6;G^9C`E+hyJ1un1rvzH!Xrj($pscP}bzT~3 zs=;}Tz}sH`YB$5U6wAMHzC{xOFhEkzaZiN*!71FwH^3d38Z!Z407j<%KR*CIUim+t z_|72_?(6dq0rvq4#KU2N9M>wCx(*@kf$2@${HAeGnAy`hx3(Vtp&tm6oi#Z>A5>tJ z^;|`|Er+RQt>Fz0o}K%PHUP?sU2zWHOJL4?$g$LqV$#$3eHEJH&`a92N51qp=C~&m z|Ic5~Y@!jd>4Y;nw>4v@3e^i3-i~Wk-uq+wvvB{)D9p4l0N%2fUqOL~4mf=oABtrS zL5h6tuj#-avpl%JSpC;u&hAPO%Bs|m6?vvpx|;9;A*8?c_HAS7iO90>NV=oJBr^rn zREi_xVp%@O*9AXQOUS6c*uH#ZXWoD*2^7k+6dL8-Nw+J%_kcDPQRBFE=lxfCkPNc? z=Zn>;Tr9lk3sbGDG7*$NrcGww?LRWQzE>%vpUcef9IzwUg8UqFhW!v(tkI{Dxi4u< zttgVQxKou<c@p*bzgp|l9{fz8(%R$Wqhaye&A=Z%hQKA)y}O0 zO5gv74mc{Ee4ST!ZS%KB$&U};VMOhY-Det8ewu*-P?tQlrA19I16IEwN0IhwFDMZx`Y6ErVlsusnjdtkl0WA=S2 z(*CsA)VpLO;TDF4;*>#K06OWZ#v)Fy01HygiQ{6Yl@2i{t`3?sHg8%{+2=FZkLbDU zzwL%+*T=QRcVv9hmM)eal&OupUN#??Ezy2%jZ=`p;FIfW1@RCLME_+Z;e^^x&6Npc%f)KTi8;^kKKErqC z_{Pl_#rsjlBGEwX?7i86G1Fnn*6oI*h|fnAvLHwr@aMo&r&+$V0GeKGqovk&H@}$e zosN36T^)$fHr#Iw&u}h}OcZ}p(_eGb&5`FxI=Wu-=Lt>Arr@0-wS)IZNo)S?d#F#< zII0oiiN`73NDk;VZXX$T7g!l{{Wy_66F#~D2sN^IrC2TsE(A1N_gn1EfK!L&#(m3f zkLim{O!7PZ(J(G-P1*d&++a=5XQD3GRBjQoqu0PH%-LPYCcze<0l%Q_B`Z#P_YiCJR;=K;Ms*Z+V%A1S?AdMkcW8RAw7`w+7u#Jndw z$Cek=lZzkl^GSD`z1>Ffj~K~Z&W3xKArXtJCa68Q2Ff?s*@IDj#|aC2Cl?@Y7m$GT zU01&dnSLsN8T#gZv$xU(jL&-!mUJ>k#Vq*RK?1kQt94$cRU^RRgiNLZSu7&;&D>%V zIz3X9X}V3y!X&_Gz;jPQx+pl2eH>1Sln7?>s0s|5KM;lu=gG^5C@ zWXKBQ!j$N%BUfXrC(@UtN}%D@OM%7QS^LeFf%S#*dfz`?m1Q?yAomjO%qlfNp^{Gs zPrN`cbxtyS4Zg=!_i3vq6k}m9M@I}l=d2`j`M`)6GHn5&4%JCc6s7UtDZ8Nl;IY3W zU)T}- zhDPQTLzRSli(2PWNl5kPHaug(j(`f;VGz^K?F9`LRcV*>-uoti=K+l~ zw8N3U_5Soz2at8q+^MM!!C?s?gyMMO z^BgIdx^LI2X+xG4>4;3*-=9O7re~M>nA!Eb{6Ak)VSae=@8KEY_iwp?!2haHuF(}V z)akrB1zQpKanHbwajfnGHi_qwr;epaiaF!I=dSFsp{+)Qg%&#BK2>y`r!_zghq%uS z59EAN%#J+0c19MC7+{P)o)<20t5QJO#H|LnEOu%|&|{?2ABV2;g2cKCqdm$0?CW8UkGxvfXa;O z0JDV~R?M=YBgAH1=&bX1gM(fM2Yp!uD^4-|qxGdc^%zojJXO>g`FFPBdw@7kAOj}@ zkTuiN2Q4qS;vN{fAS3wXE1}mR>>oCUps%d5q5L4$tNa*l6wgs#^S}Lh@Wemp=)6%L z?NJQ@uvYjV-GxEcqz&9bd_@_W?|3iP6mkGkj{t_TUCV6oTI{Xl#ba~%NQHZqET})4 zNzS~3{kwfr$kogi0YK`C`3JsJEB|E)AwGbE>a;*5dhEUlKW-TW4PMD?QSo5Sqt5~! zA2+-JvqlMoSfRYs>G;f2#&e7w=mGXqVT#LiwNr-vmZS~*I|%53_?OoK>j0!5C9OH< zH>h2(fftqIERKVyTR)E7bdLkcc^v`XJ5__|*3an^_!MVglOj|>(hCeKTw}ob`2QmQ z^M?fg@`2zp>7~Az9MDiCJ2Mpb|2`m$H7^yU(DG7~uN^d!;g)%Wcp&9(xD(>@5A!k5I6$LJxsMya?_|p3- z#QgyU&d1-;vf}96qZw?{tN`rdv~h80;*K0)EE~4gN9v6X*09|k)7v(+fS{5G* zT~sH8l8iuW>av2OcVo)j{#@P`D5s)6_pbyCwlg;PM`R&TFsYpb-@zVOLxxdX<79UN z%@W-7)VFkpE%GgH=UXg&Gad?jU}sh+;rILJvO^%{oxn}@HB`<67C9ZYKiGfRXU?^s zwThw9ePv}h7iwMQC|&}?ns*)e(~wOMnx5R&J5}~y&lTPP`l|-vpd20xj^X zla5R~^?keyxVD491-tTD^lzG50rb^-ily#f9rPFjD$@(|rIVukv|Xfk+%5L`-fnbP zq{F8h)3){%s_GLbzakx^;0U|ylSrzfb*v--VFfG_- zy#gA2Yr@}%*8s7(@8dT0<=~Z5DI%b|It5HA&k{NH1 zB3eFsJnRwA-D8agR)qs+$e8U3r!?LVh@NZD<9ojbHAXh%t4Wv-r6LPt09#j#Jh1m1bU=<3RW@Bg)z-HYD(3eDM$i2W zFN{6S-cNKzoM_s2%x^lXW^MHQ@nE}VbzY&>Idcy!=wAMk-%V??*u?mMX-YUGChn zS7D5Y$D(v{r_ZWjQrkx^f;Nn(k#^Mbdoz9|=kmRaStI6j2Q+Mze z^o;^FPp664ul*LAp0hc=a(Zc5>G(i*%Gd2_g!aUG#+dNBG*)Jsw8YYXoObFA26UbE zG>3q%qHOgkKJR#V2k2b^P|nFOH#Cz3?AMmi#sHB3;>+7gr8&o_&8A z@Kp?Wi?~^i9pIWrJ5L@8k^Flh)fD+l^QI}A;Jg0op7SU>bN?^Y1L{~f9a=2S*S=;= zHK~4XeCg88=+{$U;cb6ymxdiPFUBg2=8dM@fyzA{W`<9UiC%A-Kg)ymUtQp0y4lj@ zn2>uzabMKM6+pYd{MRCa&F|xxX$ST!p9f=x2E&RhSH@tx{)~zy)2jTPFsT7YItKnJ zdhUo}c~5L(sA$-6Va#lV_Hq4W3_L9{tIIrzQ86u-XUcwTTFm!aDKK>N-SkU*K>&Om zyg1cmC25t$(_VDKHVHB|u^ku?kJ2pr&Fwm5V+<6KDNOZSWmzocSm$F0dX1CDpu?8h zwHAv*{n1Uyu&l+owWrRn*d-&|nR@pb730s?QyNbyYW(RBk5}Wx7MOkw-q@`rH^lMf zgKX$ckkT@qc4RUP&2@fPEDaxfPX>YdzX6o@h9)_8t?8BG4|MAYwVte`YU+p|O)9){ zVZ=N@nBnsYE0(iC+q=HP`=+91?q^SS1M*nM$f78jw=;$d-+a94Cr^gkkHnpi>s7Z! zoJ__*r2?wv$7fh{qyPG8Khm;x3Rn>iaBsf&QJ?z$u;#<3UfwSZVJ}azH3n_HBuHDT zN;9Nr3otH|Jr0uwXLX)_`SIRVzYh_$Yje#UN5B-j|Ms%(bliA)D1s}`G%14ukQ>f6 zf$RMC^w0tAMOzU`@;KoA^)^ADgol|S_x)g>HrD)>BMP|U%rUB#gYNo`I{_tk`UlqFv9TGsO zdoF*hv#1o)D1z!{QqEeIf4*4S>R9TlSWPj0_xc6xgEmVMBvWiqo_i!F9CSbJBS&=n z8$S8@iMXk2!AkshiKnqKbJO3{>KgjP@QjVqm^@7m4GvYJ#=N!_-P)2jth>BRBY&<1 zs0ta$#Zg2ANi~MqOp<{nji(L?gzR@_J?@S9ld&r6Ut0 zm8C^a^tTK@OgcXrv1x^lt&?~pnXF|bJe^Br89^~x0H49N(k;N-3uG`{z3_rJ={eGQ zSRvtgF&ZtM@%wd*j4dOo-xOJ8((Juog`v3WV-Q_pOm5=VFxWglT3E8Ox8bHcEluR& z(+eFbF*Q4r#3YCG5+O^zIbw{1kwo5VbJS5YjUPRjC{%CbL8gvC5(ysc7+ylWzC8|F zd-L9k7VYtAIff{p4anDG6WJNk47$?8>n?{^Af&u2*AKRYeaUVZ% zL$S6AhkyO=J_PkH;J~>b z-}rJy5`|geuRmDq`3%a(kZzdZODQ_<`WuRk4ZPh;Bko$>hSG0`x&COjs{P=8_>o*}uMB z6)kvr=)3hwoctmbxC^}8HDD!heUo07x}*fE?Pf1`i*6gDK~saN_X{P!kU;TX8NO543uPjqVLL3oBQoz#TZQ4yrQNrZbn?Z6v}FUyRe(5*fgkX_<2_bD8JhpP z94qQ-0aPa$Q_}R2)v#b&w`kG5F*^hnj@{MglEn`51Gbmuz}u4T5%WU0~qc+rfrnxO@17FGzRY zBU;-s1ghZm8+FH2;-T#Rrr_{gqyr{Nss8eirnp|Ga@75aH4>yxT!p0FoYPEc)7|g} z^S{GwNXg;}g8Zka;W-zzKya5whUY#S$g#}$dXC$wW#j9Vsc(WB8}Qzv_hg8AB_zg4 z*0>8Tjh^3v)zSMC+*Tvm8JRN0Cdt4(a$W~45~denI`KVn?}OH+8&_pR$vQ8z*Om+W z$f($-F|~_>NUBnmy4*3tZId5VAie)^67ve(Fkg_Uk7M(usFdzj&2 zJ(13NIh*8y#|`*tHE5R@CC6VXHoK}68MwX4bbzE&isHTZ#n9Ix?bR-58ZDH`*el29 zW+p@s^yb=Ie=EoC>c9DH%d>p&Y8Qjwj9JRRlrBTQDnX&m^_{Sr(}eNF?RhM04e@i$ z;ir@DNze)KaLj?^LRU?+oXl>)Xhacl_7^MRjY6P}h$H@i^I83n8o0xkqV| z%@uS+A`d?5ZghiEV~*=u?9u98X>JN<9Ghfaw^gFs#vcpVQHoeqV-I|PD4FXX$X+R& z^%1D^dbg=i>}cx5;==Nyy)zkIR2i_WBE>d@;Y<~Ea?@7OVP7k$P2@8*u^MvCnh zpXSNG4X@}X2KO)1V|))`Q{;P98ByPO_3o%}>ob5zG2-KEa-n$ZY&hwv^`XonR%m+R zK+*J7u@uwlzFRX07BY+D%#MUqnsaT(Cipum-bulBTq*g z&F7{kMa<{c!kxyVkzGrn;bQyat3FG%rSMw(D4PY;`y`spjOK=b_%!gNl>q!Db+<#> zWmJj3*A4sXoPfMt`%>Sm%)_qc(-Zq*28v3FdJv&qzhQ5##`R5Vj3c^1XUjj`+IDg^ zL6xhOgAP^?JM44L8u{5Z;Ij;<7eDPo$%rid;ql0c7g09WHQws7@6zJy9n)i621~e< zswjJcez_b=>h`?ra2FYkW`Z@e4 z{{468N?SaEyZ9o(U0BF8%8C;AO~Ryo$X@Umq`X=2_p8U9hzcO0L&q-CQF4 z)<_iEO0$9Ha>j`Iz{x)MQ^`ieKr$LG6Y)8FKX2OHF!K9tda~0qCFws>W*AqCr`7R7McbA+*l@=y&EJcoo~pg#w{vCU5*XNU-}i`esC~ znpsu1{)V4MOzW6Y*Hq5my45+mN5bnxl=zcf?+dnD2IppW7f_^_tpMJDj(LmAbi zpY2`sn!%ycDITjv6zgsx+K){sv14?{`+AJs0cj^c>18#ti0|-sWZ8pQ&LI*KUxnmc zpiHw2qoe8cx`8Xj62^&}^d$@CHDn4tmQ3H=v$Z(*`eAqm0LROd6cBma&QDckxLjgGy!XUg!h8bRhcuip9} zv{zq7+7a(CmRd#YL7I!1ZcW|l-AnNxljK|$8W(VK>%H)ANAOP&R^m75_s7;2wh?Y#K54}1 zMYE`ZwK9+`_kP*+;{d&%7QO|wj%h33>$_K?`3P(Je?s`r2lW~dNJAM}z!N^*`lK<02 zR`AVmh7S1`#irxEZN1Nb184;lgVqqu&itM~+yjCK?t}(Ktjm+6n^5>C@W;p}RP#?F ztkkk~cj?`P=PK!di^u!xt4g=+-Jmz{y{x74o`l}$DaF6*^;vWpI~Ap~1v0q#-h93JijeVYa}G+yQYKe+ zL@}bFH1;FFx?@Yu7$N_z{VQ+7T-Ow%erQ~5$!z?4_!Vva%E! z&1)Br^hbL~?pG(YjD27I2ykIs?yc5gSD~VqQcWv}VZ+w-z_q~-M2u{_prm!CWWYoy zFBsVsy9&tHsi1A;+gX34$4NJ3G!uWD!|vL!-4xkTW>pCPl@s6Si?4fctma2q?wjd39&9xG?k`a1sHJtl(zuqP_dWfO z@n7#L^xrf}$_J5)9~NkWbXs_Ny=8u+P3gb<5K`;+>l(xt41q2bNS)~}2eM0G^@TmE zYNu=`C*Y>sz@?f;S7w!hcv+8TU-ZoVoirtVNVM)DEc5qg;`GL0XOXYhcn7@A>k3+0 z<+U4L?ok@=IuXqCW1+8Dh#;f+lT<3NKQTZ2L59P`m-Wc&`=$t zZ<<$B4Ds=gkNaJvbJ-G0Sd%g>A=9)XH)*f5sbs{>{dfkGKQ7$++uun>?CDJs(xE^? z-3vgj17jKm@g!K^P*o)&O5>jLm{uh1#yMq7s)hfEI2F#gUJ|BZ1$%Xnd{vOD+u>J< zVGn=@t1>2oVp$nEe;3opCznem7$SqV;Uj5VgitsoC<^H|{qulOr53?sP4!g;^X@;0 zVWS$D4^t~G>Z(?l>~cZVfQPZ2@7TyP*(ugx6Qi48HCMp0Bx?I%A}URwlO#L$@$EM$ zA2LD*9;QE!NXX=CfHEOG&_Vp#~cfLfHGHS+d z#8VaZAy0j?=-8uHuf){nLm3SM5pr{flVQ_K687*ZZ;$1BDwL7w3zX7OAin>% z7F9pr?G?X;fQjPA^|{kB^fp9!^L~dig?)xWZVSBn&HPHs_lV%b)mEdOb8B;d;V}s> zpEP$WjI}>{qq)qV!iX84`~IK!S4=Kal7d@RFu@ z!?Hz%0e1Kt@eZt&DkSa-3EQWpSjmGk)z5I5f?8PJ92~wi(7e$yxc9eF+l?omtlNNbC1V9`g2hP*Y`mV zh!aN3|A)Od|A+GXzwPH^E&6eUeD+Ab*?rC{ZQ>wEH1KgT5M_=7D*mBtL$BOTFtDO-SJB4acTXPRSmEM$Wlr_Tbs2EeWYQ|NQ)Ig@)Oxl#_00*@IA2Q{8; zjZnx@c3RIwvJ+|xy?e`fFx{PLekGzUz$HJZhX3Xiy0_;)@1#28pe`b%s{(AuYrw|l zCdRVwyoyN($La}V+4sA6F^`A^q2ccZ?`Q50g}3-qRU0s3iVf)sCH^$!au2()#go^! zMyh{C9A9U)wM*#C>rzvE0GwmtG>e#hDg&Bj+%-1;V>dZ@FZakbv0R30Wq6tL>kn8} zZ^1Bg&WE~JM8BCtjawaRZkBAN!b}SBuGXf2zmR3s-TGiii9)6}syUX$n-;lg0C0m| z%@Niv8*m14e9HY*T@xBF)QcT3bGQ~=V&US7vYx3r{f(Z@JrftZT{%3CfA?5fTkZz?Y#P281vM1qjRs6@}6lfWib|lwOYRi}CQmj(x=($yN_l|X;KdsvXsG%kW zo&bY}Aip}2x+`uN?y5tR6Qo|pCy~rOAE!i3Av{)qmoNO=Gnb>ApL_Dh|MU8iVePDb zZRxZIQtH^LkFMT~KpKMGUR0Drj8c)0Dgr019U?B^99u+3N#)&M*} z6Q>YAD${6Hs;?Dvbz-N3q=C}^H-M*sRy@-gWvJ|IJEviV(o*g3aQ1pvr`YYuk3UGR zG+3>gV%p9S+w41OJ4#?&bHfpXDSHEfgyb>?U*z6cwKZ9WxaVO)tIMq$jHAx)1h_I| z0^>GL7yfqC759pxwfV^ctKh>{b&PbWSZ2dwkay&_4)PwMJ7r+WJK}ILQ}H) z-S77CbXb~buEDc9wNafD5fx^4p_%*+cHNEVt7CO}b5U9P=fg7I|3;>h4ksm%u@;nj zlg(a<4=<^~a#9~Og=4pMss|;`hgWq}FLGP=>P$s!O^Oo{qqzV}aG2!|$-9{H{W3G= zTL)kaSX#7zUU+Ts{$)CSU6~`UwSyul2g+lF_LEAKelt6V(~Vb2tIpZQ_6uVxL|%2l zk$#*PsA3-&x^EEC%M;wCZz6YQ-0yH?s8((27z>zP;RJGU^BPm00mjVMOP<{$ybBPu z?o4`~;{!i((a<0;g}}g6at6wBwUx;#HrwtpazXUt$n-%eO!Zz06-Y<#c^}>ir#?&^D58B!jL8(c32Yctu-3e=L)jADO|B)`qtfL{=QrxxP(ix<_1%4}U*OC_K?_!le zR71H^0}1fEK2$fA_AMq>Z=sw^?oa{$N5(CuIs;MDZ)t8_=T9s8_9AX@e9q!c&*A&4 z)5bdWP+kDzAH8z0f9qw4VRgkp$8w`<)pi5lyJjKDv~CPc24S7^qb)I@5%da@4U!lq ztZ#*+870F#r?s~><9Gg;onZh6E7)vb-`cXVi_rigF3`c|mv<*W@2U4f>rm$OL|L}O z+~BK8h*K+geryqx-aB_UNkd%Akh2<5vm8CN(S&x{@7Bp3aca@0O?f(*0E>%*++Qbl zzOgkdm)ojTrX<(Cv~DJ70$6EyDf7Wg@oW5SiS_54^AXMTE1S2E>*F;kM8^b!M=O}^ z%GGJYhA`n3J&Z=p_tU>~VON4VrgZdBn$E!ZZopNoiX(iu4AdrEO7niq#|`_m_sAv#O9~Ps|Ru0lWS>LG~v;-^2~$= z`s3TXc(flk_`1yccH;}zCh^2lt8+SBqM_%)OcivdoJthGw^W5HQyK;p`A`q*nK8x8 zEMMc=nT||*vaB)M;LfmU3&kfEAsF=qp|n=D1I?MVC)S5Wn4>Vm|ubACQRBc3k{{$a)(4-h_-Wf%?H^w zwdVBmws;O@bua=T?=GitG|b|4Yf@Iv@Cn|n-%^iP7#~2^Xf3Wo3zLP-nho{xYQ91j zk0WcD(#8){J|p8iD09)lOy|278ghN`o5r^*OUSm;ww@O(AS>!bXPJBW;mTuiv7wl> z+yW$Ywk{_LN8y>>UVR4m#Ww}GMn(UcV{#p&!L(dn@inih1e*5brN`Eqm1?Lpu?1zj zjGo#{^@@)4dddezJQ+7+wQbO=!~^WOw(k?A0A)ORe{C*seBps^0+T4}_yhU5r$ z`w;*z^Z?hyfR?y05!l|OsDbeT_L7qW*jnbQJlCDX`Y#WMUc3rYqFv@fJ=E64ET>QM z7~j6PT34T3sJKsqguX=X`qU$;1gqho@9lGdA+PGz_&z7h!|D$SUk7$^2L;sA=+HgN zTxvuE$4UkL)yga`_4)mh6o04e@_(M$czIIzwLJb~^FjQf@9(=EH*ksA8rE+Ub#nI{ z0B^23#3YRnhn_EdGA@a-&sPtrZUvkh+lCZFC;@1h5AA%69m-K#Di z6x<9XN>NhO>~&p<@SpYYqn|pl4_MBdg5sO46KvbV1)wVuX9b+wz*p?5)wxinhcRRs zWA+5gD8ngv3jMfUF?sE8WMt zo8RN&!v-rYg`M-kW#&!lTumS31H6cdcJ76< z_H$vWd#>0JnQ>BBkM@;ofjXuWF(0<78)zzZOpr=jk+^dlmqfnjRH zmz|HA@)S(DqOT}b;|-!_XGO&kad*BnS-Zv4)$Q@e9{Cv{yf?W$gdhh9iN#arB{Ufk z60&Zg9J1OIIC;3V{IM|mS3LSs(76uuZ7>fttv3*?l~r2d=rO?=0uzfnof7Y(4th&= zS;F1wMz=T?X=+Kmx%=sL9?Yhg@ub6Vl(L0i(Xmzj&~%IW=u*J%BV?)T-#&zBXidf) zZW;dP2&P6nef%}y-eG7+PS{s`Mdrh?^Juf_{rP`p>(Y~OS&i_ep}A8IaR<&NTWK6f zC|}u878AVg*8BtMML8Ah;apl)-Ws~z%4>!lnpx`Bdj2$(l5{>KtOpPENB6wP1W!x{ zmOpv&T|5GX4UGS;h~b(zI$K%B$WSYhh$eK)tuaGXBmQZ}moHQApJDhL`bpCY{6@VrF%wD`Iq)?a!<;%wxj1tQYkCg}9$B?KnM@}Qle zVz~yVJf`$=-L-<1!#Qqp<_`rjsP*SuoPCh#p(!&_>#6qunXvh1(5Xa0? z4ZiRN=O{5vPvx&IJg21#Q&FaTsN8b?-%h^?snWA5m=^N7e(WsKX@yLj!xUEkd|djs z6GwI%A3cNixQ^F@VhzU@A4)p0f{qkGOgEYmzJt}v!FO#)hJu)=4<76A>-g{ir&UFZSk^w-qBw_s?QlDyYc;2w$Fd(ZKiONvRk{FIOY!oKPU+W# zT)_DnNB4;Zg`WW5idZAOo)AJnJIU)S8=>}ptnB8lk-JX4Tm|5Bps&n{-Fm&%?4LdR z-zJ{_h0{7;XLL>woUa)!6z&;$Mc$|>F=;-qZIv|LkWHfuBp>%Z;q_RK|zd^>{ zAbE^6hSr?(t z-|`7L1siiY<3cNhENp`4(QGt}c*s*}(6fKUZL2zF6PLfT_2XAOXg!Hc)3=BO2YQI4 zJ6k%Cbi9cHTRv)8qGh2~FX^hN6j~C2Oun;3@l+`N&%oOxRq~QlVt=~dnqU3q`>Xv| ziThAw*Tyx_)hX3Gk2Uvi@UgM|d=`C`t$$dE)>^DlBn;R9qIKVAuR7ee?HLNOl4^E} zZdYHy=JlZPwxTo4AWPs&y#rN-e&FwuYRN zo0ycZ=^~Rdi&eW9C#FZ^Vr6P%bsqtUJ#t=hGf7p^KHntYv-qKY$cz%Z2RGm}Mt-kV zx$gO0_mQ{qq4rmzxq;RESI?bH+ltuPq&#lzz3;G~W82n8Mriuy(5@YKD=$6nYeNmp zGD033^_(o*uA~`@aSw-C12mNKEPdrf65R8c+4pO>m||4_j@r+xj@1c|(6`#6F-Ay? zCN*tDhu&iIRhFiRSn)^9l+#1!3-2!n#0PSd2>Pn6@gX)R>FZk#{e1DN0V{py5JU!S zdN`i={+~6XF2T71WX{@&0g+!P(Zj5Ae%&AIXlacM#>u|To@XylhaAD$d0Rty^1RIe z@W7=@J5*AfM6YXuUpk~k)56-&bUX~48elm?6sVHdxd<)t0CE1c-H=s!}1{yo*$c8C45HH1E}>{y7;3JK4E z*%u~uFS==OS>syn+W+g8-c<_WxX1I?i$DHu#{aGNJAnX&`t7dz9i}UQh{pmsnw;eo z)qr0aQt7KA^J7~JpIf&Gef8t;x6R`v$`B_>43EdnlBF!kuKl9zplWb_3UtF!cD#2F z)!$P8v&FF%-jBJ8vd^1q+^}hxaJA#ZAC^{z|FZ^kX!|m(k7L#SZ5_l_<{TZ^kLG6E zh^`77|2o$jhV)Bun@ye*E)1z^4=rR4xTK$g^?l1&$gt=Q-=teaMat?%OdKB=5gc3D z9IX=D^XqnT-pgdmG(F5hNA;BS^@xt{8O}2JtucpYcQ@#ktX**sO>s{QsO!8EgyKmph_Sb^fI!81lJZV&n(?P;|$IcP;dOavt1#U>Ye~Q%k^>Z z`6^6crCdMdh@(vxMwEER@F`_19ez#Qja8^Z` z4-~mT7{ys6oC2`ay4TngJihbG1)0%Ff4UH|&QfbCVgNoHy2Et$opF>(^`K;Pr~VCP zGCw1ZzKR-~t<&<8J&H$PxRqTUPpm+iJ6x0EDYM~Vc6_>C1S}}o29}$*k9I>Ly`HNc zt>9iHK}4=qbgBEk40Q=9chYW43wp1H*i+``+&tq*&r6}}YSF<7-NSSTru~4%UCb*S zWdV-dg9syVNfq-7_t$Vo`P#>obd!Ef9V2LcFg*$!XvCHm%NXJu`)+f{a^v@s@$Amm z8t`nOmGgxTZ@HGWMPrIvmVMjy2{dZEn!i@oae~~evORObg-bZF81`$6X1jCeP((c0WpD?K2fDYFqpHCmyapc01iHw|0 z8~<5ouD+Cyww-+4q?;^i(q<;WQ1VfPxQj($-X_fEFI zjiEVrdD4=MmeQqD$*X-0-6n_e{&pE_x5&yj_dI^)_^s|W%+J;(!%>3TI8=^bMG}FQ zjYygtaCw-$Tl?Qs|DODVg@6hbf|B@K86Y|JDh|M$%e*bTnOIIXm1n`iV5xclv6`?C zRQWNUW4`xPW|;2u6N7TqO;?{NSmWV4YM)+va=1F57J)Ejo|v#!srZai(CvF#lNx~uQ;?#RB$u@vNCLRBIWny(|Vf@!cMVKjzB8$N2Y zHq>jj;TI+cvvG%gjaUpjoAVAr>hn!qIU6Hwu4ak#4HX1`WfQ+ucv`tMG3fM06Stfe zY-BnD(mREJwK(1a_oD^$5cdquJA1#RN4eaNLaoI^SDb4Ot$Y{FwNDMS`{NT~aCIvf z-A$xv%x=$qE&TleDq6PFxbW#kkZNi5?W&=z@4C#-UpNlBi!D-7%c;E2CQ04P7{5D( z|AIi;CLVKED{TL^h#uOZ`?X;S%NI)J%4wp7PfGi4tt=2BAZn(2SfG)JA)SM|J-@X% zd%5}qRB7$LypU*iP0TI@HIvpG z+ZCARjZUqf?Ej})?IrfZ-KcL#NetnL=3ZKXMX!CK-XF3D9o%977YEtZ(>q2|d~9z* zR_Ije-;4%0XH2gGs6lXbpD@(X5R+%`I-qUp&t)r9r6-|YO!~dKTAi<4;2<97UPPr=Fz>=?*N|C0}^W@GvaxE2Amdq)0KF3Q1Qd z@o>K`;%5$wHSC1Ao#2{y0*)Tm63ElSxNqM&=X@Jp*t83)Khw}<0tM+;c+*p*FL|wH zKkRR-NoFk=Y?0O91PQdU8*fX%L!obn_v<89$K`4#efKyTehAlX%s+*my(%MDqZ*zu z)=!npWgTkLw^Ae_cPSRo!dANH^)8nkyu9a)fq`V@78B8tAo-Fyf)4h~wCL&GYHq&;r2d z-jp4k%%%o{9a_3py~*YA3YtR$L!58ZCBrGACQnNKw6=JxYar4uBVDxn4p<(xlz9dT zKH{X=1<793Yd+H-oG=|S=&*X}Tx(XUQ2Qsu zuO~Xj;kJMzy!3eP4ZFme3)i<4MbH=RpzyYHTT2M*s+Fg+0X{z3d)Iy^S>X7RFgHi; z1gqCmH10_Kjo2jHA|bAE&cf;2q4+hsk08|tA$=C(sAO%Ye5z4|n!XYe{TBOsVrp=s zrq%2!Eo2gvqta%kmMnl?6LZbao2wzKNq>HrHfDiK&4n97i~eMLkxS0&aJU#JgMes8gz!h5Gjv=wSFgU*tN-hc)EGPJ!N{~c|3|aDrn7a3S8#M%1cdLrFuPw^e11G&6Lnu5hS98jjl#H_vIcj6#w2pmY z3^_%I%-ceFM?cbpJO{m}H9lwQPt(?w_ZY~q8*ULS9W)Gszy|MD=?e+#UZ9-1Nt)?p%y)BB-9Jo&>^yU6;_Wv9E%P;qzeM z<6V@5$tVr8$}L@6O2yFlK-}T8K$rgBu8O9=0m&pmauRZTM(UZv%1@;=` zS#jvDLLimekQU4BoDDUE=pwr+gs7b+v4P9fu?-)KDwt}f`n>|h|9JYCs=GN*&0Va#59iT3-; z#@gCEZ6}}-Qx&jv-d3jC)$A~)8aDL_yC``4ZbaXg6S*-pGX*bW$|g;c;DI`^PX?z$ zOj#Q94_(5n<7com%A5sfI-sak_g2~RycxK7aUa*^&p)S)x3v%PjPvL0Stc}(xJ zWKf-PE-Rk*i6Vk--N><(fHqwC-94?*7_T(NA=@B2Dip~tMq?{*)(ojnSHd~X>{})=^Tb(e;-wp zn0SesxI^r>A8~+dc3$QhntE70;2BEtuKp86J`DkqrPv-EkTOtGNZ_Wz=tEI<}yYXzyBD9yEo&Rj&o07H9u9VwW;Ea!IbHRl>VbWmJcEGRS+K zyt70-8J-`RU|jg!)!9&&vk#57T65{=%>(ma0*jOUP(>t1-KmOAhfolS*!*7&5wA-ecMAY*Z)g`@ZX8ZBGGn6U-xiC#7|K58Gaz!skzAebwI~NmavL-YhkUk z+H}T@z46M{+1C!nDS&gkQFJynAW5ajIlo53-DDFOwKCrgBEbsYYLN0n5%7wBnA zM4~KgRkr&A(ENMMVjv}ZVGk+wxfM`4G`DGo2xKIH6P}CL)}$oB3S>q<;L)=k*m7kD zAw+t0eAb*@RM~!#b>^i`Ux2v>I5rlIAGvFj$+j#ld-8bwk8dqQ!hHH~FJldGIxCq{ zT(>1gBdUICpESkPxVYeO`rcMTz`pLl%L=LTe1Q5!g!gx}Mz7g8Rv@2K-sm-zSp zKD#0T!p+1%yiD*1cMSN3M9OZ=8omvfon`p!PUS3%=GOE4Mrw~%q`}JZXqY8Qek518JiMVxGr6TpH1lw~ibg~I+*`1%7Fhu;5UDcTY~>NIhu}qJNP&Mh`!MpS=h!4$n=>cyq3F zxxAJV!jqenG*)AXBnf;e16rvZ&1&U6+mN(cz?-@0ur`~Qz_n-vK+kWFGESW`bsnGH zVJI0#^=wp)p7Q|J0D~_8rrB8{=;Qz$d-trv40yye4>9UZNNJ}TuE z-T|hkW)xl8bpO?e)ZGG)O>90Y*Xd*j*&SB|yioJ*CVT(eu1D4*ylYtB>ReIIG)mEFsUz%G7(p# zUhg+{oUu~QgG`WpAC;Z`2Bj}DB|e{--6C?(`2;v8RX&&Zmf!RZzILr%BA8lI%{F#z z>VX2cb&XU&+LUEXF*W2|n2?xe*75&}+Lx@O$Xoq~DVwsk-JA%2c?!90$9!fk@&bvF zu0jdVF3rzGXK^7)R}FP%HP)(?J4gmsMIES@SnvYam`^VNn4AH*g>0N7O#|qK&Zc;T z;Y2^WHb^dHe(O8+`{5}Xt&x2qMRH8_j^!Ea=cR`$@w(gK%+b0ZuK)3!H>2rKFOwY} zF0YMbZ<^H9 z*@mx*&G{;W;t$srmtK|p`M}I{jj4tX91dqTex06G>{zFlpSYsy=3H{^rt7gO+egMY zyYI`MFtG=9M;*`y`&z*1D14gBXaf>3h^LUl*o#;PQLakxL&ma9L#Ww|9OaBv5Vnah zznIq0Me6532sr;ih|{&docO);CgVvN_(aX%nEqq_ZzgI{nU><Rui z8>E3ggZi})>!BusyRU(@qw?PBd0L|g?fbL1svW;Th%P+4;}6C86RE65h1=_)9Xjlx zRKS{NPr&D1>e~tpHVV!Pg+imB&FusE%RF3VO4t4DzXVvE zK5vH$UEFAqR+>x?z@YtF(yEs>vXSWu2Dt>sdGK?luK2mQq}E>o#!FUw;jZPt1>=5(Pl1BN?43&f>!Xhn z66j&aQ9tlcGP6>3g4ljuIv8gueQd^gv+#pi%W}ZliX0BvIHJ1Ku^{rDnj^v8=@a)k z8k)_P){cY%1YJ>ZhI0Weq-fI#s|_*^lISQuvf0&OER<9-)V{_!_9vX7Ghcsp7-V@C zpOslDK*#y1r~b4odGf;|3KIbB^w?0u>zcGIr)+s3$O>R_WI9TIoq^xo_$9mE=d+@! za`n+%wx5CmTdtJe#0mWQ@PXG6@Rktev{WzdQ3V#dR(ZnSEqbA+1~cUrJO1F`$ZJ4g z;mwnlWR#!&pk9Sb7rk$`|77`cOtE2NiJ3n+VC5lSVQ0FW-98{Pt|MsHukC_69SC!) zTA&c7yzQ06(G|s^_<=(G^e(sb7Qvi<;H47{3POBgX1d>o`1HR#S?P7cHgk=y>%cjw zfaEc$i@agPM`3dlVL32DfGIiKxmbyh5&C~tow)`P6av4C{ZKy`@=-T1DJIrT#@1&` za>u){1r~CBYw}m*GDdRA`mS{c`9t5Sv1%~6Z|uVCyjR1?EEx@ZZ<$CzIv?b#;0e_X zbKE{7$&aibWw>86r+o=}JV&Kge2wh*?Eb2#&_{2cy2m^Px|lCkS2!cbS#rG(CeKb{ z^(Y~;KR!55Jo!3K@Cei1J>-a-jiXDUGY5Gd-RhjuSnr-cC0qoEd6zCN1o zVZM98jyl`ec3c;tp35GXdz|HHnz5EF1ks5#DGFK!QK+HSa}Y;5e%zB*RF;ko2&oQ$OK)EYm)DEQn3$&HG3`^ww$^OZ0} zJRWjMzhJA>KL{tjv|>3EezBB0LrrDFp9{R?jOQ)zodG4{TXGXw{5N%foH-`H|A<$e z3#?1Tre>!IYeg*LX)~PhW6s$U4wT}dw!$5=={D5Uio?dquio5gXqKB%iA%9Y3E3Y0 z)LaL*d(h4-0|d4z44bM~m{uTrV8d1CW3Q`2q-g#N;JA=`brL2&tv0x--=~!s%I1hU zmgOX*TZFIE4pU0hMvITs8k~<9PbZLIy*d#`a?wV1cR3sG0_r;A$`tuJczcuKY0_Z0 z+}$}GHxq7c`HuL?u_#t7deoJRR{8L2si5_RY5M5K6p-529}PE7dX@1b%XTXK*TAX< z^))^MeFik4<2$^#kzO$};Sr856q$ECUqO}T4gR0+ZN32PKpR{spzbWWJ&a&X*)Uq3 zOR4*iaK6J+?~ksxte+r^Y40_Af9P@!igVjMnUZo=FdPdUdx~B<*?rxDh$(7ElV#|d zzJUBVE)2C6a9Z;zHgscEGM7|TFzY_Jvfg}@kUCZKT{^I>q{B5Yz+7d?Gwg|;24QnH zOq4tz44(JYiJfV$V{@Rfm>M?oB{h?uZoLgP_9eA}oP&-D%a*S<4^?KPrYhf5$d}7}h)MTfZM4ubg1n z>FD9hCifyj7OIc8Ej-YV+}qU%7Q^!K-?|PzQq4HKVnTaXuEY}uyY}NbE-O9yw!-aC zniO1I`7uQcqJ9*OQP>#x=L1-qsDZr*4l|o7>#@>ToWaz87H_K4Af-i!kqBdpn9V9I zxU;my=h7h$HJ0S3ugF{>rMUd)mPCq|@4D80kS8L8L!q@Q{56j8A>*R?&Dt57mTl#V z!f8Uu5igO1R!cSGsC60_O$|FtSKQepVh?A0d2kVeT#n@l8v7&JLP&5u10hvolXT>Q z?bYGK5pmfl+fK5v#uz!vznYIwS6feOo=KB>^sRAGpJ5$1LfoFLT)ty-7>M64vYN1jrcFCZ%RPuFM{Voq=%!+3KI^`_|t6iMei{gTn##EcSnA=~)% zr4I6u&wyn5C>`lHV2AyTpq$l~Q`W&Mltf<>p?2^3fGvv{_v;&D2XeHs8r>)7!z(8` z$o`Yob4}rDeP2#dKPR_LyK;;qb2UBY}eV$YsVJNqWpm^E7PI$hn}k+jJNN5 z`egU6Cu$)0k&|>#HNT!6IAo2#AtIN1856j_pN`cq%}Am`giq#A?=n(wQ(3;eM2!!C zmxbjA6$ldH{Ei2uUa>I+7}wq=1j?#SFkOifAF7UVQz&+8u81-P5Qra7f|yboh6n zZ{;4LLWCbL))uZ)ZjTY*G0yuB+I9El(d)&>7fm#*A%)Rn*C4?IxmWD6)KtdPDeEFN zOM3pb6RV?$!YOcbLxp0Zg28sF=~TwKg_Zvv@7ApKMv4?IL%^o zm~)Tc=v}mcF1I|a??NlWVtv1xo)jLU_FJr4ChN={lB>)!VDFQ@c+)b%$==?yp)NUn z%M+vVu#Q}=BqHwdU(l>`rThE4Q8Y?gSi*&qf zbGpM2RZqL$=56XaH33DxXX#gW;IhYWo@@KTggu!cCy1m=N1O7$9|)&$b&YODji@t4 zbGvpPDPI*6FdcySUy--z)vj*H(#VjMni0u`^-SHc;A4Lh*6I&fsxB)+7;{oNq^Hz! zmey*uuH~F9o}gWy+wnr*Xtk)izWgZiw|78M08ha;@0_E^+!J0z!gajc^B~^zgx3_* zRre^#mMhPuvi^}`*FQ}A#J+`{-uG1w?r)NZJOIo<8ju?HIfo3HiTQZ8^qv7XIa&vG zQar~m0$fiH-{xV6~U}Q7xF9uKc!X6oJJLjWBOgnmDj#OO}mj+we zQC-c~FzptT1-}u4v33rEJoK)c^Khy0olV$0F1{)=x_XsMd_xNVNMqwR4famJR(wBn z2bLAOYD`E=4=IdZj0@l#Y6e`se}j1|B@U!vWm!hNdjePx*Q1lwe;_syBIhuqGrgNd zVedLZg7;k0jFvzs-f)|TP#f_!?9bFiSD8`%@C}bq>1cvH;z#3j#W$mKD8UAA&68WW z`OqD&u8?Qru1&=iUXPx`+hI`{31mqSkfmeWSi8eQQOS|*qg(5 zc>(A+CLlm!-S^XJxwPoJryJaLnFE)$o(PhUc!~Ou4hgs z#B#fwn0M_5&a!nO_$p{@cbrcEF$$Xl?&f++DibEr-gG@KbwBawNpT=Mu{0VJj_?Mk zhffEd*coUhTVx}zY0_b)b}tiTgp^yI5b@@wQ#BChfif!3u=XmCxNwtu5TlFB4@O(y z#z^s#H7Ph|)8z!+2i{+qLB%WxoY9A?-~!}|IVvn#L-sd>Bb?`4I`(^FJo(5c02D82 z&HRy>2nSN3+;?i;w1=k-U5vtxg4ZmAlU6$%BZRaF;?Jj~Q$3SnPgPyni~R>U7DjK3 z^@c7$D*q*YCpn}_r{xlaLs%wyA*Q&s*B<0-l%!+ozj+x={Wp4PqU-SRNz>UDhHaYB zAEeZ8m_o2O!Prj1tb{(bptbh&8A{OFA{9tejLc!7@Kga_dzeV@8O2SX6r?w}Bi*fZ6%JpFYx2MZZB09xFzIAg^jivzt?NaLBhHfcG zFrVv6;Gr>O~bLlT#M!^thla5AMDxe!#lLN=`5E34}z5q_T;+ns39b~M?Vk_$!v zlv?6HEIbW+r8hhRx+xod9%|z#xBu~|!@sLXBjWsKOjH{=_b%z}`wo!Xdc>Leox``O z31zF39L~Q8&8;oxS`2h)A+cG(y3(EP%$cPCRT~1m1L2n&Vsm&;|4bD1{;QC^5DzQB z+=vAS@my-I)TnS!Cfk`!{5QYHDWaHa*XQM$Gn9O-;n1`T$?quSa(OjA=H6QsCUnL% z+cKI?*?bfNoM)H!^uWupaJuXr%knCK18oYlXT0IBm zBU1dP8!iRu$gGE{5gH?So5XWrx$7cKQN|faY`R=;L(szSV$IRJJY|m=ZPRq1FR}j} zupVZdcAP)6N|^uvr`gZuS0KMmd0{)r>lNn*czJv%8@@Ha8og5?WQMN zwT5!JIKWKZ6j}Sz54b&69C=yTAzwIQ9mnR&ZPsA(Dc%^eq6r!W1oR@pv}izM(3V{2Wd;0 zyn5;!Q%pMyEEqqXuq*?WW!07iu#nqI7GjxQ-?%SbNL3&$;C;lb5|6J&B;_#wJlg4A z*NIEhhKuA)itp3O`Ka%H%v9Y4*!n*G{S-K2y(Op<2h00<1Sm9^Wen|7H?LCeC@C8~ zLrNiR@yMvn?mTw2w^Txk^p> z=ul=i(pT1NwP&zaN&7yrR4{ke&@%AAG1iQ9uw8*fyTrbq21aJtC3-O+=;P76S7ztw zRk^Ga*{u>-frv+(w#m<1(KXvrtk+{3l{sit*UdAbQm^e@r89xw*hVE;KMwb}y?WY$ zF2mRwP5`g>+nv?54(Y6cO;-JpIjpdi?U#wuMnvjDULwfVRUly+5C z!b^K=`P&PwdEoeW;JLrB!x)YeKLhaQr+?O+2~v6T=_@+y4`6a^?RO720w7CU`pWAz z`_sz0m1_W{%cghW-~7V=h{xoU~s=MjVN}dJ2O?RMF7QTL}RXLi#y7$3BwpT3F02_K8Azjo^2VC)YjY@&JAoIGsjX@DDbb#xCPrJFk_VLx4hGFh!~ zP-RIz^0-Gfz9ZZ6ZNg31wo|!Zz)#laVn2MzD(v$kD6z`t`Xw1TIV_Ms0Z`5lXs3La zJ!3v~>}Movluy?GxQY5HtSfpT2_T2*T0Gn0(asS@wV;Z0Dg83FT9j-}!32Gu? z00Q3Yc!BEcGV&-pTQu~jOvH=|)l%MJj&q;^$q+%fkl+erS*PbpwZKzsy$`b3_72Sa%2~X;HW=Gs@W?INz$YBi=H>Dp?kABzz4Uf+uE%X-mv$aZ z_+0=PA@Dssx*1Xwz&}TD9ZBJK51^-S1y|g;=&|xBL}$mRNK$yl%5ue0#Lt^sx1#o@ zX97q5y}1Uj6R;K`55wo>BaxU%pUWX6GV5g7_RtZ9$Tqc|4Gq4Vh?*S^P__MGmiCYa!d` z0Z_p25-os#OKmHtfZL~W{1LQ5P^ ztno9s-qRg&XD-2!{@nrcaDOvsm#}JD@h?y$Th@kgPS6!l>Dj2QYtGRtUw(v!km9sH z&o}ylbWIZ#38Tj|qMHMLk0{s%JpAVR&h&Y?95MS%A~i7Wiz7X=ut#b2eeSjBd3p!M zP^W?v?+z3h*-tB!(ady>{wDa#bHXxX$BbnEQzX-QB}`+lXK%|et5-xbye~oAJ1tLU ziazT454!sva#ly~S*vJxQQ*On!g{m+|N8%W59|P^N-j1eNZ6izHdRh*8&?wmYbCau zbH2J@^@Tq&@5oI9Jy{O3)2DbMC9iRGoPM3yJ}M|Mm8KTWB%KjHb20e>`;*AETvBOf z^qd{~5Bt^OiDC@PK>y!%<^98y074Gns{KPAPN!|m_XIJ6`4L&ccHvZvD-ZJWoUhn@Ij*&kO$ta3Fw-p8(Wp_d&WJ{4%62|3+&idWd!lUP z7XkMexbzfsbi*}x$pyQg*HO#L_9SWrNNQ4r%`JxYsbS#KOfBu7`UVkMAY|Y!)&;=7 z5p%xyvwepD-@o7S5xO7es(W`_wD*=AV$ZYxU&S64&tu6Xljxh**vxNpu#%Mko!jpI z*8J@cNJxZYq&U^HWq?uv@@gTo-QzwA_#nU)O5I&T-w)GBdZ<<@9)1Rq-%5-#h69JV z{om?ZffAg*D4VX1pX3&MlcE<kc4lO6k$KyD??PEg2ub$7?U22)Z{xPIipX9elH1;+GHxSV+5C?8 z_w)0g`*B^@Ip?)s&)4gm^Z(bPq>~%;ZmP{5rlyfw!g`jf2W~8=`|lhM=;z3R4>|~^ z|IpZf<~6B}7!?7;8A|Yf%VA{WKpC!{l8>0oZGn|U2|^vO)In?l{%S<~4`Y**aN!Ke zH<2=k9QcyZtqC_jB>nFPTkykwPbB{D2e!PtHZAo-+0YfAJc~c(mQTUOyG~QGpwklk z!6OIy-BQt)QYhmqa8l|2A0~RtTm8?!-R~;lE!LpqLiRsGGp+!!h}^g3C3{j^`VSYX z|9bcTs4&mdKz_|iCgoc172wKB1P#mn?~OpthC90bhdLsjYp08Ud%%To2urj7w!lEI z^Oo=Vv^g?mN0LC7|69&a9WYt?o^-{;>Bbb8EHw_ao=yd#-&<#Zfz*~N`)3aaKfqLu zFL=hhfBgQd+QJV6??nONl#_knft_A}1zF{MXMm6Fpx8vk%NIZJb~pf&^@dhBwEcVc zeJQIiRj*th@B4z9cQ(z!TYf;%HP|Za;9P@YTncn=g}uC#s`g*UVdnwB7)cQlh!d z{ZGfcVc~s#bZBpmE~sZwmYVW+%#`JUBlYI)`bV{L0kZ9wr4O%4?%v2sbB~q4l8x*us*HsI^(d+ZVmK(pn=dtLE7_Zmloc}6F@~5cI>zXQ(_>d9d z6Wo{n)wd-`J3P$bhUb4?9DM$+5$Dc-=C}(2QoZ%;@W^x1@6V^#TdN5VpZhcg5?em0 z8H;y#|G2EA=xqG*oq-y7%_-=?qb`LMR!OKqx=`iT)#q1r{@BxJhX}%Qlw&>t!({su z-{UmRv~+8#HT=Iz-LU*uj2${g8FH3QWght#wXA`z-3woqroasv2inJrS+xv0&=D+- zt1McPjDhnsaE9;4smA-`%evL}zHcQ6v08uUmn3I}vbqOMrGF*-X)Q^V3kZf3A}7{p z9I{*0(spAsD38B5v;-1zkIxjmBwOF_^(3{t%5iy>(-6my5#CI#GFtHMdf@s46nvNJerNt_&e%-`oN;S?|c#YdN}U3-@j`pFW*~!Ur zbI=QUY(%U2lg!cp-lOVcEf4ydWxo}5e;;z*@$T{$K+<-5ig((tGPLJAGEu9p{u0zt zO}V)Hx!1LV9hN=sW3(G{-iipYjTm~IA+rgxjGQo~cik^Y>nd%gsZw!b!V?^ZF-Z}< z{ZDA8{JFhfSy$nCwtNhQ~R)7x35G{(>o3-`G8O@=koe3PXB{ zT2pos^UIuwzjk@n6NN04!}i?PKbFNTf$-qoH4-j|tKQZq;*76bV#gXghDMmui$s0a zfmaN})pg%r(ypPGw2jH2Bhh;GA9sm|FE|&+KBwVUy^aDy$MlO%({;JLV`Wv( z&ev;cLs?;Gxnr{pVy~GddvgZR1RF9bW_4aHT~x@IN5|!(rjN8RTe}GO>}Plfsij97 z?-{1Pg&bp@d-I2eBx}8wu4kn1j@(`91aGgU$u*)j>O}qSkxlp&i$@IXNEZs~X2vUr zV)j;*abFI8NlhYKN54wYs{n-ZF1-hNpG+4Q*P_~wB^P%&7L^f^euZwv*3$;Y@tf|P z#}b!I2u&?Qcgn&vzaeq)x$HOEzyA(J6#e4n3a$!}8n>ous~6edgT4ej6^<5Q;<>t0 z^;`NX!;5TsuT_Ze+x25sc#rq}K^}kBvHW*vcMYomvMBbV^G!l)#7~J%@j-HQ&Re}; zhnB_O^Gzep2;^IA-xh~ry4AczlF}hwwPLJ!(RzpczrWm)v8=j}d-Y5G-0loB^}A1B zy7|&?!a}0k-AE+a7Hz1|StVZ^b*RTVWA&Fdr^V@!ti!6E3Iug~kb3`-7ZnU1tV_^5 z+}kT>OxRJp8_%)7YQhTl<4xKix6|!nU9hbaeaiLj^YbLJUFhJqY_u*@zVGNhXt+z=9BZC z5fV2EPwq-WG;v*7$GysA6OLl!`IgUxf62$qD0;PK4RyphT;4Caswbqc@z!lCHS({5 zvfn>Yaw`XG$KSztl(?3}bbz^9_9Nh@#E;O}Ft~ZYs)AbfSQE#o@KuSDZ`Ktvwj62{ z&c)}=%gCCO01-P@uKt<^ks)VC!KGv7%S>7TwvctZ zr4^Fv?EU5FcbxdTW59^VPuH4F6P>+fhTE{!)$(@uq|ShBT@F7Dg|9CGjf=a&@SGdC zr2g@PYtU(K5vJCV+#>D>E%KO~uUe2m*UX+6EfziiFF26byC$gNZXOoz=T*C{;6Qqb z`rv|y#oWwiAWO~j^h$)Hd>8|s8QV3b)a1hk%`>z8HOf)k zZFAE5*jcLuX73CL!m{v_c>42l{z51pxPqRTCgU+?$M)~q{d>|Jq2+GV5&Wep0!iSz zA6$V_XM5K`Qki?;2S1E*xvRv%~W(`2ZGOIs~6wOLg@>EcurqrwKyqDy87a0lv90Q*>kjhSXTN zPt~##rzcpp{c7i%T+~N8eB2r~Md{lYQTPv`;q{gWKe%M4dGR_DR{oBUe==X?N%77Aij8y-yWGfK}O3$*k%C@cI?>m9F)vRbdH+bh|OsH z0ev%bFiG9TYjr}#JYH{zF4roqtL6iWIxZaGxKC-1B_flUzuXB~8w`Ut^X7QTP!X4Q zmDxnp-z>=-MCP~R4R6y}DYC|)o4hThNpT=6yk*&}T8DWpyi6NI?1ll6XIX$kaT;VL>kbiNhzgFlGo22Yu0tJqyrOoF`RZ)E@dRZYWD=ND}F9msG2s5LR-DPc|l9G`KN!)TniaE@Y+hnN2ly8GHoyn!6mA+mwwVh+n0V@{(#;rNz@~k zP%#KFIe#{kSp~&@pNz7w)1IF-Vo$#PMeB4LG76?=n+$(;!6{+vTrRB$tI6N`F5HBh9V)}vVLe7@NP zw}>J(5C6V9z-ENt$d5?T%yBuo<8p^38No)IHQYpK=a+r{L!d+zYr~D}f^XIBsV#e1 zKGr{00Jl?$mBp^%9;$QiS>Tns)p`X8-HG`14V4scZE=M4no0eL5~KnZuU4$Fe$Wwi zF8@v!18Cu1Whf$+eKiX&`1su$^;|4RU$T{lj9QosR??nM@^j?+Xm8nEDsYRBY_6_` zzg`hYSWOcWDr0N6B3EIun$V-qY6V^j`!;|b@qW|((YLcc>d6_BgqY9MMHMPrLvShe z0z*ms8DFh~VqBbiGGaPi{tK8^<_;GdCi#9~A+SXs#uiLxJ&D%F&XasWkl3h`U<4e~Y58jSmT}43~%9tiNOdI=`Urldr%f8JE4w(3h^mDyqWc55bv| zl}etI)vg)_$dI1Ws5{1YursX0O3Z^~&dRNqE9%uRtnz;vc%qw8HXt+HVT;m91iUYn zr}W)$dP4)h@K@`1L*ZME>BP#r0H;`Ex@`3e>t#+t-0$ydtfD%85)k|xtfqWQL9ar1 z0v+m_nMV#~P2N%!T+{$V9P;|h<j^ zlp?!UHyN9G&r6hLpbi4JesLXXY8;J^Acenc$*=qj?ev1??Fo^1l)(K&sQ%$UiVznd zAV&ghXx5SZr;(j;JI*xdf1vUK^S@w?(or|~{b+7)L2>{+I#$yXQEJFoqkW(GBn@!G zQ>{R;(MTocAtYC3Kz6A|!zur2^-6)&^F@Kjxz#l6R?Urk%-xS736EOpbJYuI(_Wr` zzy#0m-JEz0! z@LT*^A_eQ`H|=B*+h2Wm&lD;mf5Fz1aQ{STn=Jt-WjdDMMRx0k`WbPQ8XR@BC~ukIyvMOctY5Tg$_ zQOdo?%CfhU)&tc052FN$BK)LkG}{R~8^CHj!D>7e>Gl^k57?RGqVW&mfdYaEU05vX8nZ<7wd@F+UJ$i zpGA|~JF6Z1t&}0{z-n{$i8nZ5`X*7+^s(p@W!(i8>Ga-urm$cGwF22pVHB=PBn$xw zqjKkfaHOn?ljIU-=gV$Xs7W<{7aAf2%c!t>eqWTWU;W5#f0W?guhEE5J2INk6?P*fH2uhuo{;b02tO$*k?4PSRxM8io@C>R)##by(n#4;F;5C26`xl z>D~ld4kON8_EZQsi8n|Bcfxx$8T}iar7{9DbihJ-*SRw`#YR!s0lx<_$b!5ml@qcU z8uD$h3jim`DxH7X?VIo^BM@*~O~QNVE{MG()-UM^I7oT#o2yHeQ&_=D+K#IAmx=Y5 zayPy?T%ErXH8y3NPq+Tz0@uvBWbzE3@u7tF0=){0J}e6qe;IXoyrR>}N0$G(z0Zr4 zN%^#VhQ@E4&I_&IsNz@>eWuwfiV8Hw$XN&AN%XTU9I z!g0=B#qN!jelPFOnXyP;@^ZOenH7ZWzUyu=#OtVM=&>736@+DDOUg+%At;$Dge8!S zg#bN`3j;6T9}0)l7O@^0u(VE2?riM(Q?U-63T5>Mo2%XEL4^vC1OEG!CqJ#<^x-6r zo!x{x@L09V`P<>aW-Cg*FTJj+Pv%ne@*eF?>7X-9!cNlrWo69$^M!2VDg>GXC~^4-`ww!bxOdvTTg;USr;r=6^pF*eQrx?vethPhb9OVx8x zyqBk3zXW5Zm^P;N+a)M{M1}e&F|A@o5C4RFeC`8wsnmNfXQuHXv&38Zp4!z4HRYj0@igY&l0iki&)I*G;my2uGE)3JRp`HQbRHK z9xGtD?Mn#}9F}8krcu%8{Kv8;t56oKBywEgM60R)KyvqQSoLVNDl1`-mO{o)>fYeC zNm7M~?zl8k%7V+TX*tm4Poa8rDk=0NDG5z(Jxv63=4?AC@a#Z|=4Z8N!2Lw~h#%nU zn=D%_(GE|EYx?Pks6q=dV9XH2eT{4sL|4uCu(daSxBhqr6m#v}L82;jq;>9TrFnz& zS`c=>0IFZ7$cCYq_rCZ(aWBhAwipO)takA9n78MG+mMGtY$(5b!6d^ZmGgI-6q)Ul zxVb=eQo}s3w+kAUVwix-EPUO%!J=dJoN@tYz#{At-u!LUSoNIw46Z^mE%6~Dmb?91 zx}I>h0qZgmwwnQ-^3w^2V`wxt@j>;D+0>xFXZoFp=VT{7LbeoNq@hx1>3Q7M*f=Sz7jLAYiKvkXkVyMHqD1n>1J2);d@&d`A_Gglq= zx2~OhEgFZ~Df&K-k%hSFvdx&g4Oq4_eK>!wpCGpH7Et^KA3=Rhg^A$c@(JW3x=fO@ zE7UNZ#&hH!*tuM~8 zu3J}O5nY}CihjhCTdxC2p8wrENj)}qC3{*?bBT=9D}T8QOW+I(Cyn@eI^Eq^jT9~9 zP{5G8e>h}%l9|xnVml-;;5KM^0CQt~d328<^-wg35n~$eZ8_1FW31H;)4vKjoe~SE zTeH|?%(kFb|938VYLI3m=*3Dj2huvn0m|277jxft8C7D+gOAQW;NFo+qEc9-9>0ru z_C6&pq-TZMha8wYnxSmBv(S&I>Rvgggu*wF=+i};~!1Yk@6NH^(S?|BrL#nNFA_47C!glElvUV zHiD#$3fbP90qzo%L^!AQ*|ks1-gPgN+vB|qKYOj9!GVoc_Di#Se-)S(WO? z@)$_R+sdMxx+@v?$|!RkKhXFQ9LS~9>8Ek&>H29dR`2~pG^?aqikxjdWz<)VQE z_OgA+?{t(`lxtG$Y`D{5BBc@(IJ68E!dhm>SRL``bO&<(Ov9^ebegdKFARzX2nJ*c zhb{KwV(*rv!=Hy?Kd#Eucs!(Hr(*8Kk2Fjrp_@St^%oXa%+h_orQe(cLFr)l_GS4m z+Ew#QsH1x=jVtkLz9q@;-48kp#_h+ ztOWdc{#E4gvyuBbLP@jDjxk;FMbX-!KIN3M?YR3$a{NzNlm^h=y2NV1XFs!wtSG!r z0GDljTANH0;rGp*T#WSXJ&owgjtS{-H;?|dRD_x)%=`RC4WTG?8i_Fbk?4ef{w|p_(zt*a`8}; zCGx4xY=sbc^v&1&ZX2xv-5j@du4=!R|y?#!DG0`_lU9;L?Poz@|pq}$6 z#*o6Jc37t$IYgaOfJch>3uzECapRdJye?pGV&A*Jkv)V3GFFSVq*_?IyVCUDJ1`;d z8V&`$%9OAE#kPJzHmH<7HFJ+AC>yOezwU=8$wmH>h8x@pc2id4jnE4}k3*SX>%jMJ0QJ@L|pjQ80xpGX?m zQ?2#g`u3ed2dw21(lB0k00*7YhI%A?@XLQrwY_JG0e`cWDAtHPR}#TtW=IvL!h!o; za$_5-_nY(|`u65Mj-}Vly4n$7MQs6>$|&Qozw9&o;CU&?EI+_J{-v_e%= zCOQRGyQ36BDFGVi>ZRhdSA2@NtLig^F^5CcWhPV3`VbEpbp2BL28}?nb8^*LYrnA* zA;Vif%$2C|Z1R^q0W^IDd*IxwbEu~7?xq6o96=V+I%NMo$*Au|eD51elO)MmsnRwA z;pmsGU&*t34n1P@W(WGSD{iv(LA__Zdzc;n&4GMzqKd!4|*o^dezmEo?(QTjgn>RVQMY+OI%vlNL!>S)TS_wri~)_wJ8UC28O7 z@tbV}y$eHY31zdIWUa0HsS@|0ho{(>&JPu{STt-XVY&r3)tw%A?PZKjt;OsiCM@prV*4@dKkVR@}OiePXL zMsYyYv84E%6VNImE-nTf#)l+Kcn*r$a7!y@3Vfq-^t9rsA+wc(mL&dR?Z{HZj}_6j z3!l?L58A_a1*qb*rtzl7K%*dtXqHb*Y+aWx{!j&fnmx7a=yXPs3TYx7sHV-5ltbofafTe8~MRWu>`J=$7cv8M{}U_Myqz9}HA=)`jWWk5*rCK!`pW zd-i>dWJb=Cf{_#vE4|8oQ3gT`X$@3aw|5TSE$EN?l|8wq)M&9IzI)r-rLDS%gvoHp z5tA9t_P{9XceWbCEgmm~o!((f-li=Kyf*EZh)2-ae8~IKs(Ox6nYf*mp z;B@?cBEoKH6C3_uEuQ?akNpqY6}IN)pg0%_x;j4YK8m(`w-u>;fa5*>Me!i{v#gYi zY&Vc@CyjEJn!;+nByFddO}BbxXChjFK)jdzB!sIV=p^Cu>fOT9@m8T=MGyapsV9z|FfFW?vjGwdgFY4)?~RO=X-=6}S+S8wWWJ!gO%2a+!*6c>hg*-p@T&y(8oSy^g6<~lr zShemCLE!+oeve~5__7!?;Ju1Mjq&;9i;h-;X%ILfz(<9r!(WDVxIVQV60c~?KK2uHAHjt4f&L%gWmxkIdrB21fN4n;6Mz<{45{LkM&jwe zwD_Hf(y)Khp)aS8uquFWThDs5!|w2UZlwnuZBFMHiU+NNnW#C>+p*}gWmQEgvx(>> z>)^pX00P@OI?x&$T>Pld>HUDEMnV!A#4M2VM~r5jA-jmHOBU>(%~(zV+^!+!#@`ep z!xUw9=@lo%C-}vQ0dal&u(RA-W=RmYN5kT#{u(HvU~=-6`a-nCtJAD9~C#uND35P!hQ^`X!tSfLc#yK2?9s zMK(c;j}d>Qg2$P{?GACe?1)q1yQxGU=`zFH8}Eb^Ahj*dyPyhy!%o-VXD*!-Q0Qnn zkQn1O;GFpOoEC~c)WIuXra!V4d==5&3{WesP3RGDR8A8&64$1s)Ww?7ssAgcUj(Wd zm>2R8$cj=jR#AXjKeA)R3wy%YY6!q&sk7}Kmn4$b#EHvGycJ;QdJpQU9}L1RsPSdi zrnc(-n841-I_#vqRumhtQgi}fl2F`1F{pXK-nATg2e~`=4ndf zb&m(%p0oL2IJ1%&iDWzipv{edeZfy!|19(m7xBDA0fL1APLVef1aOm0oGRm)jPNic zSU_6INd)Po94lA>@(0r~mv*g#RvcLM0ynxJN5ZQ9S@Fi@a|}I*7E0`5ek%&PztEwFgSg3en*Vly9X+bS2LcpyOB)Bg^F`PW(xpd3~pPBVb+WC z=dZ)ZQWP>!zXd73Ch=u>mt|Eu*vfMwW|ZObG(B4=AbGhfg~Y+Vz%sHW$yLM6)1Kc% zEV$6%-Nx83Os7YnjQ5_P)2c|vqpb2gSWoT=7=o;fUC^}yX z;BZxF6;Lj^&MUxL2z#LDjJ^#C9H3`+vEy`G05d$*etAd{NvyFfD9a;lT%5Q58?zZm zIRsOC#bvukvP9dn8xgJqb~L#F^I#50`=V1QwW5mf@W&azAh;^~R_{{=-}qRK7jS%A zA2#gQJUMs4Ni0SPEX6T=`P4dBw9sh8<)Js9Sql>$zSB(~^y_MWoG~@lpO0)RCm^e4 z`g3FA9A9|8rt`e>*)#Iq{j&s=rtAdq?K3cujwYhf2<33TlQ>W>(fDp^HT<6en>Sr| zI-BvOsn$JhIa16zvpxiLOaj)i=k_EG9ELZ=QV#R1N$<&c4tja#Rp`9VMtinI#03~c zLCyz#L472p;9x^75h&`9VtW9U1lYrk)96jlP#dmYJsgzubF{FIuS)k6Rmcf`{Z$R{oa9BbTd8t2!h0Utfl zk;GkK_pW!=r#TeLOAn;24zLbB2k8jZ0ECG9<#*u);DrX-bMaJN{Xs_ubfS{oSF+Nm zz4ym}dM*4x-kS%Pf)+V79oqoOgEtp>>gtB&i|?kh|Ev{U>d}$-_gMuC9MCC?j{zG{ z!sAq;t{(G7DxH;NYc=5DY5w;PJ)ykoD?qchJg{AtY6*0XPlxE>Sq1ozUnsG1YQ5aZ zP1@LKIdXq>X7YgH=8Y#`gWr_P?1Iu!@)P!-Wtai%BFc9iLUEwh_{Y+Y!TGMQTHDKMvNk8XW{>eW zlV6((!~&u^eGj1Wv=771VI{U@dT5x%b*fg0=*>&KZ#z88QuxL~pERxZA0rb^Utx4W zCjPe&zOKM&$2D8gQ1u@fWdMt2F31IfX+2h{k#a%ukm5GCWve@%zU`XI%=uFe>G7Zf zFXCcBoG*IrPZ`3!9ygw*gNLB51_-|TR{XLzGWzIRqu7uexKSf2#ericX8Qr4C=RR7 zPeA-;?B7Txxg-SHPXI7WO21T*F>91$NqQY^Awg)?D6F-_F{ZoBCLnj`hL%jZ9aJj= z(>kotBBqB+hT#^ui9LbPG!4qgs`St+3aJA66tc0ie%-B4xh?)~s>Hkc91zqTM0Fag zzwqV?AJl6l`xl=|lJBYy^N)EEL$eRbFs%?2*>&1HUhlWM8v2hEmGPZGM}uCuWf;O+ zn7Pz)v0s8Libc9U+Mgw?=Wf;>Mu*A;fR5f1V03h^+6dIWiF5&y68l)AC@4wG?#B5f zF}+K1X2;g{zK^ZkG~3~mIQu2tffpOHYQuvXMg|xHIVmyufO^ay8I>Z0aa(U(P#_Ja z^g#1NQ1dNPTk^yOay)=8v7H&?A-p1e^wfP8U!9N^_H(R zu=TJ-w_`)^I^bB-uS%$PVK*IqDHnU6$SeH8#pOXLkoprQA2_v`!8ld~!t2G)_nXB* zpkm@cpS2cE1>^ZOUA@*8)aF-{<>z1)nyeK0006LH}PHL=sd z1DO_bizQ`nXzJJOMAZavOj5*W=N zr2@5NMvl2Lt2?zyg(o>j0oc%C57k{i1zC4Yn$nhE7Gw-(LUB{7#v8mhQs{OM+jv;C z+=AQU``@YqS*9g*Fx+zxHmye%4Z&}uH=I#y2`lJ>MuUSx2jK3rueeD)3BD#oWrtE} zIKjZcjoQk5XwhY#v8DZ>fD6Zsx0M#&+7D8IYJ7kdtNyWT&C_cg_{g`Zw@U~HxT}7} z2jU~(GL4$jwl1mIXT;cNlywI{BvAXSiQkYJT6Yc<0R0Kc<->*#eakguM-~Happ7IG z(lR@Q40(9dN_5UgH3|o+cEj{1XD+SOJ1nh27+7#WiR+e6LRF;D5eLVJ$J?-N^)1md zafgw`R`R5JAQZ~qw&&PxZ(6;!C;{5;=%~Ie-kEzwLM?@V$Z!kZjE>nbb?Ekxe&y7J zGaZdyJEbYCx{gxYH+qN&@H2m*pvRC}xjK86vEmciw6M|k z0n)ogSM}j2k`!_ZsLC5s%6p6b$B3GBGz;LYDL+$5jcnPm|9%Votiw8R!M`fp?f6x0>uNTBWX?YZ;&b@E)2f>a{*mYgJ)XyWX#DT*j)`ynlGO`9zid<~e*U$p0?Nv$u94g8<`blY{xE3=odwlcXW#_B+I!3bbPy zMUMgGI(^q}##o(A-rPx#Yl<~bxor#obH95AKq0}&6_|u-8LDcP(>qdj$1n zrTr7t(y6&X&q0l5qG>;*Hn$F~VdoM)@tt3(G;4Fzg8AR?wg1 zc$7x5p?$@V)uA^-YWBdK_gW(uDE`OV=`m_FwH4v^YG;l-Uy>* zJ-q1DHbo>`#ggBX=;!x|AjMqaV&$(otmVR>GvUpu+sJ44$TvvhBCbRks%TKE!u6E@ z>yu-Y_qYQm^|!Xh+}k9MVPO~TME`V*J)$D^c0vrf83PVUzJ+{y2WJ48#mIKkJ#Q>N9o#3=ZjtW!v_S+do0h*9kUStn@b9-t#|p?s9>is0 zwP~ED=$^j4Uk^GNr!&2icSCv-W^pG{E*a=>?Ud901avO#nkFa-OaxYa#ey@xvl|me zxmD||#cZD|;~}IS4h>r@zFIQgnpoE>=eJ<`ZE_m=@@N}m;q@Cvc;yd)lLx&+&ye^V zMSJaqqC2afPKh8COyf9yT?uE%?x|$h4hlohiD|rZ3rbfNFA($w>}4= z3w6C%5+q&t>CC!K{yIn!YKS9oP9|<=19huvnbV7md}F>Q-0LW~?@8(CWG4g>$J^-+ z-u+MTQ8+_TJ8+gCq_`8*2!WFJUu__fpt?!RZSQ&D*E3S8g?r@Ytl{Z2rU(2r`1U>viQ< z)NeS+CYt;EMFx-ywlhQAmwq1vWmJo!GgKN^fZ zAVOf)Bnu+H8do3Yl~6U2&Sz>fOP@i%pv5Qko^4(M?XZ}MRnP@LzB6x9?^}l4)uMqS zI2Nz}YwH4bX0!m=%+mJ|Fh1aflsBwSL8|Q&1EKuL;?x;6Dk_N1egGtm@bc9kR%Q)! zyO??(>pV%TjiPe7M?Gr79^~|XY^c(81Xfv;JI_ysEb4_;IAVI&%UD7s+lTk8hMvtBPfq ztcW3U6&h z$QB~L-~#6&%Ch8be5X$N{NI~)mqNstV*hAwvig15nELEh+EoG>s1(E=v#Mrw!j{;K zUIHFWNC2UptG||!OerNibQ_UYWt0O-`L#oOTLB!Osq*RvkfpyGc+rzu#}O@lPdTFB zfUZJBvCp!2+dJg~d)LaFbp4O722s)=;FT7s-whDn-jgl#mR7oD^&3&8pcXD*+r zRaL(_Xa00>lUSN)-qVNwgtc2Ll>u%K5#)(^`YgUZWr48R5=oSkl&){a5(52sNuY9$ zC+~AmOc|Q_>EYwboMwG{P$CioJ1scAyHcZ9n{(p z{TVl5I|J>V!KrGwV%4hzv$O4f8-7z!N+V{~z53x6RYMuZ5QVtQI5Se3d|?1uSCo|D zx2RqgxTdUb=wuI^4!7CTv;;zXJ82f|DYy;V|0D13LoGj)$MP}Z25WI^ zvD=^W$>0`9kACl*uOJmI2|F_~vidN@Hi8Dj`|;(NqkCs$2w7|caaHC2HtyHu+*@(*8s11R9=WKXEerr5pwJN<;FA0??Q30@eK3-z>1n z@Udxrg;Qru@zN?5EeeEng*;67i-v@MJtC&$)6f3w+; zc0$#5>?0=Jd5F%rN%RT}Ua3iU;bwf+9#ZbUt~o%QgC?#G1^RcP-NKw(OlOk-V@moi zzAiG%yW2c{%ULkMYd{!(h=FIuyOf*`s+V~rjd!R$m18f$7kfSwh5;ul(}-L|#(rXo zI0@B6Ut1X#3nG6W>Bm+Y^OKCu)*xk6{TQn_KL3YpqZwwni+ zWg*U4KI4tY1(4Ao%xB{D6B!G^9^kU@9~9{!?4BgU}mZvYsHXh)lKmjbbMk z5CaWQ(LA)^a^3ls@XMPA8ybHOw)vS}I5)A&b4Ytvbo>9=wNgWLcdXt@sPj|%+wt0z zlxu9->yvk=HcOwn{cMjjhgaih>#`gTG`%+udS}o%e!WC}U-d1aao$SHqVgkeWciU% zjtnKG$(%zr`JdaXk*RW#7~jRD_~s(DdDh-IELpVmcLW83P{HsnUI#lmj-%5Dt@yn5 zA@t*>d_>nhDzPDpwwu>7vSKTvOY%^3th^;BusnL{jr1vuHSMO~y&>{s5&B7s&hJ#R4X%6{{7JE&`35o$ zr09H2ut?s|3x*y#c-;1y$g>vz-S7_SBFySA2>uWqtNUi6QG2Qzc}5Rk0~w5#Va%~$ zwT5oBq(|3fuWQxt#YY{`B|^F%k;XkTDuJHe`G_b$4DN1nniTPx&z8X`BLwPP1?D)H z3?ZwD{nBzbH)S{Fw2VSbZB26=x0qw<8#pL>D)n1OPNvH$e=texlXChKVGCTAE;7LV zPkI~8hGW=nRF=hH^ECq>z%$U4GK(PGa*8@vFF^hCAVj(c#9Xn zaIfg-vKub2G!Tm2pxBc!TL*SIh5>}eNMCQot3C$w;B%Tul50v;yExcc)`$AL0HCZF z;V|~sAyw^w%KV$D$vbU8u#} z^TcB5oha3D^Oe@q3HXNmE9@kmkJc`$*7UdI5v)kGt^m$K_; z2h1@UXg{Th>sAU&-uXphh!&$hz0_Ws0jU^jEqSg?L16C7U$-)l>&`c3i|b}p!0wxf z8-jD|WdAMrY+MrzY1&O^IbvwIiyA$=hOs3thOPtUwJ#{CR+a2B+_i!v^vJd7(AA{!=>%m}* z4Q**U-rtw(Mva!(R8@U8OL;!7l$qCTJQy-zjLW>LY!<F#B15x(zIW_AKpX@Slj@OX@2j!TsSJKo~U+-%X_JAtGeHC)^*KEW2 z_cYbrnun+B)MX0aAKdicYX<}|237+U=Ha{>dl#hlH`|q>+A{RtAE$`eNvOtFA|uz# zJa0or(}rZ(?#!Ifv5aiJ#wKaaNuD-A;v7)_nsZ%HA8P{5qXij$D2DA9Y8ATgM(s`* zaYE%Z0-m7x`pA({tX%I|GJ7vJY*;`68jjDlhp9iNKSzGF&?t%zJsd52T&@sgetsL| zMN%~>B4Cb$_l~x64*x{2{y9AOabRi;n?Ju7Y|-6n2ON9P5Aw138qmwPwlUGg74UH+ z)A$Q@Dv})2e(XOuPklbD&$)Eqo_p_N^7fX_#%d?Xo_{x*UQN!`6h@~{;cyt&9b{d_q89CvZno_OM5 z5RUQf7DKu#QHE#>^5Ivb3>mV!-1-5Oz$ZDGRMDzU37k$S#+n=G>;1bIa-bb%#kaxx z!_8VS5u^N&d&q7SwiZu(l`%G8^GVH9_N%PR>9wbPLFmEL8{7q?%qapc-dRPCW%tmr zLhRum+6aIOPNsYqppM~c538>HhoCU5EQx6iD)Q?yVU6}GQ}DW+2JpZAUz0PXHMP~d zdpZ1;S$~pj#_px`u}eawdKM)bSwDxe({*l@^DidIMLLz`EOxy?cIs+g&#BdwhAAh$ zVIYvFT*U93C2z_)dD==O8a@76K=KX_M3YkM-^LB;389gO!J(NkAPMv-)xSVoxAR># zygG=zmJbqeA)eo?9GH2qx_KzZqTgaF`TYqE<7!USll{f7exn(n)J`)*BQ_X6iviL8 zZgD3_o$UO;2j)>SE|8ifRpCOlR_V_E$@(-$lQ)#v`R?0*kSr5)uMl-P z(xD-Q`H*IboP%(mdTh{skoPW>X`^4{G=x#vt%N7~yOs0{2PoZs8MnrVBJO)|#6I4X zJa)U-t`!vc^6AB3y~pX>;7t(4)b$jobskvdlSlp5c2by`e(wD2&7&Tr7obQo{QuZ{ z>$j-BuVMHQ64D`{NQem1AWF9&-Q7rcNXGz*fC&f+DlOgJ3?U^sNDVy-l0y%}z`($J z_DK5Y9zHNu2t*fd84v*xpT;y3O)Z}43?3hcJ>Wg|{G3Q%{0>YWm*#&*&reDF ztlND2N%KVC^VJM)+X3Z%94RpWxs}iQCP8Ao5Fy~Dj{H^PZ`W=Rj+t7wejki|T)8X6 z4=xv{`JSWC@UJ=X;PR;MFXuxiUfRCNdj!CIO8;}!pIt;UG_PtSBC!*NM0O5GpB2gAq>aX!)AJy%T=w@8R(Tu2K-H4e{$@wX#fvzSr2BT#`2P$5 zRuy(We03T#bY;2dsCtlh); z4Hkqb8R}++xR2jS6~Bfv0Muy$9bi>SvGLZLY#!&tust-k*q17Iz8OSSFWRbYi(2P3 zJK^+iOD4w$;Un?QF5uZGdR&m7{01OaGNQh*TH6>bM2ZGqQholvmmCc^qtkJDZUzum z#n7#6(jF7n6olM@4xnIF0NpMCcer4%KIP5Q@BUg`LQaXfnw9@=;)%%Qn7+`mp%^d8CI zSn$S%si^T3AFw!5-HrrnGpD0F;mLe$@)ci=X)HSw>E+DPwJn{jIsx+f5E4}2=BZW+uL&j zx>moqR~kOI>p6LyqaH&Ie3A)04@;HT;J|SRH1Rm~a?pESOCT&?tvvpUGfE4j?es<) z^J8o7-8K%1a?E2ON7iZOY!(Dbfqo78fNjI=o0{p=Bk!k3?uVd2G(Ng*k3g9jdG?FY zy~;F*W$pO}!j_V-8%_x*e?~mlF#WaR)*_EpQhv78HlwQt7MFNE2;F37>gUJ(?5AQpHU$x6t#9V*7y zlifBp&M*I3-XbJ0Vn7asX#<)EPzYhG4}@F0$edUp#I6_8PCM&+&PD#Wo%$6u14Gf{ zo|~&oI-u{aKsS_sR0FaDl*e-NUmoXgAr*(majClCF(^m2py9Ygxc5a})ZwmGCa?t!F z12t?#k_>UjKLwI#k&=CtOYrno4VH0qPj912)LyqNbGj~^)KR|{uE59vcK)0& zJV1{9?eYVGMZD8rCI3Q04DXSw^4j|Bzs}uK)HDB6P5qWSU>Ph8`UIlt^aD04fQNSJ zv}2kMjHu31q;PaE?XZ2B_T%QD>5rv;opVz*t|yF&5coj1gNz`wGPyFj!gKLi zqNBdjZ}rr#5rd^daM?z;>~M#S?omyUY7ORI`)k%Dz#nf7K1{6eYZs~jAeAa8g=0w% z!Wv(?&c3Zgj8VfV08A;yM-hVb01#~0dJo zM7PGoMF~i=t~{&!wF~L;FW{5TB@Z{XtZ9PTX`he0X6Gbhv;X=pd<87C?t)sw{Z^E~ zN;1~U#uxIp-i~t#WP#URUF&LD81+7i_aVXL;&oaC0Go+T@D0*nVg_V z>o>|~DGa*THP50Zw{9eFu(W_moC>AL`LjALXz``)H@yzpx72Yam{R;`qS2%9G5&nI<_BoX0wtJRC9$Z0S8&;6QjbV zJ{B@OprE2B5~>VA#&+_Ry7gArD4W=n@t*k``kwlrO#qNH>5nVezA(5hi+d{GGyZ+2 z6g+WP%grC6cMGUr+rO%%Dji=P&m{ol8r6nx%y9X?-C@25u)hJb69dn#Zz}r^Q<|z= zevTJeKq5InRPj4*J^SVv`K6xw!yjP;pRRQ{aF7^kVUiUtMwhp$4r1PlC+h0RJ`YxmyNoK&z!asgPn(H<1UhT1Pk+!SV`|P{?K?zbtII9!C zAvxK`1c2O-B!~(hDG>(7U&&MouGTZ2eT`UW<3ZWDoMl_)uVoAs+^{1IGLD)&cnG#s zld+mhH36>@Wu_Fs)HaRpG3XBjRT^b%titLu;NKVjUOg(g+R1*f zWh4Da>2VYZ9x~^;!@#<_bVm)jSYhy8$MHB7YjbVeA?QpO6cWCc<)%poJS*j|7W|Ln z?BLrGGrCkP3k6ObxP0$vrw^)}sC*{g@9OWl!CO?{sQJT>!({hJoQjF7nVpiP=!k%4 zg)-+l0@Xx0Hxjk75GJ%+@@r9u_t=a&wGSj0I;-F6LkvI|7OlULhYE(umEsK~aXR4b zJ=q+@=!Ad76Xtr5>&qWXsX#XKRi@MH*TBBiC5lj5HR)}y^Yc98({lN%pr)y_beg1crj@`fu05fL)piBf78djN$MDGD1)r??v zMUf!t^x&Ha2T(CEm+MopQ}y}gfVl6G>Rw@D*?NMusl@# z`M^Im3{+MkyU{}bH_CkrlERjmCnL6TeTPgXmYA*him6(F+!*nMxE1nMO4P!N>M1uOcPWn=DY z!uy*TK`(vs5lqnqqpM#lO2;;o6j@&1cz z&X4B@N;cs5*)1@ibWiaYfE~= z_W25#9V!6K8NUuh&%Q}W-ry_vcrJ9-iCky`;6y5>J-;Q&94(bxls|HO?F`o9viGSg=@TK zfSB$871*NOHJvuh>G1x}zgDDV=q0R9l&y1tF_S{JNv3V(TT{q&0w5K`nMCDz1}ruv ze8PaqE}YW%4lsU?tao!-nZz#q8HjC26p#9GT1V)N8d}l=G(pNH^L1&xp>A+FI!aby z;cwrmL4Q42MqCkJ!G{RER+V|C&8e$;ntiFE89cM8Orqreg4u>NfPI?+%^`r1%qAsH z%rPMBie>;l2mrI+EdCt2_ueyW2w=Oo6Ve-K6e&;l>CFbn_3;197WLq6?^!5_4N{v@ z;OAT!o$`M;2No?^m~ou6QLx|8!9xkgo2ex(Vpui|k_w}oAE6p{`qg!v2DlKKf()}E#_h7ZSvrQ@GwNC;_iwN-{fMSx1ZQ^_l(z6kLV~D{J_x zDG;QJTFD#1wP}f_k|y&1V5=bUg@Oz?*-dBZ~_-{}dqCDwhag)lsO0KUVz2?}lFmzyZ# z^b=G>I~{Ffv6g*kH<9amJRv|ALxH8J-_xF-;l^{A zMUJY|?}C+|!8H2+5ly3-RTMBfd%%#%y zLGpo@W-Eok8J9~7Rt=IB5;XV##ub?5c;5>=(>RJiYFJ|AJr=hxtx^Hjfhu6%rM{1Q zj|D92twnKkO>Qdb*Rjysm&XwS8h?qf>mV%aO2=h z*Ry|@tLJ#E#hjy2V*_7ib_jjB2!VTh)1n~LOu8(BzWlN7Q9Ef${URGAOI|3e9U{w^ zd;m9A&+1ekR-rw1_l4eh=a}ef1zqmEx0Ba!NXZLsk$NDC=`QxAb_bYvW=>b zd-06+bzRE(}VI}igU48-GiycHS z1^pVJ8szx>Yse9ovqF(WmAvs+dkB-(IZ)fRI}kDQ;c0NBb|&|N-<@zV(1GgryU)8T zY(naPYyE@D?cN_cN^jQ7cArml+`$Y-vXUxk_6-6eifzT z0v80Q3b!}b!X3?b4#FP|GJWC=;HY`XBByxNzF*_Yk|;RQX}r~6klvQcg|@h(>|mtS zPOO>pRBJ64TFNU#h4L7Daq7x0N<$i5J>SG0ds)H8<3mOI&*3jJY%5a<z9pdPEQ&={>ekuXS2S40Cv)gUsgS;p^h6(lG&hvuhp~4Lz47_ zGTo}ATjYiUMgFax*K1|yMxYnFWrp5rXWDoiUFb z8}&pHjWZ1T&$uQp`(SM3*4^gNO#$5$pNy*D%90(L!@VstJBx6RiNypkbsQQ3dVov3 z6>q`vnLm>n=gY~jwUak#cBgAbTJ(T=#iI-Xrilu%jVC1?^Up9%So6y;q~*X2;;OXu zqSLJfusirtLxfV}29>1(6+!xuK`;j`u=IUB92bjY!Rt`F6=ubKCK+4;*8O8RVRO00 zDwuV(d#_5c5iP1au{jqVagCE|Wp<4#c^DQDkjT6&{@D1V zaam6aFPETMxe7`hfiKD}T&o|-c&T#$aUJW*=QIEcU5J)M!8Ff@HTP*W>5Z>T; z5{>LXBR@xLo&w}>DQT+c@F`R4kuSy&MtBc6GUfAL3kNX1PH@0z zzPa{&P>FcM^jJ0+t9s&W``gJ#NMj+i7v32k!%md1FeQdbEAXv6SO^V0nDj zi@Fco^z~)r@l(=9trGnq~{l-Pz8^Sh|#vVqBgG7YU-cTX*N3t9OJq}e*Ja$V$JJD;Hhm?pnr>NcmIfiKn+!@NL#K#8Q zB`xW9K=>SlUReZK)MW@ebj&3P7`J^}x?a2(S&S zBmp`q#*}P%-`Y#YlI1{5K?vv=%g$5}{i#3(cz3YXUDq?sX7~WFi_vi%&r_A5wSzL6 z861h$dgx8U22NIrHuxn+{73E)3J^QpgJo|hGKBqxEEN&MDCr3fVxxFp^+$rctgxIa zvc07XKE4A9JMgTAXbIA3+>1!YgdJ-{GD9%;j)hZ3sG)sEwdCB@oYwyS~cP__uk(_U}BTfY{ecEctB-Abn6t z$TWi)|xodPkt%OgHuc6#J@BVhR{ z*1M8Wp_%fBkkXg8l+mGmPrxnK9)pXzU4y%4e{caZO+-5Q#g<0GoTW$+R{3^4(Cuzv zvOft0X8|cxIvN4vQ~oMI^)2(wbuC~qLg4d8>cIH8<8;~x zJwaC!h?wQ_ENQ%Pold6xr-_QkdE1X|BC(z;65(Z^CBBlqy;X}(tf+DE!<7=#(F*(3I8l%;gtc?y9-V>?f za76@1T}IFlMevI#klD->1(9EmDr))01BTBz$XM)R!v1P9v|dt1!_Ei3noF9}0=DK; z%wW%NkOZC)VFx?x5*;kjCt6DsWYu4oI6=|}MTvu}>|#MSTh-rV)WH8E8rx%|~#!>NtSkobt1Y8$(CSzRw zPdaK@DzR~mDEj0g|lTy+DA)GT)Twmu>^uQ@B%X6M^OQb5WshM|k7cAU7|db57; ziNrxD8_yfc4|A>w-v1Gn@f9-=@W45@jpILk;FSIg)CbR4l;vl1@$WzRp0@x8_c+@I zvWs@i(k>9dO$5{ZA3fLz!QQ7!=oAqGS?3>QKL{KimHzMPUoTZyfz}f%8o*P9A#Bc` z0b61ZY{s!m2|uc-2$$-o0@YQQ&HqVLDJQV`)xis|k(~@eZPVOJm^0a7K+VGRz6gC` z7jV6{IH+Sb{edG6GN@;z;7gj4s=hpzSDRfVz2PuM2i(Ec0gj-0K;6b__8ZUJAAO2b zLS^uRA9jTH>FB(mGFK5Yh}1eMdzYAf>R>7KgZZWy>lZ%f+)fscerg3>+}c`c#Ye^C zM05m=-NO~=MiJQm5#+%Wbra<|n@S=BpIZd?{q>3I`IA2L5yddP!?Vki7z@wsWCe=f0Pgs^zpBl(QILVu}<{GSOP<`?b<`BYDD|Px| zBEjh0W)ObAjy0yp=*aaqNOr7}8*n4soIBye_3X8C`+G%Jj6wH)HelZ6n2FvFy;ddg z4H0*-_NOtC{^)m#$1M}U&12{3IJeI05qrmbAS2aHv(Y`!=YL;JeL3`>DjlWA>oRf7 zO@pmFefM3Xc%040O3aPHbNW^_UuoFqd#z+3A(bTW8Uj90MfArdzvv5p?qq`Jsppal z`=b0CG!n;%sI1CM2@_fcJVE3~`9*LTmqL}CY>1dOPQ;kDkCy*6Z1k35jr~DHArtTc zPjv-kljAbEn`ipI$z(}YUmv2)L?W&wsROD9X4&!KH{ava%_bC1zVt5X2UPLeUid?RSdA%1R0X?vE>}=Ql2m0kLc()d}pd1fp9H+%cL1r~sT(gi1 z&g~IyNqy*#epdhWucYZ61+JAl`-UD+45;36bQwH{xYS63&9ucBa_1W~zk!f=!V!HP z-j$s1xX4)Iw)AM|e<#{Z2a|Ui-Ra8e%Q;)cz9!$h2SO(VLbqNyS1W*Izvn9++>?sb z6UMXis2v}6F%<1$gooDs*>W`=s-|`a#MCcq%er*?ai| z;O=-kP4XXKmc^IBLE6gF${=1|KkldOJqH?%JyXQ(HAx;*axSfFkU%2sYk{StQIsuV zec``ZwkVGk4_m8Cl=;gEo!dH~`QGz_R~R+OR~OmKa1YpU;Hkhs%uN+=^>e@C>Yd#%N@G zo`H0&Y&mcDeD$(A{k>gH-liN9WW&n4c(V!o$48l47~tp#bcY5c*LuRdNym-Izsm;+ zxcFbMTDaUGXH0v|5wkY3A~oasl}yTCd5EGbu4$a%ffHeNMWg>>Pasw6;V5VLJHTG)Y z$TYj!2AJ=NGa4RXyH`(+8raSI)TU6Y*8|F~fYO@z87FH4P1PW5R@HhX zWz)b|^zH}C^=3FUKPa7%=CL9RgE96wXl1NvTlAilv*fFA#+9|fammvKn(x!MBmxMH+$gW z4}>hmbj==F0p65X_x-neyVqPyc1WUdn|$q^VkbzJK~H6#^b{kYk(;?Pq7HjU5O~BW zUR7apsD>%(uGtgb776hrTQR13_XBdh<-aZ9mSbNs*4u;PEp|%{xTZ*VbtrjG`6Xs$ zre<%(&rW6qbvX;!SfT~jGXBr~!}xsBA)=|zq2!Vn&Gp151^Sa5KoglW(z4-JoDi7J zN5Ryt60vLw!zt+pXoB!Hl8VvGk(9+mDo8J1vp*-8x5m(Vb{7cWjjuAPTKy9lz*L)- z;JKd&vVO&X?)NV_p-elr`iHC~jEm}~t=F9Po&Ol`I2_%c;AcpjvnaWVjXXZJcu2$a zdUzW7dwVXj>>>6(NXT?YzRv=CCRuz$u1nG?di9Lpo~1!k3#vx(DgyR&xln!L2?rCb z!CQqI(79M+lIb*}$6Z5r>-tIZpprD5B&t>sQ$iKcCk4@dz3`E?Q!WAdQ?1Wt(3P4! z{4KrIz?rj>AG6n9=lqcp4ZTk#Ne%?PWRxWYx!Yxig%(8mP=l;SUKYim7XRXx6ztm9 z9sD1Up)`W^g@W*xe1ATj<0|XWktfbUFw>cD>W!mHNoG?Oq#V6!!el=OW)WY1UD%MY zukIHqOKC90f}~aB#yfCGW28?m=R0mtx|<>BU4_;b-W~W0bXaQ`7n`Q<(6tZhMQ!vj zBB}9Ch5$W5z}T}dwc1EJQ;@oCS+P4V0e5YR?tCn41C1x*(&hLm0tj-J3tJr{+ zSM3P&!}9(!5swvRHx;eEn&JEkINz-c21M_kLshD&??RE3nE_q4Jx&c*}@ zfIp1R4F?)Fu64iBD^jQQg*6iBixm(r*R(JqC;!|g&XmS+ zG$T>xml7Kxcl8Gt<{i*nrdyR5oM^r!^pWlP>V-RtL^fZ-5lIdhI5Z&P_^d5Py`#vY zCfvabH==p$JZZA*>D$#zhhy{50gGau6hSSZ%l%z;Nna>~N5gN75&zl0ZT#@4!H5uu zsq^)u-ua@S)b0598-^m#GlyhTLv}A9R(=$sG87`}x2cAcw;Q0j8tZ#RaV(UES;X=qHI_79K4VmM!)9)J2k zm*`Ez9Zj31Rz6wh!cLJB7tZxEQ&8iI72QegbUC)!Q65X%C$ckh%znyh!;9ZPOU#4s zFV|nPtn5I?4SdLf*+O-Y_m>?Ma9-7K6P}wVdR*PowKIcevIc@xi%JoHzs+E?+(aS z?2F4+=)#?IYDsh8WN*0i&uWUvBHr_{4xwttF7b|-2(pQSdqpaxe3sL)3#74|bA}Mx zPhg38{@4w{ZBOg&fOp@8VtI8=s^toowyf_TVn&t|?Yx{RVq)p-2ngb8M;eex1q0nb z6x|4ECYn3CFx_vazex>XJ60}KX@K#SkUv4f5Xi{6!C=y-&{LTOE8N|0<@lRuNVyM0 zV9vK!sJdh+RmEgHM@R>>$c}TV{lF{sFiD04HxH)^cGq;+X;+-u$u#BW`H-K}eL=uF zfo$7h86I3X(DGAhu$tJkPh&Ny*=DSm8&3<8(oG058r_totj(BW`{dxp2K77hPc56j{uI%cps+H;; zDz0E6_kL5%y2xsT_oU2k#ES^SYZ(Hiw=rxqLgoA$)0{c^&a5|E<>hH}7|X+tJMHB$ z^a{6k-8@xcrd9U~o@dA;OGUL}R~wQyGe-me%`o&n$)S7jsX+X2)}T+g)#cIHev!tH zyJZ{4J{m7+>b2to7moA9-YJW?UTR(#R%e_$F4a`t>6Hk5_nrs;5zg#2@3=iid^~{q z{5ksj!;kzUPa9?%jOg_o)9(;$ZuEE0HFs3O54|#{F>bQ^>AWAu1Y#!pe}&8W6$*@F zKJ}r%1EMeAJ3#7wmTE7Ys-MbJgzBAtj_n7pS(6hMkIYz6OQ;Q@;y!%h9wkUod#r^f1y2_#CGJN8Yhda!IB zB*E!~*3(s2QxpSAH2_P)Tur82>B3G->rx{y;~mb5W7xL-{AykJG&UM75*V+MFFM@% z@uR~US;`I+es+otE*49=TYPRgjaaKV>^818>-C*~wU2C6kRHz}^Yy>FXx$er`g2l; z(O0MMndbKmwrrF;!o`P?i|F&lG!}0tuGrN7{0;W3*IQOnup*p&kTZMo&(9)+MzBXa z=MWo31~zH9crx>0b>xcYt$i<&Vf`Imqx(W3S|*IeC~e2E5dBO6&tD2OmteaApvxAt zODFf5O$rE2ucJYo^aPK+kzBbq^Q&UEQ`ikXF@ zbmMvuTJs3&@+7DNLZ($$alEd3?oVnt2>v$&P>3a-S1q9UVJ#hjZsIM1;|=gk zpP-27v%cCh@rB?vgu4ZRc#{JZMq5_V`#cO_d#bP6+#0_Aoe7n)w|wfvnk7IC{5}8r6q#JPNom< zUZOCy;?E!N?r|CwMK1=y<=cj-f-mXz0S0z{N^#2WWz5d_-`}=*ei*s*)F=C*LNr(^4E>D;IKW@^oN>6kF-T|Bi); zl8Sqr_gGI}MT`YCU^CL@Z+hJz*Js5$ZX+1*;?3&`JKA2|mha5i6foJx)$DVx!e#>p zPXA5oFVS4-bq;9ni*{s7_@2SG14Xt~zPVGQR8OU%FP)7Ai*8ehwT;7GSz&gcHi^l0 z+ETu5yQG|mw4DuyY`3+~R}4?KQeAdLiWONxH)i9FcXxll-x`eJW`g!{iPD!~aT;_K zbFo}VakcKtNbwzGIy&@k@6h)`GBMoi{sgvZbk`i!ni@0efY}?6#EnXSk1~}U@xwwQ z(f&;XN^J=|sc2+V>!{_ioZwKfRcAOP;%NJ7=}>p0EZe`F)yF!qsA=Y2Bg?8#WoK0> zt5BGi>d|iS5zGB`Nrsl~mgW2z^yS9ObF-(^)a_NH-{R66=Cu*sYZZ}^`n`*X>!OiS zNKsbLh`2IXSSa$$jPlR9jvHTU)6kFCku_T~9V3w;YRLI5gkl>Ceq1}sQFxxH)v-Od zjc#3iwmSUmq6w;{As(z@``zZM4Hr}nfyg5hoY)Y$LoOGV#A`IjpQveMv0{%n)HrR- zR32%M-Vh3>(%2aolwD&SJ_&&B@A7*#eVs9B!&s`)we2m$dwYdNIxJk&{DnnYHMzB^ zok6VUPwI}XZA$2JX8SMAuI755z-!)AK+aD%9@kE&o#0y~XmmWy%jWANG2uO7}*#TSe8)q~FN%r7pGUdF{0h zZHnq3Vd4~{__jy|)b8O(ZKT{XW5ZVaQX>cbl#rNLyykAaKg*TaZK3|Ku&jf+71UXA zA0{Zvj60#{5>C>C=xHuzK4daFuMWfcO#5cK)FO{mdZNT1+eXT>nN-caA##WhT%R6j z#q)94n9(oB--?==+i`e-s1v=d54C6tuX=hb;#bL13{LCSim0BGLU#OmH%i;fJ7Pu$ zsS@Zt6vg7$98R9&wGs&i4s(A1kG3WY(O*}ha?@3MdaxUp`iThco6s-lxWs;!;F(qV zZS0J*d>bnUQVhTU;p1Ge_0_n(b>}x?%C=Ft^6afZ#PRud;G@$GsmKoQsI{3`| zsW*4i(Bb14n0T9akt~=-|{O-QvS$7T^ZJf zLB8I2dE9uO{5EXSLpE#i4s&*K+_4$FsA)TlP5lA!efK&apW{308BI6waW#E1VqQx* z78tq3o(*YI@xdFAscH3Jftj6GjrO$OVoM|V^{Y-R2zieZ0r>^ghU-TFyYX>mK=uZs zZPW49Z7jy$hWH}x$!U2I7Pq__e3uxnGtutw4Bao~6#b}T0V>)pOSJ18Ga;9C7RZjn z&}88s1x7WU8L@yZzff9p0MU-eau7r7(sBD>IjbRwLZRf{!3>Fku)F7nbyEJ*aT3^K zBwNEAbTl+R@W`nl;o#n}e-$dflCLgHZlT`hy&TzjOOsUOGKwGjs3%4V|+$ z!2EtDkuxBoa?5vDkiZKrcHsG#`+XoH?QZEnc+GOJZz=DDqII_Otn-!JtH7n0u(5El zXsH0!la!I5<+Hm*?!lNEhD=*qy@p#z!<+s*Wa*shx9XazXfH3{_v4J8h^xCvubxMy z7(Tz5gFyVnG$1`@eqRNX6V%@OJPj${#bUxcqrnbU0Va zPoeXpnq}yjJ@QelTheX@cxxg1fyWvBm7}R+>!HNE_6o^EE!pLCxLC~i3u}2&V_)GW z@Y*mzTlGL7V~YC?E4@|x$(&Gt=s?M%3i&u0-4@7FcxKC-T9T!#Y?-*Pf@tUE=R2=; zNM99k2}4T|QDTJSnBUKr_L_r+qan`?ey?ICbycDJFHuqik!mRgZnFY*>m5r%$QX91 zwI+fZpG{pDr@lWOQ4H zXD?S?`hPPcE|QirO-Twm8g{~U)Nw>F2gjpZyb>QlWyfE97yzf=EJDqnJZuS=iYJg3 z^Mbz!MC>imNyR)KGTLPjdfv3>=m?vy^s3_z2x2x|D4?pBFO;Bi%D``8?L>-CW>6hT z9KP5-%nf~bzXXXqaTa^!<7DyY8Qf~wr=g`R)5U>BM(BCnQmc@}l*wx}hDe)*`x0jj zKQveU^Vc`Zfs8r2zT9~vwztjM@Z@veovAMpH4^;C zko_NSY_fDDIo+okCwG#h3MDAkZrJxfD%>m*y8Y8Wumv5m?auJb0REIomm2z63>_M` zzZL)5#Ub;q;mV>b1ga6f4;}d^hV&aO>CcL@HwhlfTv@2xDEP2csZ=<(0S*B>|5A{c zy+|`=#tU6@Lsb^fso@e>I;Ylm zYwI2`?oO6}Y^fTX_%hM+m6ys~aAK-$qOMh4KZqofB^#FS@73=y-+18>yj}*MeBBwv zIwaAEvqZH9={zXeMwI`l4Uv5n^h)poJNW{gy-&NaKPdnvPGiq1BeTtE6%fmc9+?nS zo|Yza$g%vOfD4dhuTPb8+2C$<8HBs}_j&PNgW1k$ouFyeKv}j3TNma^s^#*R(p78a zot#f??$1#&0d*oGsO7%bu5Y~b+!uDX?|e^axEzniqZpvKezO|?-g5PA z6U#>)o67Yd#*UF0Z1RzqW1wv!WAN0f6LIHo1OGoGcv}W>(3#|mBMuaneq)6PbvR4e zA2=7^G~<4Cs}#KuIx9eMyp7$*VGXr{THQ}WSDqaUicMKh!+u`O)QMDG%^4GS`pi#& zaWa<1Ta2>k_mvX4B~5Pwk1hjSwzG?5Y=^NpCnQ^66wWxlz9d(sTzcDl#pk?ckKO?4 zz2!0jkP3q!&@z4e4Hnp@JrB2gQQJJ>gObt1q3n>YoV9wC?F0RXtshF-0n3 z`SW+3v%F7bhm%^FaQ}=aJI@fZg0$!n39;AAAj0@*bSDSy>vziN@@S8*3S zotO!x_MnawBHE`BG?kf~uuh#8@h* zep;zKYVo*rgxM&{y|P}P>?QdJQj|c>i33SCpKH}*d5qHa?+s&nYb4SZoUI;J4Txyk zKHq>04&d*tdFQd(&a6&M_A?K!+WQ8&TTFhaZB{rh&MA-Tw}cyRytIy}^j<6%Ej$)S zBzCNu;2nu$%GEO(A5-}_EQ?lUC#w8(fi_>>-u3+cRct55f18ml))`rc9Q}q>F(`1x zW+M(G3Or{d1#`#ygNd?+(!JSbZTK; zw82fQ^{ekt*Otkrg&F-yiwi;QE#wboNIMSdOVjnBl>>hdg_m-@Nem8IPDSrWNp>}! z1+L2{9!fGQ6H8?j!7MX9j2%*%$FUW$6KU$0yxOurYKYjMun6-+Z6V9j0iWQz%%yi|6oxCI(+4X0-1PPXy~?J{QmQe{R9&1`&b4WN5peyMevu2aB?-~gd&VSV!D zFS<1;yTzyT!Nrm~UiN+`X%wbh4&NoMdv~=vsL?p5Z8wD}?rUu1t%<#AIg+Go^A&_G zF-*uZa#^s7yDObr*+*$RVJ~7~UEOtGt}Ff0`XD7hu-%f7&%C5O4d<8kL;W4O>$OU2`M0V0bkgN`P* z7|Cb1j1st+>Wvx8o=i+(QCwS`ndjMwp=z*5*?GSh^-8Zk8Be){NC>XfFwS)$z4yd> zX{Ml;(@@+2?Ou*|e5JWfpB`vGIqrROwyP87Dt*d8P!(13awLR&E51UP?M=+-Umc}| zF*zTPq%V00d~(fge{6kS;~|nxp<{<&BH?NXfz;~sf8z|MtyF}DZQJ`dbj^LKirGZ= zEyfD@RM}JG`0K(VK3gTW-AOlb*gMZkQAx5MsB~y)M}}=re-4F(#@!jwv}Jzd|3SWS z+0NL>8D@D&Ujnxjl#<2Eci#{SJNvL@I#eOfx0h#Ym8~I6H9oq?@gRgH#Zrz-&Q>M! z=UH%rvx_$IO>4?%8}jlG@magAu5NntiHYa9Fki9u`vVx96@|#;VfJ?)qP%$tG4)eY z)EHw{XBM?_%v3JIa(b*1b56d$edoj}NOp^P|CF?OsVRRe5^t01jU1JAM%Gh9?%Q6! zG$_8eBin6^RjoSD>^(1xf>@--o@f*!-zNoapn9K#g{RzE%zW)0V@#f(z4uq*l8x@t zyNMa!XXi!4nRTNzv5prXmI=`pw6e>BTQP*A#{HJwqXi3uw7$t(r#4FdW4m9$b)|FK z)<(6{COLH6*i3Tj43_`05m}$xbAyqe_5vwCrqVpomEU`eP_Q-8H;GZDCe@Qpos8pu z&UX|bHpq?QoMt_j6Ucy}sV3xLUB;Gz!TNyIq^Zh%f)<(bLZ712NZiRIFG9k3( z>Y-BvrgoXh$m-WN~s|VCWQob4%6o$zf{|!cQj2hiC1a=RwGYTzy-K zedZw}_dyN$a_qKX$GzZoYk5;+1}t^LNQv0t-Go0a_A&-sPf<1mbAFhE=6*Bl?01Gv zxzeqvja*1$YFx=qljmK+^vg|C4M=#=sRhr##HdokX&a2foW~!Efp$#F5R76oYx})N zH(uhVQBL+NS{eK6VHJu)5kmVIKlb8^ayp3`HAB%c{+#I+><-MaGsdC z`YpWsUUe~_!YEZGVyu)k@ClQROTm>NL}-C}yAO_bWN00g^Kr0GNO8T@ zdT{ajie7J^RYT`J!j1gSK*Ky0if=eDuW&J4rnmnK6*f|$KFtTy-RcGF0(syI4nZq@oZJo}t8w+L|x=MlK3o=Hk z-TcHy${4g23q$6XSwr~7+msQN9C|k7ea?li;}-%bnENN+MK9ENW##iL%Acctdpg!H z56fw^XfIEUS0jr*hVBbGgsN&{B;{~`v| zntm_u%!{${g;?y`mT4WYDq11%J?}eR@@)QsK-3Y;wUkp1&p4b|Tz{XpRm@yehGpH~ z-9}iJ3-xt8$^yi^{$d2)6}wGCr{Z(Z9ST=m8a#So3XMl!?J}jw8;^oOj!BXn#~^e zm~2{bU^O*JVy_386=i$9{gVr;!I=j_Zsy=T=J-jmVWlFzjf%{5Jo0as@#p0+n*-Y- zA8azBbePA&&&}e$$6nHQuE%;gN8Ks?aPCIc$#FnHGaoML>>_O9 zj_ICt6m0%W>nvllwA`szvvkT9mmLbY{axs&b9NPKd7gRt<-35j`Hk*j=!DnqCfiY zU7XW`?0%|Ev$J&mW7&(`-qbW_B>bX6glEseenNp~t!=ZJil{M)b+y&X6!!1OY?+@2 z{^mk>%j#8B>bTv2VqaTal=$UX9!;dY&rP4=)O*V~`kxD|QOI)EuU{TltC6Pmx|F)* zIDZdO{jqVAsrIw>Grx%G0D?Ks{B)ugAN!d-e0VO$jq$WiO~As{Qj;R*0&6&u@u8}t zcB9OYJDycpJTZRpF5`)R`2FaPOmT|6d+MaeeWdB0b4x__s1ZRtbF7H-JM{}wfM0jP8J0=zHXq6JxKELj!HYis$|E%E9U$hLlAjlD4@^KoXq_BKYCs3s$evb_5pGd^qFWspR_;&3S5gB04XNYk`k zm7fakVn6tg`^I)zkKxY>t%C27J*;XB1+zf7FFr5^RiRXvMW|kFDqxOfd-gQ#+ahB znP9^{C6ZH7B%$==P!3U%<48TFgF|RJF6R()Xbxi(MINH3JjB?nX5vY1_7?aa_$l=&_$ zWaabGGr6VyUaQ9Uy7VJ*kQM}@Th7A?M}6S(l~E!r3f96LX1ts9lI<|c+Fefmz)kqY z#5^I>W~}po(h%z<)PSQ0PVF+yr0eGvgBu+^73U@msAq8mWv`)!%`jksJ)u@kJCt%} z8M9st zKPV;Vbd6Hv4nJwg?Dp7>3^7m`{QcvR%QDqEY`L>W`KX~{N^lpL=Z2F?YLy+>y8zgr zn#n_cyK}GV?PYEOWe$eIB}3AE%TSBsi!ttI{&CMmvJyiP2Nz-^Ct3iO2b7n{%gc1} z)V7Y7M@caZRo-bm_{|XfAUIZboE6{uEf~P*9N00ACV2h!7A9mr<~Z}8GuL_p;bGn@ z7A?rn+g1;!7hc~9kxz<%qao9w{+26soY^ye5q{PL7C6H0!}PWh0IY}{vGYlv-OsDH zgjc>uzaDz6C?n|@BVRJflURAuoDk_o`qAjdWB2GUq&{O_?=;mB0|akoIVmciDd5Rc zQymfj_Be1ntT_Fr`DuqvS~z*_d@_L#BPuii9KYHSJ3?Z1BAqyyetB>*(8S<~WK?E; z=l9spX2pcrx9ppk@AN>;yYydq+~8Xa%^04ULM=1?hSnGGvlKvMUGjxuQSgL4W84fh zC>45-r?RG;${$fC?wT~_F=K6DukMkvb0`1W>3bXY`WJLWN~opU8nUVyPR%vSAWQ8p z1Nuv?Rj_AWo4!#&w;XXQEgbm<1sWQI9Lu|ODd{j3mlRSw#z0Fd{#OFTCH2>BefWJM zv7u0f7vu8w`vwL{XvC6FJH`e4>Flj*rPd=^66SAqb5okb*<$F?OC5ZsqkzQJMC-vq zy#vVxr>~2$I~94+26 zDxNSyyQu!Ar+fOPwtzw5^+d<$=@0sSvl6F;mS%6MM|gI9s$D;| ze8=>JK4x?UTxxsR(v>ZC^-y0bT zFC|SB_833-@sfM8nJ}wWJ~g_7#gL{3q|o$`GB;yM=mmR~-ua<=+w@!p~#%`DKH9(l19X^!&uBU0r4V>z* zt=_1)$k$!|&T?+Ec3Iadj|iNAB~xlaC9n}A#1vDdP-JIllo+Ov-t=TJh$#3wi95M? z&y@2gyyXtklaLp-w~QoB6I6X0>W|&8mpb&CwQpLx^bsWUVaf3BNP_JvN=9}*BaDEO zr9GP$Ur&EB)vm_6;Vl6z_tK{d{#O5I!=`*+4E>OYf@RS`_wKbk-u9$JRnrIqMLp+E z{^yUe+*OZXytdHGKh_yQASGw@i*1{_PF?zfW}5Bcs8PdzmG}tmCF;}*diDAKCDXuG z_dGbp{-Fk-mj?ko1{C=Hi_NaQOitoCGMPAl3bX4=mOAR7wzcpLwWlIKyn zdBKQK0nBnVQ>*wzO@7@O+?pa>=f7dki?hEL4E1_E&?rOrA`gEXJCZl{=hJt}h}{SQ zQ}j_Ar;N?egt#fwm@4$79Xb~D1MDU9^rXaas&Elp^r%WNy}jRn{U`pvYpWSM;uE|o zbj#-Gy&jm65Y#k17cYBlr2SG;DQy8r!Oect9x~4#&@T__*^SR;gcYq=ig4#i+x-w* z-$Z0lNl!VPhS4V`EntmrdlsZ1UV@gfooGQwv~pzA$W06P)79C zKg!+`a}0Z%#4)L3Rfn6jGdf|NmbO5j_Ln9cOSBTQ&)VZ{7>*$OXT906i$ZUXJya~w zm6ng~{sTegrPfJBqJM6&WU>(1)Gty zmX-6ql<>n+1_dE%(OEo8GJPCiq?F2V2^adiD>E!NvRN0;3bmH9#Gp{RnRHNK{l86~ zCH0bB)t2l1avM_;9`fyPx=bgyTLB+&xPQx%gHI9#;~WjQ`HO$&qYW-cv73jRj_E24-?9wnPw`9g(7Kff|Nrin_FWH|^m9B1OIc&gbrc(@eGg0CytLyLg?3m((c z)J`@6BG7fWn{Z$=_F;^q8|M1`&6=dTUr(!E4CpsIN0??WKkUj(S{)#~_YBNZ>Smxy z-&xX|d3a-|oAlD1dW`u(lnG8hxWqNsG6o7m`axg*wT*n=ibmwxZO3OK=bGrof=z16 zB}9F%*tM@gzyTNf_#e40hH-pM&1ZoM3V)j~e5iPAFm|$K=XibJiKj3fU<)wM-kOl& zL#rcbsDUD?aHH5AN6(^gV8D_Lw=jtbWkP=yu6>AL@yju+A$y$oNPJN(sjh#M$;URI zX|@USz1s_0z-bz%+({)AdU3gC!X(#h^&O%SE9C^lm9%VpyVbENRM;Hmlv(;8MBfC7?n?Zn6%OU=Zg6Nf?Nh_HkbxpO&dAdd_jqh1t*z-BfH4j=zA4yc- zJa~mhG(^6iq)(BolM6gDV?UGB^8h4jg*qj++LSM9H&nQ(5zrjgE_-vDyw<_-+MY5C znxgs7I3p(W3A39P&Mmuy*^W8vqtOC))}j6+@>y!zL!rDcH~zu&)423Uli#_9^G{B? zsnLJ4)X*;uRI8EK2nu-W!RYA*b<3(yoy9d6{-yXY4+ciPUMb5PkoTs=n1bCT%{cat|2UH|byPa%m8Bz7D#K~n! zqHt*P%V9yXZ(+c>;N!5b(URN_H1Awz$ipAcD$g8NCI1H;=t1Gt7&(|rYw-lbp!VV1 zy7Zb52u&lsbr-!W$Q=H2C7B`?14&!#dS_zUMW0wWlaTWF@e7zfcF<77Rh8wQXfdeH z<7v0CRif@nDe+6>@VCLsSEaiNitO=ZB7;%WFP%Wy%l@Hlo~_I*5e)0ux5DwWN=ow{ zn&H48WZ}4S*GklI8Rzg7sb$;6%5x$Q4ID;BU8-P(#WLDBGg|qLJ5k8OT+g6PQ`~yl zx6wVAa0_`{plzxFLoiG`d~O4by{kq#f4cpBVVyP2on}Qb+JjTH1xG6P_1S>3Ip##$ zwWDdUvE0t9y^w&v?Lxf68Ul)grqFQ*j1(uc6~u46wC{CVbMRx!aO-92jx^_-SO3y( zkTq0(!XgK!{(P-15yc@;OAx$j(*V^%cDqsaeA;O;eakkl>is06?z^cVfnNG8F+W-@ z%c&U}5EqGz@c59Nj99^cCRFu&2*0Qk9&4pk@2S@g(rb6Q6(KW2-@ox$ zoMAG`Yon<%-NpD{V#gdx)jl1OcJ@Dr7drR}BGxYC7r_V`DkTZ$9t?|e`!yF`%YePt z!=-L`2jXi;N2r)&{wtW&$W1gUn{WN;D0PYcHqGB1UcgW8#eGybzx{A6hvh^S(t;#M za9%ok^>uGpDt<(XVYn_&ceK@^0Jn3dT714M9cG3zjg=|r8n{nku|9fbcvN^Su)1Ro zh`5^@leVxWHIzAGN__du@l#u%YTv-P7&TNXx1Qk;RAsueAHlerNw!;`L$Fr69)J6b zG>GGse8>8J9NRbM(|bBNze1{b4X=Q&0c6VxHrfr6+g_zDOx3^6A2m9Fu$L5VCUN=v zBXXA9uFd*(t|Z~7sp^I1DF7;tfd5&g39 z!QwwA_lpA60yXR#jF9i_DBatzn{rl0p2-GE)7J{q0|LXVwoJ01H>{9T09{K=t#3qZ$TCsOiNBe7!&YUxTC>f~Y?kBFS2V|dt+o-; zz;S82s29B7D)ERV^Jv*PNxe=5H87Pkz>3<&+%b)9Nh-e$(H!MS15TzCRu+-5bS(Y> zl;H-sn)c-Q+%0w?Md__hy(6`6vV*fW@j>eFM8pJMAXE1e?x%wKIx6vUP zJ^9u*&pA^S3dc|D#u!)qhW%y+ecaani-!UQccRLU@peUT9D#Dskj0Ogm|ijH>uLYRTXNj$e9es|DBS*D3G6a?;9#1BlUobq&;)VntZ6D&%RTFAL?+gijNMz z9lI{JxD{VxdsTYnlTxriNrx!qdndW9Xn0N%390f0M6Lwiq8 z6-qs#$d`zOvNIZ!X;-V7z9BsC8#L`K0dT9qa$00MOk%wNb>Lt}++$SxM6|}{JfEka zm$0+7tv{E-PUhIx+t5C1Fb5}xug-5xA;F1L@<0-q^UlCG0M(PS{%5J zcPNo43G}A?$Q-=nuGB+^Dwa^=BQ2$&qM`>5HhCP$@%ZTUCjP?e%}}_GhlbMitOy4- z@5>%xuHHNf8Q@IkBzhZR43nkP7jDk0ey3NWh+-oa=-(*UCr1c(ibGM1dS2R1&6W3WxTQ?>=C(=WwRJ;Q>U?4L503UlcB^nLt+HYGR zQt7y>v#f3xuf&Z=-vyk1?}8*F4Jh4x9ci#=EsuTQG+Qqqym=Mxc#yw0eXMz6 z!~TFX&*iCUK`;=5$TUR_7=OmnvH?60m*b(kZa0Rc%ad;5DD-X14RoexPS>y^hW1jPbo)sS0(ci>S8dh>EJtR=lUB6XGJVI4PN1 z4AIoby!wGGT5-U8u@B9HQGLZ*G2YtthjaAgSzD8)0e{WaQJ}X_a;iYkZ?5%6 zQLm=mnpnSB2esg|95m9lwGNpj}d40go;=Hx7y*V L;}y*1`)U6JAn+wZ literal 0 HcmV?d00001 diff --git a/examples/others/plot_lowrank_sinkhorn.py b/examples/others/plot_lowrank_sinkhorn.py new file mode 100644 index 000000000..d2e4a6513 --- /dev/null +++ b/examples/others/plot_lowrank_sinkhorn.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +""" +======================================== +Low rank Sinkhorn +======================================== + +This example illustrates the computation of Low Rank Sinkhorn [26]. + +[65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). +"Low-rank Sinkhorn factorization". In International Conference on Machine Learning. +""" + +# Author: Laurène David +# +# License: MIT License + +import numpy as np +import matplotlib.pylab as pl +import ot.plot +from ot.datasets import make_1D_gauss as gauss + +############################################################################## +# Generate data +# ------------- + +#%% parameters + +n = 100 +m = 120 + +# Gaussian distribution +a = gauss(n, m=int(n / 3), s=25 / np.sqrt(2)) + 1.5 * gauss(n, m=int(5 * n / 6), s=15 / np.sqrt(2)) +a = a / np.sum(a) + +b = 2 * gauss(m, m=int(m / 5), s=30 / np.sqrt(2)) + gauss(m, m=int(m / 2), s=35 / np.sqrt(2)) +b = b / np.sum(b) + +# Source and target distribution +X = np.arange(n).reshape(-1, 1) +Y = np.arange(m).reshape(-1, 1) + + +############################################################################## +# Solve Low rank sinkhorn +# ------------ + +#%% +# Solve low rank sinkhorn +Q, R, g, log = ot.lowrank_sinkhorn(X, Y, a, b, rank=10, init="random", gamma_init="rescale", rescale_cost=True, warn=False, log=True) +P = log["lazy_plan"][:] + +ot.plot.plot1D_mat(a, b, P, 'OT matrix Low rank') + + +############################################################################## +# Sinkhorn vs Low Rank Sinkhorn +# ----------------------- +# Compare Sinkhorn and Low rank sinkhorn with different regularizations and ranks. + +#%% Sinkhorn + +# Compute cost matrix for sinkhorn OT +M = ot.dist(X, Y) +M = M / np.max(M) + +# Solve sinkhorn with different regularizations using ot.solve +list_reg = [0.05, 0.005, 0.001] +list_P_Sin = [] + +for reg in list_reg: + P = ot.solve(M, a, b, reg=reg, max_iter=2000, tol=1e-8).plan + list_P_Sin.append(P) + +#%% Low rank sinkhorn + +# Solve low rank sinkhorn with different ranks using ot.solve_sample +list_rank = [3, 10, 50] +list_P_LR = [] + +for rank in list_rank: + P = ot.solve_sample(X, Y, a, b, method='lowrank', rank=rank).plan + P = P[:] + list_P_LR.append(P) + + +#%% + +# Plot sinkhorn vs low rank sinkhorn +pl.figure(7, figsize=(12, 6)) + +pl.subplot(2, 3, 1) +pl.imshow(list_P_Sin[0], interpolation='nearest') +pl.axis('off') +pl.title('Sinkhorn (reg=0.05)') + +pl.subplot(2, 3, 2) +pl.imshow(list_P_Sin[1], interpolation='nearest') +pl.axis('off') +pl.title('Sinkhorn (reg=0.005)') + +pl.subplot(2, 3, 3) +pl.imshow(list_P_Sin[2], interpolation='nearest') +pl.axis('off') +pl.title('Sinkhorn (reg=0.001)') + +pl.subplot(2, 3, 4) +pl.imshow(list_P_LR[0], interpolation='nearest') +pl.axis('off') +pl.title('Low rank (rank=3)') + +pl.subplot(2, 3, 5) +pl.imshow(list_P_LR[1], interpolation='nearest') +pl.axis('off') +pl.title('Low rank (rank=10)') + +pl.subplot(2, 3, 6) +pl.imshow(list_P_LR[2], interpolation='nearest') +pl.axis('off') +pl.title('Low rank (rank=50)') + +pl.tight_layout() diff --git a/ot/lowrank.py b/ot/lowrank.py index d87fd913f..c59bfc092 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -19,6 +19,7 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=None, nx=None): """ Implementation of different initialization strategies for the low rank sinkhorn solver (Q ,R, g). + This function is specific to lowrank_sinkhorn. Parameters ---------- @@ -30,9 +31,9 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No samples weights in the source domain b : array-like, shape (n_samples_b,) samples weights in the target domain - rank : int, optional. Default is None. (>0) + rank : int Nonnegative rank of the OT plan. - init : str, default is 'kmeans' + init : str Initialization strategy for Q, R and g. 'random', 'trivial' or 'kmeans' reg_init : float, optional. Default is None. (>0) Regularization term for a 'kmeans' init. If None, 1 is considered. @@ -51,6 +52,12 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No g : array-like, shape (r, ) Init for the weight vector of the low-rank decomposition of the OT plan (g) + + References + ----------- + .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). + "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. + """ if nx is None: @@ -59,6 +66,9 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No if reg_init is None: reg_init = 0.1 + if random_state is None: + random_state = 49 + ns = X_s.shape[0] nt = X_t.shape[0] r = rank @@ -130,10 +140,10 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No ################################################################################## -def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): +def compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx=None): """ Compute the low rank decomposition of a squared euclidean distance matrix. - This function won't work for any other distance metric. + This function won't work for other distance metrics. See "Section 3.5, proposition 1" @@ -143,6 +153,8 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): samples in the source domain X_t : array-like, shape (n_samples_b, dim) samples in the target domain + rescale_cost : bool + Rescale the low rank factorization of the sqeuclidean cost matrix nx : default None POT backend @@ -156,9 +168,9 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): References - ---------- + ----------- .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). - "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. + "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. """ if nx is None: @@ -177,6 +189,10 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): array2 = nx.reshape(nx.sum(X_t**2, 1), (-1, 1)) M2 = nx.concatenate((array1, array2, X_t), axis=1) + if rescale_cost is True: + M1 = M1 / nx.sqrt(nx.max(M1)) + M2 = M2 / nx.sqrt(nx.max(M2)) + return M1, M2 @@ -222,7 +238,7 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N References ---------- .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). - "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. + "Low-rank Sinkhorn Factorization". In International Conference on Machine Learning. """ @@ -282,7 +298,7 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N else: if warn: warnings.warn( - "Sinkhorn did not converge. You might want to " + "Dykstra did not converge. You might want to " "increase the number of iterations `numItermax` " ) @@ -293,9 +309,9 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N return Q, R, g -def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, - init="kmeans", reg_init=None, seed_init=None, - numItermax=1000, stopThr=1e-9, warn=True, log=False): +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, rescale_cost=True, + init=None, reg_init=None, seed_init=None, gamma_init=None, + numItermax=2000, stopThr=1e-7, warn=True, log=False): r""" Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. @@ -331,16 +347,20 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. alpha : int, optional. Default is None. (>0 and <1/r) Lower bound for the weight vector g. If None, 1e-10 is considered - init : str, optional. Default is 'kmeans' - Initialization strategy for Q, R, g. 'random', 'trivial' or 'kmeans' + rescale_cost : bool, optional. Default is False + Rescale the low rank factorization of the sqeuclidean cost matrix + init : str, optional. Default is None. If None, "random" is considered. + Initialization strategy for the low rank couplings. 'random', 'trivial' or 'kmeans' reg_init : float, optional. Default is None. (>0) Regularization term for a 'kmeans' init. If None, 1 is considered. seed_init : int, optional. Default is None. (>0) Random state for a 'random' or 'kmeans' init strategy. - numItermax : int, optional - Max number of iterations - stopThr : float, optional - Stop threshold on error (>0) + gamma_init : str, optional. If None, 'rescale' is considered + Initialization strategy for gamma. 'rescale', or 'theory' + numItermax : int, optional. Default is 2000. + Max number of iterations for the Dykstra algorithm + stopThr : float, optional. Default is 1e-7. + Stop threshold on error (>0) in Dykstra warn : bool, optional if True, raises a warning if the algorithm doesn't convergence. log : bool, optional @@ -349,25 +369,20 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, Returns --------- - lazy_plan : LazyTensor() - OT plan in a LazyTensor object of shape (shape_plan) - See :any:`LazyTensor` for more information. - value : float - Optimal value of the optimization problem - value_linear : float - Linear OT loss with the optimal OT Q : array-like, shape (n_samples_a, r) First low-rank matrix decomposition of the OT plan R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) - Weight vector for the low-rank decomposition of the OT plan + Weight vector for the low-rank decomposition of the OT + log : dict (lazy_plan, value and value_linear) + log dictionary return only if log==True in parameters References ---------- - .. [65] Scetbon, M., Cuturi, M., & Peyré, G (2021). - "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. + .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). + "Low-rank Sinkhorn Factorization". In International Conference on Machine Learning. """ @@ -385,55 +400,74 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, r = rank if rank is None: r = min(ns, nt) + else: + r = min(ns, nt, rank) + if r <= 0: + raise ValueError("The rank parameter cannot have a negative value") + + # Compute alpha if alpha is None: alpha = 1e-10 - # Dykstra algorithm won't converge if 1/rank < alpha (alpha is the lower bound for 1/rank) - # (see "Section 3.2: The Low-rank OT Problem (LOT)" in the paper) + # Dykstra won't converge if 1/rank < alpha (see Section 3.2) if 1 / r < alpha: raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format( a=alpha, r=1 / rank)) - if r <= 0: - raise ValueError("The rank parameter cannot have a negative value") + # Low rank decomposition of the sqeuclidean cost matrix + M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx) - # Low rank decomposition of the sqeuclidean cost matrix (A, B) - M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None) + # Initialize the low rank matrices Q, R, g + if init is None: + init = "random" - # Compute gamma (see "Section 3.4, proposition 4" in the paper) - L = nx.sqrt( - 3 * (2 / (alpha**4)) * ((nx.norm(M1) * nx.norm(M2)) ** 2) + - (reg + (2 / (alpha**3)) * (nx.norm(M1) * nx.norm(M2))) ** 2 - ) - gamma = 1 / (2 * L) + Q, R, g = _init_lr_sinkhorn(X_s, X_t, a, b, r, init, reg_init, seed_init, nx=nx) - if reg_init is None: - reg_init = 1 + # Gamma initialization + if gamma_init is None: + gamma_init = "rescale" - # Initialize the low rank matrices Q, R, g - Q, R, g = _init_lr_sinkhorn(X_s, X_t, a, b, r, init, reg_init, seed_init, nx=nx) - k = 100 + if gamma_init == "theory": + L = nx.sqrt( + 3 * (2 / (alpha**4)) * ((nx.norm(M1) * nx.norm(M2)) ** 2) + + (reg + (2 / (alpha**3)) * (nx.norm(M1) * nx.norm(M2))) ** 2 + ) + gamma = 1 / (2 * L) + + if gamma_init not in ["rescale", "theory"]: + raise (NotImplementedError('Not implemented gamma_init="{}"'.format(gamma_init))) # -------------------------- Low rank algorithm ------------------------------ - # see "Section 3.3, Algorithm 3 LOT" in the paper + # see "Section 3.3, Algorithm 3 LOT" - for ii in range(k): - # Compute the C*R dot matrix using the lr decomposition of C - CR_ = nx.dot(M2.T, R) - CR = nx.dot(M1, CR_) + for ii in range(100): + # Compute C*R dot using the lr decomposition of C + CR = nx.dot(M2.T, R) + CR_ = nx.dot(M1, CR) + diag_g = (1 / g)[None, :] + CR_g = CR_ * diag_g - # Compute the C.t * Q dot matrix using the lr decomposition of C - CQ_ = nx.dot(M1.T, Q) - CQ = nx.dot(M2, CQ_) + # Compute C.T * Q using the lr decomposition of C + CQ = nx.dot(M1.T, Q) + CQ_ = nx.dot(M2, CQ) + CQ_g = CQ_ * diag_g - diag_g = (1 / g)[None, :] + # Compute omega + omega = nx.diag(nx.dot(Q.T, CR_)) + + # Rescale gamma at each iteration + if gamma_init == "rescale": + norm_1 = nx.max(nx.abs(CR_ * diag_g + reg * nx.log(Q))) ** 2 + norm_2 = nx.max(nx.abs(CQ_ * diag_g + reg * nx.log(R))) ** 2 + norm_3 = nx.max(nx.abs(-omega * diag_g)) ** 2 + gamma = 10 / max(norm_1, norm_2, norm_3) - eps1 = nx.exp(-gamma * (CR * diag_g) - ((gamma * reg) - 1) * nx.log(Q)) - eps2 = nx.exp(-gamma * (CQ * diag_g) - ((gamma * reg) - 1) * nx.log(R)) - omega = nx.diag(nx.dot(Q.T, CR)) - eps3 = nx.exp(gamma * omega / (g**2) - (gamma * reg - 1) * nx.log(g)) + eps1 = nx.exp(-gamma * CR_g - ((gamma * reg) - 1) * nx.log(Q)) + eps2 = nx.exp(-gamma * CQ_g - ((gamma * reg) - 1) * nx.log(R)) + eps3 = nx.exp((gamma * omega / (g**2)) - (gamma * reg - 1) * nx.log(g)) + # LR Dykstra algorithm Q, R, g = _LR_Dysktra( eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx ) @@ -451,7 +485,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, v2 = nx.dot(R, (v1.T * diag_g).T) value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) - # Compute value with entropy reg (entropy of Q, R, g must be computed separatly, see "Section 3.2" in the paper) + # Compute value with entropy reg (see "Section 3.2" in the paper) reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R diff --git a/ot/solvers.py b/ot/solvers.py index 40a03e974..c4c0c79ed 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -1173,6 +1173,10 @@ def solve_sample(X_a, X_b, a=None, b=None, metric='sqeuclidean', reg=None, reg_t Unbalanced optimal transport through non-negative penalized linear regression. NeurIPS. + .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). + Low-rank Sinkhorn Factorization. In International Conference on + Machine Learning. + """ @@ -1255,13 +1259,13 @@ def solve_sample(X_a, X_b, a=None, b=None, metric='sqeuclidean', reg=None, reg_t raise (NotImplementedError('Not implemented metric="{}"'.format(metric))) if max_iter is None: - max_iter = 1000 + max_iter = 2000 if tol is None: - tol = 1e-9 + tol = 1e-7 if reg is None: reg = 0 - Q, R, g, log = lowrank_sinkhorn(X_a, X_b, reg=reg, a=a, b=b, numItermax=max_iter, stopThr=tol, log=True) + Q, R, g, log = lowrank_sinkhorn(X_a, X_b, rank=rank, reg=reg, a=a, b=b, numItermax=max_iter, stopThr=tol, log=True) value = log['value'] value_linear = log['value_linear'] lazy_plan = log['lazy_plan'] diff --git a/test/test_lowrank.py b/test/test_lowrank.py index e6c95e3de..40978b75e 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -15,7 +15,7 @@ def test_compute_lr_sqeuclidean_matrix(): X_s = np.reshape(1.0 * np.arange(2 * n), (n, 2)) X_t = np.reshape(1.0 * np.arange(2 * n), (n, 2)) - M1, M2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X_s, X_t) + M1, M2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost=False) M = ot.dist(X_s, X_t, metric="sqeuclidean") # original cost matrix np.testing.assert_allclose(np.dot(M1, M2.T), M, atol=1e-05) @@ -30,7 +30,7 @@ def test_lowrank_sinkhorn(): X_s = np.reshape(1.0 * np.arange(n), (n, 1)) X_t = np.reshape(1.0 * np.arange(n), (n, 1)) - Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True) + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True, rescale_cost=False) P = log["lazy_plan"][:] value_linear = log["value_linear"] @@ -84,9 +84,27 @@ def test_lowrank_sinkhorn_alpha_error(alpha, rank): ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) +@pytest.mark.parametrize(("gamma_init"), ("rescale", "theory")) +def test_lowrank_sinkhorn_gamma_init(gamma_init): + # Test lr sinkhorn with different init strategies + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(n), (n, 1)) + + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, gamma_init=gamma_init, log=True) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + @pytest.skip_backend('tf') def test_lowrank_sinkhorn_backends(nx): - # Test low rank sinkhorn for different backends + # test low rank sinkhorn for different backends n = 100 a = ot.unif(n) b = ot.unif(n) diff --git a/test/test_solvers.py b/test/test_solvers.py index 343220c45..164989811 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -30,7 +30,7 @@ {'method': 'gaussian'}, {'method': 'gaussian', 'reg': 1}, {'method': 'factored', 'rank': 10}, - {'method': 'lowrank', 'reg': 0.1} + {'method': 'lowrank', 'rank': 10} ] lst_parameters_solve_sample_NotImplemented = [ From 3649633d50043bb8cfce0d10b877b68b32a85977 Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 20 Dec 2023 14:34:58 +0100 Subject: [PATCH 25/36] modified lowrank --- ot/lowrank.py | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index c59bfc092..18e8e10b9 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -13,8 +13,6 @@ from .bregman import sinkhorn from sklearn.cluster import KMeans -# ADD FUNCTION FOR LOW RANK INIT [WIP] - def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=None, nx=None): """ @@ -137,8 +135,6 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No return Q, R, g -################################################################################## - def compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx=None): """ @@ -181,11 +177,11 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx=None): # First low rank decomposition of the cost matrix (A) array1 = nx.reshape(nx.sum(X_s**2, 1), (-1, 1)) - array2 = nx.reshape(nx.ones(ns, type_as=X_s), (-1, 1)) + array2 = nx.ones((ns, 1), type_as=X_s) M1 = nx.concatenate((array1, array2, -2 * X_s), axis=1) # Second low rank decomposition of the cost matrix (B) - array1 = nx.reshape(nx.ones(nt, type_as=X_s), (-1, 1)) + array1 = nx.ones((nt, 1), type_as=X_s) array2 = nx.reshape(nx.sum(X_t**2, 1), (-1, 1)) M2 = nx.concatenate((array1, array2, X_t), axis=1) @@ -309,11 +305,12 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N return Q, R, g -def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, rescale_cost=True, - init=None, reg_init=None, seed_init=None, gamma_init=None, +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, rescale_cost=True, + init="random", reg_init=None, seed_init=None, gamma_init="rescale", numItermax=2000, stopThr=1e-7, warn=True, log=False): r""" - Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints + on the couplings. The function solves the following optimization problem: @@ -345,18 +342,20 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, res Regularization term >0 rank : int, optional. Default is None. (>0) Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. - alpha : int, optional. Default is None. (>0 and <1/r) - Lower bound for the weight vector g. If None, 1e-10 is considered + alpha : int, optional. Default is 1e-10. (>0 and <1/r) + Lower bound for the weight vector g. rescale_cost : bool, optional. Default is False Rescale the low rank factorization of the sqeuclidean cost matrix - init : str, optional. Default is None. If None, "random" is considered. + init : str, optional. Default is 'random'. Initialization strategy for the low rank couplings. 'random', 'trivial' or 'kmeans' reg_init : float, optional. Default is None. (>0) Regularization term for a 'kmeans' init. If None, 1 is considered. seed_init : int, optional. Default is None. (>0) Random state for a 'random' or 'kmeans' init strategy. - gamma_init : str, optional. If None, 'rescale' is considered + gamma_init : str, optional. Default is "rescale". Initialization strategy for gamma. 'rescale', or 'theory' + Gamma is a constant that scales the convergence criterion of the Mirror Descent + optimization scheme used to compute the low-rank couplings (Q, R and g) numItermax : int, optional. Default is 2000. Max number of iterations for the Dykstra algorithm stopThr : float, optional. Default is 1e-7. @@ -406,10 +405,6 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, res if r <= 0: raise ValueError("The rank parameter cannot have a negative value") - # Compute alpha - if alpha is None: - alpha = 1e-10 - # Dykstra won't converge if 1/rank < alpha (see Section 3.2) if 1 / r < alpha: raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format( @@ -419,15 +414,9 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, res M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx) # Initialize the low rank matrices Q, R, g - if init is None: - init = "random" - Q, R, g = _init_lr_sinkhorn(X_s, X_t, a, b, r, init, reg_init, seed_init, nx=nx) # Gamma initialization - if gamma_init is None: - gamma_init = "rescale" - if gamma_init == "theory": L = nx.sqrt( 3 * (2 / (alpha**4)) * ((nx.norm(M1) * nx.norm(M2)) ** 2) + @@ -473,6 +462,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, res ) Q = Q + 1e-16 R = R + 1e-16 + g = g + 1e-16 # ----------------- Compute lazy_plan, value and value_linear ------------------ # see "Section 3.2: The Low-rank OT Problem" in the paper From 5c4c4a88a19df208b2cb03360a9117cb07781c8f Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 20 Dec 2023 15:47:46 +0100 Subject: [PATCH 26/36] changes from code review --- examples/others/plot_lowrank_sinkhorn.py | 21 +- ot/lowrank.py | 281 +++++++++++++++++------ ot/solvers.py | 7 +- test/test_lowrank.py | 45 +++- test/test_solvers.py | 2 +- 5 files changed, 275 insertions(+), 81 deletions(-) diff --git a/examples/others/plot_lowrank_sinkhorn.py b/examples/others/plot_lowrank_sinkhorn.py index d2e4a6513..12aa67b3d 100644 --- a/examples/others/plot_lowrank_sinkhorn.py +++ b/examples/others/plot_lowrank_sinkhorn.py @@ -86,36 +86,43 @@ #%% # Plot sinkhorn vs low rank sinkhorn -pl.figure(7, figsize=(12, 6)) +pl.figure(3, figsize=(10, 4)) -pl.subplot(2, 3, 1) +pl.subplot(1, 3, 1) pl.imshow(list_P_Sin[0], interpolation='nearest') pl.axis('off') pl.title('Sinkhorn (reg=0.05)') -pl.subplot(2, 3, 2) +pl.subplot(1, 3, 2) pl.imshow(list_P_Sin[1], interpolation='nearest') pl.axis('off') pl.title('Sinkhorn (reg=0.005)') -pl.subplot(2, 3, 3) +pl.subplot(1, 3, 3) pl.imshow(list_P_Sin[2], interpolation='nearest') pl.axis('off') pl.title('Sinkhorn (reg=0.001)') +pl.show() -pl.subplot(2, 3, 4) + +#%% + +pl.figure(3, figsize=(10, 4)) + +pl.subplot(1, 3, 1) pl.imshow(list_P_LR[0], interpolation='nearest') pl.axis('off') pl.title('Low rank (rank=3)') -pl.subplot(2, 3, 5) +pl.subplot(1, 3, 2) pl.imshow(list_P_LR[1], interpolation='nearest') pl.axis('off') pl.title('Low rank (rank=10)') -pl.subplot(2, 3, 6) +pl.subplot(1, 3, 3) pl.imshow(list_P_LR[2], interpolation='nearest') pl.axis('off') pl.title('Low rank (rank=50)') pl.tight_layout() + diff --git a/ot/lowrank.py b/ot/lowrank.py index 5c8f673cb..921783027 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -8,14 +8,138 @@ import warnings -from .utils import unif, get_lowrank_lazytensor +from .utils import unif, dist, get_lowrank_lazytensor from .backend import get_backend +from .bregman import sinkhorn +from sklearn.cluster import KMeans -def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): +def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=None, nx=None): + """ + Implementation of different initialization strategies for the low rank sinkhorn solver (Q ,R, g). + This function is specific to lowrank_sinkhorn. + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + a : array-like, shape (n_samples_a,) + samples weights in the source domain + b : array-like, shape (n_samples_b,) + samples weights in the target domain + rank : int + Nonnegative rank of the OT plan. + init : str + Initialization strategy for Q, R and g. 'random', 'trivial' or 'kmeans' + reg_init : float, optional. Default is None. (>0) + Regularization term for a 'kmeans' init. If None, 1 is considered. + random_state : default None + Random state for a "random" or 'kmeans' init strategy + nx : default None + POT backend + + + Returns + --------- + Q : array-like, shape (n_samples_a, r) + Init for the first low-rank matrix decomposition of the OT plan (Q) + R: array-like, shape (n_samples_b, r) + Init for the second low-rank matrix decomposition of the OT plan (R) + g : array-like, shape (r, ) + Init for the weight vector of the low-rank decomposition of the OT plan (g) + + + References + ----------- + .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). + "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. + + """ + + if nx is None: + nx = get_backend(X_s, X_t, a, b) + + if reg_init is None: + reg_init = 0.1 + + if random_state is None: + random_state = 49 + + ns = X_s.shape[0] + nt = X_t.shape[0] + r = rank + + if init == "random": + nx.seed(seed=random_state) + + # Init g + g = nx.abs(nx.randn(r, type_as=X_s)) + 1 + g = g / nx.sum(g) + + # Init Q + Q = nx.abs(nx.randn(ns, r, type_as=X_s)) + 1 + Q = (Q.T * (a / nx.sum(Q, axis=1))).T + + # Init R + R = nx.abs(nx.randn(nt, rank, type_as=X_s)) + 1 + R = (R.T * (b / nx.sum(R, axis=1))).T + + if init == "trivial": + # Init g + g = nx.ones(rank) / rank + + lambda_1 = min(nx.min(a), nx.min(g), nx.min(b)) / 2 + a1 = nx.arange(start=1, stop=ns + 1, type_as=X_s) + a1 = a1 / nx.sum(a1) + a2 = (a - lambda_1 * a1) / (1 - lambda_1) + + b1 = nx.arange(start=1, stop=nt + 1, type_as=X_s) + b1 = b1 / nx.sum(b1) + b2 = (b - lambda_1 * b1) / (1 - lambda_1) + + g1 = nx.arange(start=1, stop=rank + 1, type_as=X_s) + g1 = g1 / nx.sum(g1) + g2 = (g - lambda_1 * g1) / (1 - lambda_1) + + # Init Q + Q1 = lambda_1 * nx.dot(a1[:, None], nx.reshape(g1, (1, -1))) + Q2 = (1 - lambda_1) * nx.dot(a2[:, None], nx.reshape(g2, (1, -1))) + Q = Q1 + Q2 + + # Init R + R1 = lambda_1 * nx.dot(b1[:, None], nx.reshape(g1, (1, -1))) + R2 = (1 - lambda_1) * nx.dot(b2[:, None], nx.reshape(g2, (1, -1))) + R = R1 + R2 + + if init == "kmeans": + # Init g + g = nx.ones(rank, type_as=X_s) / rank + + # Init Q + kmeans_Xs = KMeans(n_clusters=rank, random_state=random_state, n_init="auto") + kmeans_Xs.fit(X_s) + Z_Xs = nx.from_numpy(kmeans_Xs.cluster_centers_) + C_Xs = dist(X_s, Z_Xs) # shape (ns, rank) + C_Xs = C_Xs / nx.max(C_Xs) + Q = sinkhorn(a, g, C_Xs, reg=reg_init, numItermax=10000, stopThr=1e-3) + + # Init R + kmeans_Xt = KMeans(n_clusters=rank, random_state=random_state, n_init="auto") + kmeans_Xt.fit(X_t) + Z_Xt = nx.from_numpy(kmeans_Xt.cluster_centers_) + C_Xt = dist(X_t, Z_Xt) # shape (nt, rank) + C_Xt = C_Xt / nx.max(C_Xt) + R = sinkhorn(b, g, C_Xt, reg=reg_init, numItermax=10000, stopThr=1e-3) + + return Q, R, g + + +def compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx=None): """ Compute the low rank decomposition of a squared euclidean distance matrix. - This function won't work for any other distance metric. + This function won't work for other distance metrics. See "Section 3.5, proposition 1" @@ -25,7 +149,10 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): samples in the source domain X_t : array-like, shape (n_samples_b, dim) samples in the target domain - nx : POT backend, default none + rescale_cost : bool + Rescale the low rank factorization of the sqeuclidean cost matrix + nx : default None + POT backend Returns @@ -37,9 +164,9 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): References - ---------- + ----------- .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). - "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. + "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. """ if nx is None: @@ -50,14 +177,18 @@ def compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None): # First low rank decomposition of the cost matrix (A) array1 = nx.reshape(nx.sum(X_s**2, 1), (-1, 1)) - array2 = nx.reshape(nx.ones(ns, type_as=X_s), (-1, 1)) + array2 = nx.ones((ns, 1), type_as=X_s) M1 = nx.concatenate((array1, array2, -2 * X_s), axis=1) # Second low rank decomposition of the cost matrix (B) - array1 = nx.reshape(nx.ones(nt, type_as=X_s), (-1, 1)) + array1 = nx.ones((nt, 1), type_as=X_s) array2 = nx.reshape(nx.sum(X_t**2, 1), (-1, 1)) M2 = nx.concatenate((array1, array2, X_t), axis=1) + if rescale_cost is True: + M1 = M1 / nx.sqrt(nx.max(M1)) + M2 = M2 / nx.sqrt(nx.max(M2)) + return M1, M2 @@ -103,7 +234,7 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N References ---------- .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). - "Low-rank Sinkhorn factorization". In International Conference on Machine Learning. + "Low-rank Sinkhorn Factorization". In International Conference on Machine Learning. """ @@ -163,7 +294,7 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N else: if warn: warnings.warn( - "Sinkhorn did not converge. You might want to " + "Dykstra did not converge. You might want to " "increase the number of iterations `numItermax` " ) @@ -174,10 +305,12 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N return Q, R, g -def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, - numItermax=1000, stopThr=1e-9, warn=True, log=False): +def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, rescale_cost=True, + init="random", reg_init=None, seed_init=None, gamma_init="rescale", + numItermax=2000, stopThr=1e-7, warn=True, log=False): r""" - Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints. + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints + on the couplings. The function solves the following optimization problem: @@ -207,14 +340,26 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, samples weights in the target domain reg : float, optional Regularization term >0 - rank: int, optional. Default is None. (>0) + rank : int, optional. Default is None. (>0) Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. - alpha: int, optional. Default is None. (>0 and <1/r) - Lower bound for the weight vector g. If None, 1e-10 is considered - numItermax : int, optional - Max number of iterations - stopThr : float, optional - Stop threshold on error (>0) + alpha : int, optional. Default is 1e-10. (>0 and <1/r) + Lower bound for the weight vector g. + rescale_cost : bool, optional. Default is False + Rescale the low rank factorization of the sqeuclidean cost matrix + init : str, optional. Default is 'random'. + Initialization strategy for the low rank couplings. 'random', 'trivial' or 'kmeans' + reg_init : float, optional. Default is None. (>0) + Regularization term for a 'kmeans' init. If None, 1 is considered. + seed_init : int, optional. Default is None. (>0) + Random state for a 'random' or 'kmeans' init strategy. + gamma_init : str, optional. Default is "rescale". + Initialization strategy for gamma. 'rescale', or 'theory' + Gamma is a constant that scales the convergence criterion of the Mirror Descent + optimization scheme used to compute the low-rank couplings (Q, R and g) + numItermax : int, optional. Default is 2000. + Max number of iterations for the Dykstra algorithm + stopThr : float, optional. Default is 1e-7. + Stop threshold on error (>0) in Dykstra warn : bool, optional if True, raises a warning if the algorithm doesn't convergence. log : bool, optional @@ -222,26 +367,21 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, Returns - ------- - lazy_plan : LazyTensor() - OT plan in a LazyTensor object of shape (shape_plan) - See :any:`LazyTensor` for more information. - value : float - Optimal value of the optimization problem - value_linear : float - Linear OT loss with the optimal OT + --------- Q : array-like, shape (n_samples_a, r) First low-rank matrix decomposition of the OT plan R: array-like, shape (n_samples_b, r) Second low-rank matrix decomposition of the OT plan g : array-like, shape (r, ) - Weight vector for the low-rank decomposition of the OT plan + Weight vector for the low-rank decomposition of the OT + log : dict (lazy_plan, value and value_linear) + log dictionary return only if log==True in parameters References ---------- - .. [65] Scetbon, M., Cuturi, M., & Peyré, G (2021). - "Low-Rank Sinkhorn Factorization" arXiv preprint arXiv:2103.04737. + .. [65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). + "Low-rank Sinkhorn Factorization". In International Conference on Machine Learning. """ @@ -259,59 +399,70 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, r = rank if rank is None: r = min(ns, nt) + else: + r = min(ns, nt, rank) - if alpha is None: - alpha = 1e-10 + if r <= 0: + raise ValueError("The rank parameter cannot have a negative value") - # Dykstra algorithm won't converge if 1/rank < alpha (alpha is the lower bound for 1/rank) - # (see "Section 3.2: The Low-rank OT Problem (LOT)" in the paper) + # Dykstra won't converge if 1/rank < alpha (see Section 3.2) if 1 / r < alpha: raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format( a=alpha, r=1 / rank)) - if r <= 0: - raise ValueError("The rank parameter cannot have a negative value") + # Low rank decomposition of the sqeuclidean cost matrix + M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost, nx) - # Low rank decomposition of the sqeuclidean cost matrix (A, B) - M1, M2 = compute_lr_sqeuclidean_matrix(X_s, X_t, nx=None) + # Initialize the low rank matrices Q, R, g + Q, R, g = _init_lr_sinkhorn(X_s, X_t, a, b, r, init, reg_init, seed_init, nx=nx) - # Compute gamma (see "Section 3.4, proposition 4" in the paper) - L = nx.sqrt( - 3 * (2 / (alpha**4)) * ((nx.norm(M1) * nx.norm(M2)) ** 2) + - (reg + (2 / (alpha**3)) * (nx.norm(M1) * nx.norm(M2))) ** 2 - ) - gamma = 1 / (2 * L) + # Gamma initialization + if gamma_init == "theory": + L = nx.sqrt( + 3 * (2 / (alpha**4)) * ((nx.norm(M1) * nx.norm(M2)) ** 2) + + (reg + (2 / (alpha**3)) * (nx.norm(M1) * nx.norm(M2))) ** 2 + ) + gamma = 1 / (2 * L) - # Initialize the low rank matrices Q, R, g - Q = nx.ones((ns, r), type_as=a) - R = nx.ones((nt, r), type_as=a) - g = nx.ones(r, type_as=a) - k = 100 + if gamma_init not in ["rescale", "theory"]: + raise (NotImplementedError('Not implemented gamma_init="{}"'.format(gamma_init))) # -------------------------- Low rank algorithm ------------------------------ - # see "Section 3.3, Algorithm 3 LOT" in the paper + # see "Section 3.3, Algorithm 3 LOT" - for ii in range(k): - # Compute the C*R dot matrix using the lr decomposition of C - CR_ = nx.dot(M2.T, R) - CR = nx.dot(M1, CR_) + for ii in range(100): + # Compute C*R dot using the lr decomposition of C + CR = nx.dot(M2.T, R) + CR_ = nx.dot(M1, CR) + diag_g = (1 / g)[None, :] + CR_g = CR_ * diag_g - # Compute the C.t * Q dot matrix using the lr decomposition of C - CQ_ = nx.dot(M1.T, Q) - CQ = nx.dot(M2, CQ_) + # Compute C.T * Q using the lr decomposition of C + CQ = nx.dot(M1.T, Q) + CQ_ = nx.dot(M2, CQ) + CQ_g = CQ_ * diag_g - diag_g = (1 / g)[None, :] + # Compute omega + omega = nx.diag(nx.dot(Q.T, CR_)) + + # Rescale gamma at each iteration + if gamma_init == "rescale": + norm_1 = nx.max(nx.abs(CR_ * diag_g + reg * nx.log(Q))) ** 2 + norm_2 = nx.max(nx.abs(CQ_ * diag_g + reg * nx.log(R))) ** 2 + norm_3 = nx.max(nx.abs(-omega * diag_g)) ** 2 + gamma = 10 / max(norm_1, norm_2, norm_3) - eps1 = nx.exp(-gamma * (CR * diag_g) - ((gamma * reg) - 1) * nx.log(Q)) - eps2 = nx.exp(-gamma * (CQ * diag_g) - ((gamma * reg) - 1) * nx.log(R)) - omega = nx.diag(nx.dot(Q.T, CR)) - eps3 = nx.exp(gamma * omega / (g**2) - (gamma * reg - 1) * nx.log(g)) + eps1 = nx.exp(-gamma * CR_g - ((gamma * reg) - 1) * nx.log(Q)) + eps2 = nx.exp(-gamma * CQ_g - ((gamma * reg) - 1) * nx.log(R)) + eps3 = nx.exp((gamma * omega / (g**2)) - (gamma * reg - 1) * nx.log(g)) + # LR Dykstra algorithm Q, R, g = _LR_Dysktra( eps1, eps2, eps3, a, b, alpha, stopThr, numItermax, warn, nx ) Q = Q + 1e-16 R = R + 1e-16 + g = g + 1e-16 # ----------------- Compute lazy_plan, value and value_linear ------------------ # see "Section 3.2: The Low-rank OT Problem" in the paper @@ -324,7 +475,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, v2 = nx.dot(R, (v1.T * diag_g).T) value_linear = nx.sum(nx.diag(nx.dot(M2.T, v2))) - # Compute value with entropy reg (entropy of Q, R, g must be computed separatly, see "Section 3.2" in the paper) + # Compute value with entropy reg (see "Section 3.2" in the paper) reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R @@ -338,4 +489,4 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=None, return Q, R, g, dict_log - return Q, R, g + return Q, R, g \ No newline at end of file diff --git a/ot/solvers.py b/ot/solvers.py index 7369fb273..8a774b89d 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -23,6 +23,7 @@ from .gaussian import empirical_bures_wasserstein_distance from .factored import factored_optimal_transport from .lowrank import lowrank_sinkhorn +from .lowrank import lowrank_sinkhorn lst_method_lazy = ['1d', 'gaussian', 'lowrank', 'factored', 'geomloss', 'geomloss_auto', 'geomloss_tensorized', 'geomloss_online', 'geomloss_multiscale'] @@ -1259,13 +1260,13 @@ def solve_sample(X_a, X_b, a=None, b=None, metric='sqeuclidean', reg=None, reg_t raise (NotImplementedError('Not implemented metric="{}"'.format(metric))) if max_iter is None: - max_iter = 1000 + max_iter = 2000 if tol is None: - tol = 1e-9 + tol = 1e-7 if reg is None: reg = 0 - Q, R, g, log = lowrank_sinkhorn(X_a, X_b, reg=reg, a=a, b=b, numItermax=max_iter, stopThr=tol, log=True) + Q, R, g, log = lowrank_sinkhorn(X_a, X_b, rank=rank, reg=reg, a=a, b=b, numItermax=max_iter, stopThr=tol, log=True) value = log['value'] value_linear = log['value_linear'] lazy_plan = log['lazy_plan'] diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 65f76a77b..8ac59726c 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -15,7 +15,7 @@ def test_compute_lr_sqeuclidean_matrix(): X_s = np.reshape(1.0 * np.arange(2 * n), (n, 2)) X_t = np.reshape(1.0 * np.arange(2 * n), (n, 2)) - M1, M2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X_s, X_t) + M1, M2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X_s, X_t, rescale_cost=False) M = ot.dist(X_s, X_t, metric="sqeuclidean") # original cost matrix np.testing.assert_allclose(np.dot(M1, M2.T), M, atol=1e-05) @@ -30,7 +30,7 @@ def test_lowrank_sinkhorn(): X_s = np.reshape(1.0 * np.arange(n), (n, 1)) X_t = np.reshape(1.0 * np.arange(n), (n, 1)) - Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True) + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True, rescale_cost=False) P = log["lazy_plan"][:] value_linear = log["value_linear"] @@ -52,6 +52,24 @@ def test_lowrank_sinkhorn(): ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1) +@pytest.mark.parametrize(("init"), ("random", "trivial", "kmeans")) +def test_lowrank_sinkhorn_init(init): + # test lowrank inits + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(n), (n, 1)) + + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True, init=init, reg_init=1) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + @pytest.mark.parametrize(("alpha, rank"), ((0.8, 2), (0.5, 3), (0.2, 6))) def test_lowrank_sinkhorn_alpha_error(alpha, rank): # Test warning for value of alpha @@ -63,9 +81,25 @@ def test_lowrank_sinkhorn_alpha_error(alpha, rank): X_t = np.reshape(1.0 * np.arange(0, n), (n, 1)) with pytest.raises(ValueError): - ot.lowrank.lowrank_sinkhorn( - X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False - ) + ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) + + +@pytest.mark.parametrize(("gamma_init"), ("rescale", "theory")) +def test_lowrank_sinkhorn_gamma_init(gamma_init): + # Test lr sinkhorn with different init strategies + n = 100 + a = ot.unif(n) + b = ot.unif(n) + + X_s = np.reshape(1.0 * np.arange(n), (n, 1)) + X_t = np.reshape(1.0 * np.arange(n), (n, 1)) + + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, gamma_init=gamma_init, log=True) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) @pytest.skip_backend('tf') @@ -86,3 +120,4 @@ def test_lowrank_sinkhorn_backends(nx): np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) + diff --git a/test/test_solvers.py b/test/test_solvers.py index 343220c45..164989811 100644 --- a/test/test_solvers.py +++ b/test/test_solvers.py @@ -30,7 +30,7 @@ {'method': 'gaussian'}, {'method': 'gaussian', 'reg': 1}, {'method': 'factored', 'rank': 10}, - {'method': 'lowrank', 'reg': 0.1} + {'method': 'lowrank', 'rank': 10} ] lst_parameters_solve_sample_NotImplemented = [ From 477deed7a3fb500524f9430a493221986b12065e Mon Sep 17 00:00:00 2001 From: laudavid Date: Wed, 20 Dec 2023 15:57:22 +0100 Subject: [PATCH 27/36] fix error test pep8 --- examples/others/plot_lowrank_sinkhorn.py | 1 - ot/lowrank.py | 10 +++++----- ot/solvers.py | 1 - test/test_lowrank.py | 1 - 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/examples/others/plot_lowrank_sinkhorn.py b/examples/others/plot_lowrank_sinkhorn.py index 12aa67b3d..336209832 100644 --- a/examples/others/plot_lowrank_sinkhorn.py +++ b/examples/others/plot_lowrank_sinkhorn.py @@ -125,4 +125,3 @@ pl.title('Low rank (rank=50)') pl.tight_layout() - diff --git a/ot/lowrank.py b/ot/lowrank.py index 921783027..075bdd207 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -309,7 +309,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, re init="random", reg_init=None, seed_init=None, gamma_init="rescale", numItermax=2000, stopThr=1e-7, warn=True, log=False): r""" - Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints + Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints on the couplings. The function solves the following optimization problem: @@ -343,10 +343,10 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, re rank : int, optional. Default is None. (>0) Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. alpha : int, optional. Default is 1e-10. (>0 and <1/r) - Lower bound for the weight vector g. + Lower bound for the weight vector g. rescale_cost : bool, optional. Default is False Rescale the low rank factorization of the sqeuclidean cost matrix - init : str, optional. Default is 'random'. + init : str, optional. Default is 'random'. Initialization strategy for the low rank couplings. 'random', 'trivial' or 'kmeans' reg_init : float, optional. Default is None. (>0) Regularization term for a 'kmeans' init. If None, 1 is considered. @@ -354,7 +354,7 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, re Random state for a 'random' or 'kmeans' init strategy. gamma_init : str, optional. Default is "rescale". Initialization strategy for gamma. 'rescale', or 'theory' - Gamma is a constant that scales the convergence criterion of the Mirror Descent + Gamma is a constant that scales the convergence criterion of the Mirror Descent optimization scheme used to compute the low-rank couplings (Q, R and g) numItermax : int, optional. Default is 2000. Max number of iterations for the Dykstra algorithm @@ -489,4 +489,4 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, re return Q, R, g, dict_log - return Q, R, g \ No newline at end of file + return Q, R, g diff --git a/ot/solvers.py b/ot/solvers.py index 8a774b89d..c4c0c79ed 100644 --- a/ot/solvers.py +++ b/ot/solvers.py @@ -23,7 +23,6 @@ from .gaussian import empirical_bures_wasserstein_distance from .factored import factored_optimal_transport from .lowrank import lowrank_sinkhorn -from .lowrank import lowrank_sinkhorn lst_method_lazy = ['1d', 'gaussian', 'lowrank', 'factored', 'geomloss', 'geomloss_auto', 'geomloss_tensorized', 'geomloss_online', 'geomloss_multiscale'] diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 8ac59726c..3f31cef23 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -120,4 +120,3 @@ def test_lowrank_sinkhorn_backends(nx): np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) - From 077184f9be01196a2c97db5a92f396270bdaee36 Mon Sep 17 00:00:00 2001 From: laudavid Date: Thu, 21 Dec 2023 12:06:00 +0100 Subject: [PATCH 28/36] fix linux-minimal-deps + code review --- examples/others/plot_lowrank_sinkhorn.py | 6 +- ot/lowrank.py | 72 +++++++++++++----------- test/test_lowrank.py | 19 +++++-- 3 files changed, 55 insertions(+), 42 deletions(-) diff --git a/examples/others/plot_lowrank_sinkhorn.py b/examples/others/plot_lowrank_sinkhorn.py index 336209832..ece35b295 100644 --- a/examples/others/plot_lowrank_sinkhorn.py +++ b/examples/others/plot_lowrank_sinkhorn.py @@ -13,6 +13,8 @@ # Author: Laurène David # # License: MIT License +# +# sphinx_gallery_thumbnail_number = 2 import numpy as np import matplotlib.pylab as pl @@ -86,7 +88,7 @@ #%% # Plot sinkhorn vs low rank sinkhorn -pl.figure(3, figsize=(10, 4)) +pl.figure(1, figsize=(10, 4)) pl.subplot(1, 3, 1) pl.imshow(list_P_Sin[0], interpolation='nearest') @@ -107,7 +109,7 @@ #%% -pl.figure(3, figsize=(10, 4)) +pl.figure(2, figsize=(10, 4)) pl.subplot(1, 3, 1) pl.imshow(list_P_LR[0], interpolation='nearest') diff --git a/ot/lowrank.py b/ot/lowrank.py index 075bdd207..f6c1469bd 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -11,10 +11,16 @@ from .utils import unif, dist, get_lowrank_lazytensor from .backend import get_backend from .bregman import sinkhorn -from sklearn.cluster import KMeans +# test if sklearn is installed for linux-minimal-deps +try: + import sklearn.cluster + sklearn_import = True +except ImportError: + sklearn_import = False -def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=None, nx=None): + +def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init, random_state, nx=None): """ Implementation of different initialization strategies for the low rank sinkhorn solver (Q ,R, g). This function is specific to lowrank_sinkhorn. @@ -33,11 +39,11 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No Nonnegative rank of the OT plan. init : str Initialization strategy for Q, R and g. 'random', 'trivial' or 'kmeans' - reg_init : float, optional. Default is None. (>0) - Regularization term for a 'kmeans' init. If None, 1 is considered. - random_state : default None + reg_init : float, optional. + Regularization term for a 'kmeans' init. + random_state : int, optional. Random state for a "random" or 'kmeans' init strategy - nx : default None + nx : optional, Default is None POT backend @@ -61,12 +67,6 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No if nx is None: nx = get_backend(X_s, X_t, a, b) - if reg_init is None: - reg_init = 0.1 - - if random_state is None: - random_state = 49 - ns = X_s.shape[0] nt = X_t.shape[0] r = rank @@ -86,7 +86,7 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No R = nx.abs(nx.randn(nt, rank, type_as=X_s)) + 1 R = (R.T * (b / nx.sum(R, axis=1))).T - if init == "trivial": + if init == "deterministic": # Init g g = nx.ones(rank) / rank @@ -114,24 +114,28 @@ def _init_lr_sinkhorn(X_s, X_t, a, b, rank, init, reg_init=None, random_state=No R = R1 + R2 if init == "kmeans": - # Init g - g = nx.ones(rank, type_as=X_s) / rank - - # Init Q - kmeans_Xs = KMeans(n_clusters=rank, random_state=random_state, n_init="auto") - kmeans_Xs.fit(X_s) - Z_Xs = nx.from_numpy(kmeans_Xs.cluster_centers_) - C_Xs = dist(X_s, Z_Xs) # shape (ns, rank) - C_Xs = C_Xs / nx.max(C_Xs) - Q = sinkhorn(a, g, C_Xs, reg=reg_init, numItermax=10000, stopThr=1e-3) + if sklearn_import: + # Init g + g = nx.ones(rank, type_as=X_s) / rank + + # Init Q + kmeans_Xs = sklearn.cluster.KMeans(n_clusters=rank, random_state=random_state, n_init="auto") + kmeans_Xs.fit(X_s) + Z_Xs = nx.from_numpy(kmeans_Xs.cluster_centers_) + C_Xs = dist(X_s, Z_Xs) # shape (ns, rank) + C_Xs = C_Xs / nx.max(C_Xs) + Q = sinkhorn(a, g, C_Xs, reg=reg_init, numItermax=10000, stopThr=1e-3) + + # Init R + kmeans_Xt = sklearn.cluster.KMeans(n_clusters=rank, random_state=random_state, n_init="auto") + kmeans_Xt.fit(X_t) + Z_Xt = nx.from_numpy(kmeans_Xt.cluster_centers_) + C_Xt = dist(X_t, Z_Xt) # shape (nt, rank) + C_Xt = C_Xt / nx.max(C_Xt) + R = sinkhorn(b, g, C_Xt, reg=reg_init, numItermax=10000, stopThr=1e-3) - # Init R - kmeans_Xt = KMeans(n_clusters=rank, random_state=random_state, n_init="auto") - kmeans_Xt.fit(X_t) - Z_Xt = nx.from_numpy(kmeans_Xt.cluster_centers_) - C_Xt = dist(X_t, Z_Xt) # shape (nt, rank) - C_Xt = C_Xt / nx.max(C_Xt) - R = sinkhorn(b, g, C_Xt, reg=reg_init, numItermax=10000, stopThr=1e-3) + else: + raise ImportError("Scikit-learn should be installed to use the 'kmeans' init.") return Q, R, g @@ -306,7 +310,7 @@ def _LR_Dysktra(eps1, eps2, eps3, p1, p2, alpha, stopThr, numItermax, warn, nx=N def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, rescale_cost=True, - init="random", reg_init=None, seed_init=None, gamma_init="rescale", + init="random", reg_init=1e-1, seed_init=49, gamma_init="rescale", numItermax=2000, stopThr=1e-7, warn=True, log=False): r""" Solve the entropic regularization optimal transport problem under low-nonnegative rank constraints @@ -347,10 +351,10 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, re rescale_cost : bool, optional. Default is False Rescale the low rank factorization of the sqeuclidean cost matrix init : str, optional. Default is 'random'. - Initialization strategy for the low rank couplings. 'random', 'trivial' or 'kmeans' - reg_init : float, optional. Default is None. (>0) + Initialization strategy for the low rank couplings. 'random', 'deterministic' or 'kmeans' + reg_init : float, optional. Default is 1e-1. (>0) Regularization term for a 'kmeans' init. If None, 1 is considered. - seed_init : int, optional. Default is None. (>0) + seed_init : int, optional. Default is 49. (>0) Random state for a 'random' or 'kmeans' init strategy. gamma_init : str, optional. Default is "rescale". Initialization strategy for gamma. 'rescale', or 'theory' diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 3f31cef23..60b2d633f 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -7,6 +7,7 @@ import ot import numpy as np import pytest +from ot.lowrank import sklearn_import # check sklearn installation def test_compute_lr_sqeuclidean_matrix(): @@ -52,7 +53,7 @@ def test_lowrank_sinkhorn(): ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1) -@pytest.mark.parametrize(("init"), ("random", "trivial", "kmeans")) +@pytest.mark.parametrize(("init"), ("random", "deterministic", "kmeans")) def test_lowrank_sinkhorn_init(init): # test lowrank inits n = 100 @@ -62,12 +63,18 @@ def test_lowrank_sinkhorn_init(init): X_s = np.reshape(1.0 * np.arange(n), (n, 1)) X_t = np.reshape(1.0 * np.arange(n), (n, 1)) - Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, log=True, init=init, reg_init=1) - P = log["lazy_plan"][:] + # test ImportError if init="kmeans" and sklearn not imported + if init in ["random", "deterministic"] or ((init == "kmeans") and (sklearn_import is True)): + Q, R, g, log = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, init=init, log=True) + P = log["lazy_plan"][:] - # check constraints for P - np.testing.assert_allclose(a, P.sum(1), atol=1e-05) - np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-05) + np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + + else: + with pytest.raises(ImportError): + Q, R, g = ot.lowrank.lowrank_sinkhorn(X_s, X_t, a, b, reg=0.1, init=init) @pytest.mark.parametrize(("alpha, rank"), ((0.8, 2), (0.5, 3), (0.2, 6))) From 7d3071f1f31ac03ca7c3f9d766a169b381562279 Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 15:29:52 +0100 Subject: [PATCH 29/36] Implementation of LR GW + add method in __init__ --- ot/__init__.py | 5 +- ot/lowrank.py | 271 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 274 insertions(+), 2 deletions(-) diff --git a/ot/__init__.py b/ot/__init__.py index 99d075e5a..6f7d91e96 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -53,7 +53,7 @@ from .weak import weak_optimal_transport from .factored import factored_optimal_transport from .solvers import solve, solve_gromov, solve_sample -from .lowrank import lowrank_sinkhorn +from .lowrank import lowrank_sinkhorn, lowrank_gromov_wasserstein # utils functions from .utils import dist, unif, tic, toc, toq @@ -71,4 +71,5 @@ 'factored_optimal_transport', 'solve', 'solve_gromov','solve_sample', 'smooth', 'stochastic', 'unbalanced', 'partial', 'regpath', 'solvers', 'binary_search_circle', 'wasserstein_circle', - 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif', 'lowrank_sinkhorn'] + 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif', 'lowrank_sinkhorn', + 'lowrank_gromov_wasserstein'] diff --git a/ot/lowrank.py b/ot/lowrank.py index f6c1469bd..2938102d7 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -494,3 +494,274 @@ def lowrank_sinkhorn(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, re return Q, R, g, dict_log return Q, R, g + + +def _flat_product_operator(X, nx=None): + r""" + Implementation of the flattened out-product operator. + + This function is used in low rank gromov wasserstein to compute the low rank decomposition of + a cost matrix's squared hadamard product (page 6 in paper). + + Parameters + ---------- + X: array-like, shape (n_samples, n_col) + Input matrix for operator + + nx: default None + POT backend + + Returns + ---------- + X_flat: array-like, shape (n_samples, n_col**2) + Matrix with flattened out-product operator applied on each row + + + """ + if nx is None: + nx = get_backend(X) + + n = X.shape[0] + x1 = X[0, :][:, None] + X_flat = nx.dot(x1, x1.T).flatten()[:,None] + + for i in range(1,n): + x = X[i, :][:, None] + x_out = nx.dot(x, x.T).flatten()[:,None] + X_flat= nx.concatenate((X_flat, x_out), axis=1) + + X_flat = X_flat.T + + return X_flat + + +def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, gamma_init="rescale", + rescale_cost=True, stopThr=1e-4, numItermax=1000, stopThr_dykstra=1e-3, + numItermax_dykstra=10000, seed_init=49, warn=True, warn_dykstra=False, log=False): + + r""" + Solve the entropic regularization Gromov-Wasserstein transport problem under low-nonnegative rank constraints + on the couplings and cost matrices. + + The function solves the following optimization problem: + + .. math:: + \mathop{\min_{(Q,R,g) \in \mathcal{C(a,b,r)}}} \mathcal{Q}_{A,B}(Q\mathrm{diag}(1/g)R^T) - + \epsilon \cdot H((Q,R,g)) + + where : + - :math: `A` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the source domain + - :math: `B` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the target domain + - :math: `\mathcal{Q}_{A,B}` is quadratic objective function of the Gromov Wasserstein plan + - :math: `Q` and `R` are the low-rank matrix decomposition of the Gromov-Wasserstein plan + - :math: `g` is the weight vector for the low-rank decomposition of the Gromov-Wasserstein plan + - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target weights (histograms, both sum to 1) + - :math: `r` is the rank of the Gromov-Wasserstein plan + - :math: `\mathcal{C(a,b,r)}` are the low-rank couplings of the OT problem + - :math:`H((Q,R,g))` is the values of the three respective entropies evaluated for each term. + + + Parameters + ---------- + X_s : array-like, shape (n_samples_a, dim) + samples in the source domain + X_t : array-like, shape (n_samples_b, dim) + samples in the target domain + a : array-like, shape (n_samples_a,) + samples weights in the source domain + b : array-like, shape (n_samples_b,) + samples weights in the target domain + reg : float, optional + Regularization term >0 + rank : int, optional. Default is None. (>0) + Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. + alpha : int, optional. Default is 1e-10. (>0 and <1/r) + Lower bound for the weight vector g. + rescale_cost : bool, optional. Default is False + Rescale the low rank factorization of the sqeuclidean cost matrix + seed_init : int, optional. Default is 49. (>0) + Random state for the 'random' initialization of low rank couplings + gamma_init : str, optional. Default is "rescale". + Initialization strategy for gamma. 'rescale', or 'theory' + Gamma is a constant that scales the convergence criterion of the Mirror Descent + optimization scheme used to compute the low-rank couplings (Q, R and g) + numItermax : int, optional. Default is 1000. + Max number of iterations for Low Rank GW + stopThr : float, optional. Default is 1e-4. + Stop threshold on error (>0) for Low Rank GW + numItermax_dykstra : int, optional. Default is 2000. + Max number of iterations for the Dykstra algorithm + stopThr_dykstra : float, optional. Default is 1e-7. + Stop threshold on error (>0) in Dykstra + warn : bool, optional + if True, raises a warning if the low rank GW algorithm doesn't convergence. + warn_dykstra: bool, optional + if True, raises a warning if the Dykstra algorithm doesn't convergence. + log : bool, optional + record log if True + + + Returns + --------- + Q : array-like, shape (n_samples_a, r) + First low-rank matrix decomposition of the OT plan + R: array-like, shape (n_samples_b, r) + Second low-rank matrix decomposition of the OT plan + g : array-like, shape (r, ) + Weight vector for the low-rank decomposition of the OT + log : dict (lazy_plan, value and value_linear) + log dictionary return only if log==True in parameters + + + References + ---------- + .. [66] Scetbon, M., Peyré, G. & Cuturi, M. (2022). + "Linear-Time GromovWasserstein Distances using Low Rank Couplings and Costs". + In International Conference on Machine Learning (ICML), 2022. + + """ + + # POT backend + nx = get_backend(X_s, X_t) + ns, nt = X_s.shape[0], X_t.shape[0] + + # Initialize weights a, b + if a is None: + a = unif(ns, type_as=X_s) + if b is None: + b = unif(nt, type_as=X_t) + + # Compute rank (see Section 3.1, def 1) + r = rank + if rank is None: + r = min(ns, nt) + else: + r = min(ns, nt, rank) + + if r <= 0: + raise ValueError("The rank parameter cannot have a negative value") + + # Dykstra won't converge if 1/rank < alpha (see Section 3.2) + if 1 / r < alpha: + raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format( + a=alpha, r=1 / rank)) + + # LR decomposition of the two sqeuclidean cost matrices + A1, A2 = compute_lr_sqeuclidean_matrix(X_s, X_s, rescale_cost, nx=nx) + B1, B2 = compute_lr_sqeuclidean_matrix(X_t, X_t, rescale_cost, nx=nx) + + # Initial values for LR couplings (Q, R, g) with LOT + Q, R, g = _init_lr_sinkhorn( + X_s, X_t, a, b, r, init="random", random_state=seed_init, reg_init=None, nx=nx + ) + + # Gamma initialization + if gamma_init == "theory": + L = (27 * nx.norm(A1) * nx.norm(A2)) / alpha**4 + gamma = 1 / (2 * L) + + if gamma_init not in ["rescale", "theory"]: + raise (NotImplementedError('Not implemented gamma_init="{}"'.format(gamma_init))) + + # initial value of error + err = 1 + + for ii in range(numItermax): + Q_prev = Q + R_prev = R + g_prev = g + + if err > stopThr: + # Compute cost matrices + C1 = nx.dot(A2.T, Q * (1 / g)[None, :]) + C1 = - 4 * nx.dot(A1, C1) + C2 = nx.dot(R.T, B1) + C2 = nx.dot(C2, B2.T) + diag_g = (1 / g)[None, :] + + # Compute C*R dot using the lr decomposition of C + CR = nx.dot(C2, R) + CR = nx.dot(C1, CR) + CR_g = CR * diag_g + + # Compute C.T * Q using the lr decomposition of C + CQ = nx.dot(C1.T, Q) + CQ = nx.dot(C2.T, CQ) + CQ_g = CQ * diag_g + + # Compute omega + omega = nx.diag(nx.dot(Q.T, CR)) + + # Rescale gamma at each iteration + if gamma_init == "rescale": + norm_1 = nx.max(nx.abs(CR_g + reg * nx.log(Q))) ** 2 + norm_2 = nx.max(nx.abs(CQ_g + reg * nx.log(R))) ** 2 + norm_3 = nx.max(nx.abs(-omega * (diag_g**2))) ** 2 + gamma = 10 / max(norm_1, norm_2, norm_3) + + K1 = nx.exp(-gamma * CR_g - ((gamma * reg) - 1) * nx.log(Q)) + K2 = nx.exp(-gamma * CQ_g - ((gamma * reg) - 1) * nx.log(R)) + K3 = nx.exp((gamma * omega / (g**2)) - (gamma * reg - 1) * nx.log(g)) + + # Update couplings with LR Dykstra algorithm + Q, R, g = _LR_Dysktra( + K1, K2, K3, a, b, alpha, stopThr_dykstra, numItermax_dykstra, warn_dykstra, nx + ) + + # Update error with kullback-divergence + err_1 = ((1 / gamma) ** 2) * (nx.kl_div(Q, Q_prev) + nx.kl_div(Q_prev, Q)) + err_2 = ((1 / gamma) ** 2) * (nx.kl_div(R, R_prev) + nx.kl_div(R_prev, R)) + err_3 = ((1 / gamma) ** 2) * (nx.kl_div(g, g_prev) + nx.kl_div(g_prev, g)) + err = err_1 + err_2 + err_3 + + # fix divide by zero + Q = Q + 1e-16 + R = R + 1e-16 + g = g + 1e-16 + + else: + break + + else: + if warn: + warnings.warn( + "Low Rank GW did not converge. You might want to " + "increase the number of iterations `numItermax` " + ) + + # Update low rank costs + C1 = nx.dot(A2.T, Q * (1 / g)[None, :]) + C1 = - 4 * nx.dot(A1, C1) + C2 = nx.dot(R.T, B1) + C2 = nx.dot(C2, B2.T) + + # Compute lazy plan (using LazyTensor class) + lazy_plan = get_lowrank_lazytensor(Q, R, 1 / g) + + # Compute value_quad + A1_, A2_ = _flat_product_operator(A1, nx), _flat_product_operator(A2, nx) + B1_, B2_ = _flat_product_operator(B1, nx), _flat_product_operator(B2, nx) + + x_ = nx.dot(A1_, nx.dot(A2_.T, a)) + y_ = nx.dot(B1_, nx.dot(B2_.T, b)) + c1 = nx.dot(x_, a) + nx.dot(y_, b) + + G = nx.dot(C1, nx.dot(C2, R)) + G = nx.dot(Q.T, G * diag_g) + value_quad = c1 + nx.trace(G) / 2 + + # Compute value with entropy reg (see "Section 3.2" in the paper) + reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q + reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g + reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R + value = value_quad + reg * (reg_Q + reg_g + reg_R) + + if log: + dict_log = dict() + dict_log["value"] = value + dict_log["value_quad"] = value_quad + dict_log["lazy_plan"] = lazy_plan + + return Q, R, g, dict_log + + return Q, R, g From bd06dc40575fc52a2dea9f2946536fc803aa518c Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 15:30:34 +0100 Subject: [PATCH 30/36] add LR gw paper in README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c9b212ce..37e1ee19c 100644 --- a/README.md +++ b/README.md @@ -349,4 +349,6 @@ distances between Gaussian distributions](https://hal.science/hal-03197398v2/fil [64] Ma, X., Chu, X., Wang, Y., Lin, Y., Zhao, J., Ma, L., & Zhu, W. (2023). [Fused Gromov-Wasserstein Graph Mixup for Graph-level Classifications](https://openreview.net/pdf?id=uqkUguNu40). In Thirty-seventh Conference on Neural Information Processing Systems. -[65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). [Low-Rank Sinkhorn Factorization](https://arxiv.org/pdf/2103.04737.pdf). \ No newline at end of file +[65] Scetbon, M., Cuturi, M., & Peyré, G. (2021). [Low-Rank Sinkhorn Factorization](https://arxiv.org/pdf/2103.04737.pdf). + +[66] Scetbon, M., Peyré, G. & Cuturi, M. (2022). [Linear-Time GromovWasserstein Distances using Low Rank Couplings and Costs](https://proceedings.mlr.press/v162/scetbon22b/scetbon22b.pdf). In International Conference on Machine Learning (ICML), 2022. \ No newline at end of file From 99b38d5c2f5aea57c46aa064cbb0a9707928aacc Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 15:31:16 +0100 Subject: [PATCH 31/36] add tests for low rank GW --- test/test_lowrank.py | 119 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 4 deletions(-) diff --git a/test/test_lowrank.py b/test/test_lowrank.py index 60b2d633f..cb574b844 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -105,8 +105,8 @@ def test_lowrank_sinkhorn_gamma_init(gamma_init): P = log["lazy_plan"][:] # check constraints for P - np.testing.assert_allclose(a, P.sum(1), atol=1e-05) - np.testing.assert_allclose(b, P.sum(0), atol=1e-05) + np.testing.assert_allclose(a, P.sum(1), atol=1e-04) + np.testing.assert_allclose(b, P.sum(0), atol=1e-04) @pytest.skip_backend('tf') @@ -125,5 +125,116 @@ def test_lowrank_sinkhorn_backends(nx): lazy_plan = log["lazy_plan"] P = lazy_plan[:] - np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) - np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) + np.testing.assert_allclose(ab, P.sum(1), atol=1e-04) + np.testing.assert_allclose(bb, P.sum(0), atol=1e-04) + + +def test__flat_product_operator(): + # test flat product operator + n, d = 100, 2 + X = np.reshape(1.0 * np.arange(2 * n), (n, d)) + A1, A2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X, X, rescale_cost=False) + + A1_ = ot.lowrank._flat_product_operator(A1) + A2_ = ot.lowrank._flat_product_operator(A2) + cost = ot.dist(X, X) + + # test value + np.testing.assert_allclose(cost**2, np.dot(A1_, A2_.T), atol=1e-05) + + +def test_lowrank_gromov_wasserstein(): + # test low rank gromov wasserstein + n_samples = 20 # nb samples + mu_s = np.array([0, 0]) + cov_s = np.array([[1, 0], [0, 1]]) + + X_s = ot.datasets.make_2D_samples_gauss(n_samples, mu_s, cov_s, random_state=1) + X_t = X_s[::-1].copy() + + a = ot.unif(n_samples) + b = ot.unif(n_samples) + + Q, R, g, log = ot.lowrank.lowrank_gromov_wasserstein(X_s, X_t, a, b, reg=0.1, log=True, rescale_cost=False) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-04) + np.testing.assert_allclose(b, P.sum(0), atol=1e-04) + + # check if lazy_plan is equal to the fully computed plan + P_true = np.dot(Q, np.dot(np.diag(1 / g), R.T)) + np.testing.assert_allclose(P, P_true, atol=1e-05) + + # check warn parameter when low rank GW algorithm doesn't converge + with pytest.warns(UserWarning): + ot.lowrank.lowrank_gromov_wasserstein( + X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1, warn=True, warn_dykstra=False + ) + + # check warn parameter when Dykstra algorithm doesn't converge + with pytest.warns(UserWarning): + ot.lowrank.lowrank_gromov_wasserstein( + X_s, X_t, a, b, reg=0.1, stopThr_dykstra=0, numItermax_dykstra=1, warn=False, warn_dykstra=True + ) + + +@pytest.mark.parametrize(("alpha, rank"), ((0.8, 2), (0.5, 3), (0.2, 6))) +def test_lowrank_gromov_wasserstein_alpha_error(alpha, rank): + # Test warning for value of alpha + n_samples = 20 # nb samples + mu_s = np.array([0, 0]) + cov_s = np.array([[1, 0], [0, 1]]) + + X_s = ot.datasets.make_2D_samples_gauss(n_samples, mu_s, cov_s, random_state=1) + X_t = X_s[::-1].copy() + + a = ot.unif(n_samples) + b = ot.unif(n_samples) + + with pytest.raises(ValueError): + ot.lowrank.lowrank_gromov_wasserstein(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) + + +@pytest.mark.parametrize(("gamma_init"), ("rescale", "theory")) +def test_lowrank_wasserstein_gamma_init(gamma_init): + # Test lr sinkhorn with different init strategies + n_samples = 20 # nb samples + mu_s = np.array([0, 0]) + cov_s = np.array([[1, 0], [0, 1]]) + + X_s = ot.datasets.make_2D_samples_gauss(n_samples, mu_s, cov_s, random_state=1) + X_t = X_s[::-1].copy() + + a = ot.unif(n_samples) + b = ot.unif(n_samples) + + Q, R, g, log = ot.lowrank.lowrank_gromov_wasserstein(X_s, X_t, a, b, reg=0.1, gamma_init=gamma_init, log=True) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-04) + np.testing.assert_allclose(b, P.sum(0), atol=1e-04) + + +@pytest.skip_backend('tf') +def test_lowrank_gromov_wasserstein_backends(nx): + # Test low rank sinkhorn for different backends + n_samples = 20 # nb samples + mu_s = np.array([0, 0]) + cov_s = np.array([[1, 0], [0, 1]]) + + X_s = ot.datasets.make_2D_samples_gauss(n_samples, mu_s, cov_s, random_state=1) + X_t = X_s[::-1].copy() + + a = ot.unif(n_samples) + b = ot.unif(n_samples) + + ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) + + Q, R, g, log = ot.lowrank.lowrank_gromov_wasserstein(X_sb, X_tb, ab, bb, reg=0.1, log=True) + lazy_plan = log["lazy_plan"] + P = lazy_plan[:] + + np.testing.assert_allclose(ab, P.sum(1), atol=1e-04) + np.testing.assert_allclose(bb, P.sum(0), atol=1e-04) From cbc4d1fa88d33e66092936e54e2c867e161bce87 Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 15:31:55 +0100 Subject: [PATCH 32/36] add examples for Low Rank GW --- examples/others/plot_lowrank_GW.py | 130 +++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 examples/others/plot_lowrank_GW.py diff --git a/examples/others/plot_lowrank_GW.py b/examples/others/plot_lowrank_GW.py new file mode 100644 index 000000000..f9145b4fe --- /dev/null +++ b/examples/others/plot_lowrank_GW.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +""" +======================================== +Low rank Gromov-Wasterstein +======================================== + +This example illustrates the computation of Low Rank Gromov Wasserstein [66]. + +[66] Scetbon, M., Peyré, G. & Cuturi, M. (2022). +"Linear-Time GromovWasserstein Distances using Low Rank Couplings and Costs". +In International Conference on Machine Learning (ICML), 2022. +""" + +# Author: Laurène David +# +# License: MIT License +# +# sphinx_gallery_thumbnail_number = 3 + +#%% +import numpy as np +import matplotlib.pylab as pl +import ot.plot + +############################################################################## +# Generate data +# ------------- + +#%% parameters +n_samples = 200 + +# Generate 2D and 3D curves +theta = np.linspace(-4 * np.pi, 4 * np.pi, n_samples) +z = np.linspace(1, 2, n_samples) +r = z**2 + 1 +x = r * np.sin(theta) +y = r * np.cos(theta) + +# Source and target distribution +X = np.concatenate([x.reshape(-1, 1), z.reshape(-1, 1)], axis=1) +Y = np.concatenate([x.reshape(-1, 1), y.reshape(-1, 1), z.reshape(-1, 1)], axis=1) + + +############################################################################## +# Plot data +# ------------ + +#%% +# Plot the source and target samples +fig = pl.figure(1, figsize=(10, 4)) + +ax = fig.add_subplot(121) +ax.plot(X[:, 0], X[:, 1], color="blue", linewidth=6) +ax.tick_params(left=False, right=False, labelleft=False, + labelbottom=False, bottom=False) +ax.set_title("2D curve (source)") + +ax2 = fig.add_subplot(122, projection="3d") +ax2.plot(Y[:, 0], Y[:, 1], Y[:, 2], c='red', linewidth=6) +ax2.tick_params(left=False, right=False, labelleft=False, + labelbottom=False, bottom=False) +ax2.view_init(15, -50) +ax2.set_title("3D curve (target)") + +pl.tight_layout() +pl.show() + + +############################################################################## +# Entropic Gromov-Wasserstein +# ------------ + +#%% +# Solve entropic gw +C1 = ot.dist(X, X) +C1 = C1 / C1.max() + +C2 = ot.dist(Y, Y) +C2 = C2 / C2.max() + +reg = 5 * 1e-3 + +gw, log = ot.gromov.entropic_gromov_wasserstein( + C1, C2, tol=1e-3, epsilon=reg, + log=True, verbose=False) + +# Plot entropic gw +pl.figure(2) +pl.imshow(gw, interpolation="nearest", cmap="Greys", aspect="auto") +pl.title("Entropic Gromov-Wasserstein") +pl.show() + + +############################################################################## +# Low rank Gromov-Wasserstein +# ------------ +# %% +# Solve low rank gromov-wasserstein with different ranks +list_rank = [10, 50] +list_P_GW = [] +list_loss_GW = [] + +for rank in list_rank: + Q, R, g, log = ot.lowrank_gromov_wasserstein( + X, Y, reg=0, rank=rank, rescale_cost=True, seed_init=49, log=True + ) + + P = log["lazy_plan"][:] + loss = log["value"] + + list_P_GW.append(P) + list_loss_GW.append(np.round(loss, 2)) + + +# %% +# Plot low rank GW with different ranks +pl.figure(3, figsize=(10, 4)) + +pl.subplot(1, 2, 1) +pl.imshow(list_P_GW[0], interpolation="nearest", cmap="Greys", aspect="auto") +#pl.axis('off') +pl.title('Low rank GW (rank=10, loss={})'.format(list_loss_GW[0])) + +pl.subplot(1, 2, 2) +pl.imshow(list_P_GW[1], interpolation="nearest", cmap="Greys", aspect="auto") +#pl.axis('off') +pl.title('Low rank GW (rank=50, loss={})'.format(list_loss_GW[1])) + +pl.tight_layout() +pl.show() From 269ed3069679f3b30056df0e885af4aff375cad6 Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 15:52:03 +0100 Subject: [PATCH 33/36] fix __init__ --- ot/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ot/__init__.py b/ot/__init__.py index 5de992d67..a9c2c6aa3 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -71,10 +71,5 @@ 'factored_optimal_transport', 'solve', 'solve_gromov','solve_sample', 'smooth', 'stochastic', 'unbalanced', 'partial', 'regpath', 'solvers', 'binary_search_circle', 'wasserstein_circle', -<<<<<<< HEAD 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif', 'lowrank_sinkhorn', 'lowrank_gromov_wasserstein'] -======= - 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif', - 'lowrank_sinkhorn'] ->>>>>>> 63e44e5dfc51acf208ee088d65c980945c7da8b7 From 440f3a2170dbb896371d67ed9e15e89a494bb64b Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 15:54:21 +0100 Subject: [PATCH 34/36] change atol of lr backends --- test/test_lowrank.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_lowrank.py b/test/test_lowrank.py index ef2645a44..b7076484a 100644 --- a/test/test_lowrank.py +++ b/test/test_lowrank.py @@ -125,8 +125,8 @@ def test_lowrank_sinkhorn_backends(nx): lazy_plan = log["lazy_plan"] P = lazy_plan[:] - np.testing.assert_allclose(ab, P.sum(1), atol=1e-04) - np.testing.assert_allclose(bb, P.sum(0), atol=1e-04) + np.testing.assert_allclose(ab, P.sum(1), atol=1e-05) + np.testing.assert_allclose(bb, P.sum(0), atol=1e-05) def test__flat_product_operator(): From ab770ce39db2cdbaa773eca602043c4215e978de Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 8 Mar 2024 16:29:57 +0100 Subject: [PATCH 35/36] fix pep8 errors --- ot/lowrank.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ot/lowrank.py b/ot/lowrank.py index 1ad2c19fa..668c8d063 100644 --- a/ot/lowrank.py +++ b/ot/lowrank.py @@ -524,20 +524,20 @@ def _flat_product_operator(X, nx=None): n = X.shape[0] x1 = X[0, :][:, None] - X_flat = nx.dot(x1, x1.T).flatten()[:,None] - - for i in range(1,n): + X_flat = nx.dot(x1, x1.T).flatten()[:, None] + + for i in range(1, n): x = X[i, :][:, None] - x_out = nx.dot(x, x.T).flatten()[:,None] - X_flat= nx.concatenate((X_flat, x_out), axis=1) - + x_out = nx.dot(x, x.T).flatten()[:, None] + X_flat = nx.concatenate((X_flat, x_out), axis=1) + X_flat = X_flat.T return X_flat -def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, gamma_init="rescale", - rescale_cost=True, stopThr=1e-4, numItermax=1000, stopThr_dykstra=1e-3, +def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, gamma_init="rescale", + rescale_cost=True, stopThr=1e-4, numItermax=1000, stopThr_dykstra=1e-3, numItermax_dykstra=10000, seed_init=49, warn=True, warn_dykstra=False, log=False): r""" @@ -551,6 +551,7 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha \epsilon \cdot H((Q,R,g)) where : + - :math: `A` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the source domain - :math: `B` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the target domain - :math: `\mathcal{Q}_{A,B}` is quadratic objective function of the Gromov Wasserstein plan @@ -596,7 +597,7 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha Stop threshold on error (>0) in Dykstra warn : bool, optional if True, raises a warning if the low rank GW algorithm doesn't convergence. - warn_dykstra: bool, optional + warn_dykstra: bool, optional if True, raises a warning if the Dykstra algorithm doesn't convergence. log : bool, optional record log if True @@ -666,7 +667,7 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha # initial value of error err = 1 - + for ii in range(numItermax): Q_prev = Q R_prev = R From 96bad8905cbb9f5d19af235d8974da05f398d1f5 Mon Sep 17 00:00:00 2001 From: laudavid Date: Fri, 26 Apr 2024 15:27:05 +0200 Subject: [PATCH 36/36] modif for code review --- CONTRIBUTORS.md | 2 +- RELEASES.md | 1 + examples/others/plot_lowrank_GW.py | 77 ++++++++++++++++++++++------- ot/__init__.py | 4 +- ot/gromov/__init__.py | 4 +- ot/gromov/_lowrank.py | 78 +++++++++++++++++++----------- test/gromov/test_lowrank.py | 35 ++++++++------ 7 files changed, 136 insertions(+), 65 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 89c5be433..c185e18a7 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -50,7 +50,7 @@ The contributors to this library are: * [Ronak Mehta](https://ronakrm.github.io) (Efficient Discrete Multi Marginal Optimal Transport Regularization) * [Xizheng Yu](https://github.com/x12hengyu) (Efficient Discrete Multi Marginal Optimal Transport Regularization) * [Sonia Mazelet](https://github.com/SoniaMaz8) (Template based GNN layers) -* [Laurène David](https://github.com/laudavid) (Low rank sinkhorn) +* [Laurène David](https://github.com/laudavid) (Low rank sinkhorn, Low rank Gromov-Wasserstein samples) ## Acknowledgments diff --git a/RELEASES.md b/RELEASES.md index c7e3f598b..324642b55 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -4,6 +4,7 @@ #### New features + `ot.gromov._gw.solve_gromov_linesearch` now has an argument to specifify if the matrices are symmetric in which case the computation can be done faster. ++ Added support for [Low rank Gromov-Wasserstein](https://proceedings.mlr.press/v162/scetbon22b/scetbon22b.pdf) with `ot.gromov.lowrank_gromov_wasserstein_samples` (PR #614) #### Closed issues - Fixed an issue with cost correction for mismatched labels in `ot.da.BaseTransport` fit methods. This fix addresses the original issue introduced PR #587 (PR #593) diff --git a/examples/others/plot_lowrank_GW.py b/examples/others/plot_lowrank_GW.py index f9145b4fe..02fef6ded 100644 --- a/examples/others/plot_lowrank_GW.py +++ b/examples/others/plot_lowrank_GW.py @@ -1,12 +1,15 @@ # -*- coding: utf-8 -*- """ ======================================== -Low rank Gromov-Wasterstein +Low rank Gromov-Wasterstein between samples ======================================== -This example illustrates the computation of Low Rank Gromov Wasserstein [66]. +Comparaison between entropic Gromov-Wasserstein and Low Rank Gromov Wasserstein [67] +on two curves in 2D and 3D, both sampled with 200 points. -[66] Scetbon, M., Peyré, G. & Cuturi, M. (2022). +The squared Euclidean distance is considered as the ground cost for both samples. + +[67] Scetbon, M., Peyré, G. & Cuturi, M. (2022). "Linear-Time GromovWasserstein Distances using Low Rank Couplings and Costs". In International Conference on Machine Learning (ICML), 2022. """ @@ -21,6 +24,7 @@ import numpy as np import matplotlib.pylab as pl import ot.plot +import time ############################################################################## # Generate data @@ -71,45 +75,79 @@ # ------------ #%% -# Solve entropic gw -C1 = ot.dist(X, X) -C1 = C1 / C1.max() -C2 = ot.dist(Y, Y) -C2 = C2 / C2.max() +# Compute cost matrices +C1 = ot.dist(X, X, metric="sqeuclidean") +C2 = ot.dist(Y, Y, metric="sqeuclidean") + +# Scale cost matrices +r1 = C1.max() +r2 = C2.max() + +C1 = C1 / r1 +C2 = C2 / r2 + +# Solve entropic gw reg = 5 * 1e-3 +start = time.time() gw, log = ot.gromov.entropic_gromov_wasserstein( C1, C2, tol=1e-3, epsilon=reg, log=True, verbose=False) +end = time.time() +time_entropic = end - start + +entropic_gw_loss = np.round(log['gw_dist'], 3) + # Plot entropic gw pl.figure(2) -pl.imshow(gw, interpolation="nearest", cmap="Greys", aspect="auto") -pl.title("Entropic Gromov-Wasserstein") +pl.imshow(gw, interpolation="nearest", aspect="auto") +pl.title("Entropic Gromov-Wasserstein (loss={})".format(entropic_gw_loss)) pl.show() +############################################################################## +# Low rank squared euclidean cost matrices +# ------------ +# %% + +# Compute the low rank sqeuclidean cost decompositions +A1, A2 = ot.lowrank.compute_lr_sqeuclidean_matrix(X, X, rescale_cost=False) +B1, B2 = ot.lowrank.compute_lr_sqeuclidean_matrix(Y, Y, rescale_cost=False) + +# Scale the low rank cost matrices +A1, A2 = A1 / np.sqrt(r1), A2 / np.sqrt(r1) +B1, B2 = B1 / np.sqrt(r2), B2 / np.sqrt(r2) + + ############################################################################## # Low rank Gromov-Wasserstein # ------------ # %% + # Solve low rank gromov-wasserstein with different ranks list_rank = [10, 50] list_P_GW = [] list_loss_GW = [] +list_time_GW = [] for rank in list_rank: - Q, R, g, log = ot.lowrank_gromov_wasserstein( - X, Y, reg=0, rank=rank, rescale_cost=True, seed_init=49, log=True + start = time.time() + + Q, R, g, log = ot.lowrank_gromov_wasserstein_samples( + X, Y, reg=0, rank=rank, rescale_cost=False, cost_factorized_Xs=(A1, A2), + cost_factorized_Xt=(B1, B2), seed_init=49, numItermax=1000, log=True, stopThr=1e-6, ) + end = time.time() P = log["lazy_plan"][:] loss = log["value"] list_P_GW.append(P) - list_loss_GW.append(np.round(loss, 2)) + list_loss_GW.append(np.round(loss, 3)) + list_time_GW.append(end - start) # %% @@ -117,14 +155,19 @@ pl.figure(3, figsize=(10, 4)) pl.subplot(1, 2, 1) -pl.imshow(list_P_GW[0], interpolation="nearest", cmap="Greys", aspect="auto") -#pl.axis('off') +pl.imshow(list_P_GW[0], interpolation="nearest", aspect="auto") pl.title('Low rank GW (rank=10, loss={})'.format(list_loss_GW[0])) pl.subplot(1, 2, 2) -pl.imshow(list_P_GW[1], interpolation="nearest", cmap="Greys", aspect="auto") -#pl.axis('off') +pl.imshow(list_P_GW[1], interpolation="nearest", aspect="auto") pl.title('Low rank GW (rank=50, loss={})'.format(list_loss_GW[1])) pl.tight_layout() pl.show() + + +# %% +# Compare computation time between entropic GW and low rank GW +print("Entropic GW: {:.2f}s".format(time_entropic)) +print("Low rank GW (rank=10): {:.2f}s".format(list_time_GW[0])) +print("Low rank GW (rank=50): {:.2f}s".format(list_time_GW[1])) diff --git a/ot/__init__.py b/ot/__init__.py index f601e01c5..e7d6fbd56 100644 --- a/ot/__init__.py +++ b/ot/__init__.py @@ -50,7 +50,7 @@ sliced_wasserstein_sphere, sliced_wasserstein_sphere_unif) from .gromov import (gromov_wasserstein, gromov_wasserstein2, gromov_barycenters, fused_gromov_wasserstein, fused_gromov_wasserstein2, - lowrank_gromov_wasserstein) + lowrank_gromov_wasserstein_samples) from .weak import weak_optimal_transport from .factored import factored_optimal_transport from .solvers import solve, solve_gromov, solve_sample @@ -73,4 +73,4 @@ 'smooth', 'stochastic', 'unbalanced', 'partial', 'regpath', 'solvers', 'binary_search_circle', 'wasserstein_circle', 'semidiscrete_wasserstein2_unif_circle', 'sliced_wasserstein_sphere_unif', 'lowrank_sinkhorn', - 'lowrank_gromov_wasserstein'] + 'lowrank_gromov_wasserstein_samples'] diff --git a/ot/gromov/__init__.py b/ot/gromov/__init__.py index 4a12899b1..b33dafd32 100644 --- a/ot/gromov/__init__.py +++ b/ot/gromov/__init__.py @@ -47,7 +47,7 @@ fused_gromov_wasserstein_dictionary_learning, fused_gromov_wasserstein_linear_unmixing) -from ._lowrank import (_flat_product_operator, lowrank_gromov_wasserstein) +from ._lowrank import (_flat_product_operator, lowrank_gromov_wasserstein_samples) __all__ = ['init_matrix', 'tensor_product', 'gwloss', 'gwggrad', 'update_square_loss', @@ -66,4 +66,4 @@ 'entropic_semirelaxed_gromov_wasserstein2', 'entropic_semirelaxed_fused_gromov_wasserstein', 'entropic_semirelaxed_fused_gromov_wasserstein2', 'gromov_wasserstein_dictionary_learning', 'gromov_wasserstein_linear_unmixing', 'fused_gromov_wasserstein_dictionary_learning', - 'fused_gromov_wasserstein_linear_unmixing'] + 'fused_gromov_wasserstein_linear_unmixing', 'lowrank_gromov_wasserstein_samples'] diff --git a/ot/gromov/_lowrank.py b/ot/gromov/_lowrank.py index 8225cb12e..5bab15edc 100644 --- a/ot/gromov/_lowrank.py +++ b/ot/gromov/_lowrank.py @@ -58,14 +58,16 @@ def _flat_product_operator(X, nx=None): return X_flat -def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, gamma_init="rescale", - rescale_cost=True, stopThr=1e-4, numItermax=1000, stopThr_dykstra=1e-3, - numItermax_dykstra=10000, seed_init=49, warn=True, warn_dykstra=False, log=False): +def lowrank_gromov_wasserstein_samples(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha=1e-10, gamma_init="rescale", + rescale_cost=True, cost_factorized_Xs=None, cost_factorized_Xt=None, stopThr=1e-4, numItermax=1000, + stopThr_dykstra=1e-3, numItermax_dykstra=10000, seed_init=49, warn=True, warn_dykstra=False, log=False): r""" Solve the entropic regularization Gromov-Wasserstein transport problem under low-nonnegative rank constraints on the couplings and cost matrices. + Squared euclidean distance matrices are considered for the target and source distributions. + The function solves the following optimization problem: .. math:: @@ -74,29 +76,31 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha where : - - :math: `A` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the source domain - - :math: `B` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the target domain - - :math: `\mathcal{Q}_{A,B}` is quadratic objective function of the Gromov Wasserstein plan - - :math: `Q` and `R` are the low-rank matrix decomposition of the Gromov-Wasserstein plan - - :math: `g` is the weight vector for the low-rank decomposition of the Gromov-Wasserstein plan - - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target weights (histograms, both sum to 1) - - :math: `r` is the rank of the Gromov-Wasserstein plan - - :math: `\mathcal{C(a,b,r)}` are the low-rank couplings of the OT problem + - :math: `A` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the source domain. + - :math: `B` is the (`dim_a`, `dim_a`) square pairwise cost matrix of the target domain. + - :math: `\mathcal{Q}_{A,B}` is quadratic objective function of the Gromov Wasserstein plan. + - :math: `Q` and `R` are the low-rank matrix decomposition of the Gromov-Wasserstein plan. + - :math: `g` is the weight vector for the low-rank decomposition of the Gromov-Wasserstein plan. + - :math:`\mathbf{a}` and :math:`\mathbf{b}` are source and target weights (histograms, both sum to 1). + - :math: `r` is the rank of the Gromov-Wasserstein plan. + - :math: `\mathcal{C(a,b,r)}` are the low-rank couplings of the OT problem. - :math:`H((Q,R,g))` is the values of the three respective entropies evaluated for each term. Parameters ---------- - X_s : array-like, shape (n_samples_a, dim) - samples in the source domain - X_t : array-like, shape (n_samples_b, dim) - samples in the target domain - a : array-like, shape (n_samples_a,) - samples weights in the source domain - b : array-like, shape (n_samples_b,) - samples weights in the target domain + X_s : array-like, shape (n_samples_a, dim_Xs) + Samples in the source domain + X_t : array-like, shape (n_samples_b, dim_Xt) + Samples in the target domain + a : array-like, shape (n_samples_a,), optional + Samples weights in the source domain + If let to its default value None, uniform distribution is taken. + b : array-like, shape (n_samples_b,), optional + Samples weights in the target domain + If let to its default value None, uniform distribution is taken. reg : float, optional - Regularization term >0 + Regularization term >=0 rank : int, optional. Default is None. (>0) Nonnegative rank of the OT plan. If None, min(ns, nt) is considered. alpha : int, optional. Default is 1e-10. (>0 and <1/r) @@ -113,10 +117,20 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha Max number of iterations for Low Rank GW stopThr : float, optional. Default is 1e-4. Stop threshold on error (>0) for Low Rank GW + The error is the sum of Kullback Divergences computed for each low rank + coupling (Q, R and g) and scaled using gamma. numItermax_dykstra : int, optional. Default is 2000. Max number of iterations for the Dykstra algorithm stopThr_dykstra : float, optional. Default is 1e-7. Stop threshold on error (>0) in Dykstra + cost_factorized_Xs: tuple, optional. Default is None + Tuple with two pre-computed low rank decompositions (A1, A2) of the source cost + matrix. Both matrices should have a shape of (n_samples_a, dim_Xs + 2). + If None, the low rank cost matrices will be computed as sqeuclidean cost matrices. + cost_factorized_Xt: tuple, optional. Default is None + Tuple with two pre-computed low rank decompositions (B1, B2) of the target cost + matrix. Both matrices should have a shape of (n_samples_b, dim_Xt + 2). + If None, the low rank cost matrices will be computed as sqeuclidean cost matrices. warn : bool, optional if True, raises a warning if the low rank GW algorithm doesn't convergence. warn_dykstra: bool, optional @@ -170,9 +184,15 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha raise ValueError("alpha ({a}) should be smaller than 1/rank ({r}) for the Dykstra algorithm to converge.".format( a=alpha, r=1 / rank)) - # LR decomposition of the two sqeuclidean cost matrices - A1, A2 = compute_lr_sqeuclidean_matrix(X_s, X_s, rescale_cost, nx=nx) - B1, B2 = compute_lr_sqeuclidean_matrix(X_t, X_t, rescale_cost, nx=nx) + if cost_factorized_Xs is not None: + A1, A2 = cost_factorized_Xs + else: + A1, A2 = compute_lr_sqeuclidean_matrix(X_s, X_s, rescale_cost, nx=nx) + + if cost_factorized_Xt is not None: + B1, B2 = cost_factorized_Xt + else: + B1, B2 = compute_lr_sqeuclidean_matrix(X_t, X_t, rescale_cost, nx=nx) # Initial values for LR couplings (Q, R, g) with LOT Q, R, g = _init_lr_sinkhorn( @@ -274,11 +294,13 @@ def lowrank_gromov_wasserstein(X_s, X_t, a=None, b=None, reg=0, rank=None, alpha G = nx.dot(Q.T, G * diag_g) value_quad = c1 + nx.trace(G) / 2 - # Compute value with entropy reg (see "Section 3.2" in the paper) - reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q - reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g - reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R - value = value_quad + reg * (reg_Q + reg_g + reg_R) + if reg != 0: + reg_Q = nx.sum(Q * nx.log(Q + 1e-16)) # entropy for Q + reg_g = nx.sum(g * nx.log(g + 1e-16)) # entropy for g + reg_R = nx.sum(R * nx.log(R + 1e-16)) # entropy for R + value = value_quad + reg * (reg_Q + reg_g + reg_R) + else: + value = value_quad if log: dict_log = dict() diff --git a/test/gromov/test_lowrank.py b/test/gromov/test_lowrank.py index 752130d0b..befc5c835 100644 --- a/test/gromov/test_lowrank.py +++ b/test/gromov/test_lowrank.py @@ -23,7 +23,7 @@ def test__flat_product_operator(): np.testing.assert_allclose(cost**2, np.dot(A1_, A2_.T), atol=1e-05) -def test_lowrank_gromov_wasserstein(): +def test_lowrank_gromov_wasserstein_samples(): # test low rank gromov wasserstein n_samples = 20 # nb samples mu_s = np.array([0, 0]) @@ -35,7 +35,7 @@ def test_lowrank_gromov_wasserstein(): a = ot.unif(n_samples) b = ot.unif(n_samples) - Q, R, g, log = ot.gromov.lowrank_gromov_wasserstein(X_s, X_t, a, b, reg=0.1, log=True, rescale_cost=False) + Q, R, g, log = ot.gromov.lowrank_gromov_wasserstein_samples(X_s, X_t, a, b, reg=0.1, log=True, rescale_cost=False) P = log["lazy_plan"][:] # check constraints for P @@ -48,19 +48,19 @@ def test_lowrank_gromov_wasserstein(): # check warn parameter when low rank GW algorithm doesn't converge with pytest.warns(UserWarning): - ot.gromov.lowrank_gromov_wasserstein( + ot.gromov.lowrank_gromov_wasserstein_samples( X_s, X_t, a, b, reg=0.1, stopThr=0, numItermax=1, warn=True, warn_dykstra=False ) # check warn parameter when Dykstra algorithm doesn't converge with pytest.warns(UserWarning): - ot.gromov.lowrank_gromov_wasserstein( + ot.gromov.lowrank_gromov_wasserstein_samples( X_s, X_t, a, b, reg=0.1, stopThr_dykstra=0, numItermax_dykstra=1, warn=False, warn_dykstra=True ) @pytest.mark.parametrize(("alpha, rank"), ((0.8, 2), (0.5, 3), (0.2, 6), (0.1, -1))) -def test_lowrank_gromov_wasserstein_alpha_error(alpha, rank): +def test_lowrank_gromov_wasserstein_samples_alpha_error(alpha, rank): # Test warning for value of alpha and rank n_samples = 20 # nb samples mu_s = np.array([0, 0]) @@ -73,11 +73,11 @@ def test_lowrank_gromov_wasserstein_alpha_error(alpha, rank): b = ot.unif(n_samples) with pytest.raises(ValueError): - ot.gromov.lowrank_gromov_wasserstein(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) + ot.gromov.lowrank_gromov_wasserstein_samples(X_s, X_t, a, b, reg=0.1, rank=rank, alpha=alpha, warn=False) -@pytest.mark.parametrize(("gamma_init"), ("rescale", "theory")) -def test_lowrank_wasserstein_gamma_init(gamma_init): +@pytest.mark.parametrize(("gamma_init"), ("rescale", "theory", "other")) +def test_lowrank_wasserstein_samples_gamma_init(gamma_init): # Test lr sinkhorn with different init strategies n_samples = 20 # nb samples mu_s = np.array([0, 0]) @@ -89,16 +89,21 @@ def test_lowrank_wasserstein_gamma_init(gamma_init): a = ot.unif(n_samples) b = ot.unif(n_samples) - Q, R, g, log = ot.gromov.lowrank_gromov_wasserstein(X_s, X_t, a, b, reg=0.1, gamma_init=gamma_init, log=True) - P = log["lazy_plan"][:] + if gamma_init not in ["rescale", "theory"]: + with pytest.raises(NotImplementedError): + ot.gromov.lowrank_gromov_wasserstein_samples(X_s, X_t, a, b, reg=0.1, gamma_init=gamma_init, log=True) - # check constraints for P - np.testing.assert_allclose(a, P.sum(1), atol=1e-04) - np.testing.assert_allclose(b, P.sum(0), atol=1e-04) + else: + Q, R, g, log = ot.gromov.lowrank_gromov_wasserstein_samples(X_s, X_t, a, b, reg=0.1, gamma_init=gamma_init, log=True) + P = log["lazy_plan"][:] + + # check constraints for P + np.testing.assert_allclose(a, P.sum(1), atol=1e-04) + np.testing.assert_allclose(b, P.sum(0), atol=1e-04) @pytest.skip_backend('tf') -def test_lowrank_gromov_wasserstein_backends(nx): +def test_lowrank_gromov_wasserstein_samples_backends(nx): # Test low rank sinkhorn for different backends n_samples = 20 # nb samples mu_s = np.array([0, 0]) @@ -112,7 +117,7 @@ def test_lowrank_gromov_wasserstein_backends(nx): ab, bb, X_sb, X_tb = nx.from_numpy(a, b, X_s, X_t) - Q, R, g, log = ot.gromov.lowrank_gromov_wasserstein(X_sb, X_tb, ab, bb, reg=0.1, log=True) + Q, R, g, log = ot.gromov.lowrank_gromov_wasserstein_samples(X_sb, X_tb, ab, bb, reg=0.1, log=True) lazy_plan = log["lazy_plan"] P = lazy_plan[:]