From 5b4fb280ff4eacb51652838ca864edfc78ef5bbc Mon Sep 17 00:00:00 2001 From: Fabian Pedregosa Date: Thu, 4 Oct 2018 09:57:57 -0700 Subject: [PATCH] 0.5 release --- copt/__init__.py | 2 +- copt/datasets.py | 2 +- copt/frank_wolfe.py | 35 +++++++++++++---------------------- copt/utils.py | 11 +++++++---- 4 files changed, 22 insertions(+), 28 deletions(-) diff --git a/copt/__init__.py b/copt/__init__.py index 436c64c3..5376f6dd 100755 --- a/copt/__init__.py +++ b/copt/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.4.0-dev' +__version__ = '0.5.0' from .proxgrad import minimize_PGD, minimize_APGD from .splitting import minimize_TOS, minimize_PDHG diff --git a/copt/datasets.py b/copt/datasets.py index 14d2753c..4222d344 100755 --- a/copt/datasets.py +++ b/copt/datasets.py @@ -294,7 +294,7 @@ def load_kdd12(md5_check=True, verbose=0): if not os.path.exists(DATA_DIR): os.makedirs(DATA_DIR) if not os.path.exists(file_path): - print('URL dataset is not present in data folder. Downloading it ...') + print('KDD12 dataset is not present in data folder. Downloading it ...') url = 'https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/kdd12.bz2' urllib.request.urlretrieve(url, file_path) print('Finished downloading') diff --git a/copt/frank_wolfe.py b/copt/frank_wolfe.py index bfa3d974..4ae359af 100644 --- a/copt/frank_wolfe.py +++ b/copt/frank_wolfe.py @@ -8,25 +8,6 @@ from . import utils -def _backtrack( - f_t, f_grad, x_t, d_t, g_t, L_t, - gamma_max=1, ratio_increase=2., ratio_decrease=0.999, - max_iter=100): - # could be included inside minimize_FW - d2_t = splinalg.norm(d_t) ** 2 - for i in range(max_iter): - step_size = min(g_t / (d2_t * L_t), gamma_max) - rhs = f_t - step_size * g_t + 0.5 * (step_size**2) * L_t * d2_t - f_next, grad_next = f_grad(x_t + step_size * d_t) - if f_next <= rhs: - if i == 0: - L_t *= ratio_decrease - break - else: - L_t *= ratio_increase - return step_size, L_t, f_next, grad_next - - def minimize_FW( f_grad, lmo, x0, L=None, max_iter=1000, tol=1e-12, backtracking=True, callback=None, verbose=0): @@ -55,11 +36,21 @@ def minimize_FW( g_t = g_t[0] if g_t <= tol: break + d2_t = splinalg.norm(d_t) ** 2 if backtracking: - step_size, L_t, f_next, grad_next = _backtrack( - f_t, f_grad, x_t, d_t, g_t, L_t) + ratio_decrease = 0.999 + ratio_increase = 2 + for i in range(max_iter): + step_size = min(g_t / (d2_t * L_t), 1) + rhs = f_t - step_size * g_t + 0.5 * (step_size**2) * L_t * d2_t + f_next, grad_next = f_grad(x_t + step_size * d_t) + if f_next <= rhs + 1e-6: + if i == 0: + L_t *= ratio_decrease + break + else: + L_t *= ratio_increase else: - d2_t = splinalg.norm(d_t) ** 2 step_size = min(g_t / (d2_t * L_t), 1) f_next, grad_next = f_grad(x_t + step_size * d_t) x_t += step_size * d_t diff --git a/copt/utils.py b/copt/utils.py index e1850267..bbd0b468 100755 --- a/copt/utils.py +++ b/copt/utils.py @@ -32,13 +32,15 @@ def __call__(self, x): def init_lipschitz(f_grad, x0): L0 = 1e-3 f0, grad0 = f_grad(x0) - if sparse.issparse(grad0): + if sparse.issparse(grad0) and not sparse.issparse(x0): x0 = sparse.csc_matrix(x0).T - elif sparse.issparse(x0): + elif sparse.issparse(x0) and not sparse.issparse(grad0): grad0 = sparse.csc_matrix(grad0).T x_tilde = x0 - (1./L0)*grad0 f_tilde = f_grad(x_tilde)[0] - while f_tilde > f0: + for _ in range(100): + if f_tilde <= f0: + break L0 *= 10 x_tilde = x0 - (1./L0)*grad0 f_tilde = f_grad(x_tilde)[0] @@ -182,6 +184,7 @@ def lipschitz(self): s = splinalg.svds(self.A, k=1, return_singular_vectors=False)[0] return (s * s) / self.A.shape[0] + self.alpha + class HuberLoss: """Huber loss""" def __init__(self, A, b, alpha=0, delta=1): @@ -205,7 +208,7 @@ def f_grad(self, x, return_gradient=True): grad = self.A[idx].T.dot(z[idx]) / self.A.shape[0] + self.alpha * x.T grad = np.asarray(grad) grad += self.A[~idx].T.dot(self.delta * np.sign(z[~idx]))/ self.A.shape[0] - return loss, grad + return loss, np.asarray(grad).ravel() def lipschitz(self): s = splinalg.svds(self.A, k=1, return_singular_vectors=False)[0]