Merge pull request #824 from Epistimio/release-v0.2.3rc1

Release candidate v0.2.3rc1

README.rst

@@ -118,7 +118,7 @@ If you use Oríon for published work, please cite our work using the following b
 
 .. code-block:: bibtex
 
-   @software{xavier_bouthillier_2022_0_2_2,
+   @software{xavier_bouthillier_2022_0_2_3,
      author       = {Xavier Bouthillier and
                      Christos Tsirigotis and
                      François Corneau-Tremblay and
@@ -143,10 +143,10 @@ If you use Oríon for published work, please cite our work using the following b
                      Pascal Lamblin and
                      Christopher Beckham},
      title        = {{Epistimio/orion: Asynchronous Distributed Hyperparameter Optimization}},
-     month        = feb,
+     month        = mar,
      year         = 2022,
      publisher    = {Zenodo},
-     version      = {v0.2.2},
+     version      = {v0.2.3},
      doi          = {10.5281/zenodo.3478592},
      url          = {https://doi.org/10.5281/zenodo.3478592}
    }

ROADMAP.md

@@ -1,14 +1,20 @@
 # Roadmap
-Last update Feb 11th, 2022
+Last update March 7th, 2022
 
 ## Next releases - Short-Term
 
-### v0.2.3
+### v0.2.4
 
 - [DEBH](https://arxiv.org/abs/2105.09821)
 - [HEBO](https://github.com/huawei-noah/HEBO/tree/master/HEBO/archived_submissions/hebo)
 - [BOHB](https://ml.informatik.uni-freiburg.de/papers/18-ICML-BOHB.pdf)
+- [Nevergrad](https://github.com/facebookresearch/nevergrad)
+- [Ax](https://ax.dev/)
+- [MOFA](https://github.com/Epistimio/orion.algo.mofa)
+- [PB2](https://github.com/Epistimio/orion.algo.pb2)
 - Integration with Hydra
+- Integration with [sample-space](https://github.com/Epistimio/sample-space) and
+  [ConfigSpace](https://automl.github.io/ConfigSpace/master/)
 
 ## Next releases - Mid-Term
 

conda/ci_build.sh

@@ -6,6 +6,7 @@ export PATH="$HOME/miniconda/bin:$PATH"
 hash -r
 conda config --set always_yes yes --set changeps1 no
 conda config --add channels conda-forge
+conda config --add channels mila-iqia
 conda config --set channel_priority strict
 
 pip uninstall -y setuptools

conda/meta.yaml

@@ -33,6 +33,7 @@ requirements:
     - requests
     - pandas
     - falcon
+    - falcon-cors
     - gunicorn
     - scikit-learn
     - psutil

setup.py

@@ -11,7 +11,7 @@
 repo_root = os.path.dirname(os.path.abspath(__file__))
 
 
-tests_require = ["pytest>=3.0.0", "scikit-learn"]
+tests_require = ["pytest>=3.0.0", "scikit-learn", "ptera>=1.1.0"]
 
 
 packages = [  # Packages must be sorted alphabetically to ease maintenance and merges.
@@ -88,6 +88,7 @@
         "pandas",
         "gunicorn",
         "falcon",
+        "falcon-cors",
         "scikit-learn",
         "psutil",
         "joblib",

src/orion/client/runner.py

@@ -40,12 +40,24 @@ def __init__(self):
         self.handlers = dict()
         self.start = 0
         self.delayed = 0
+        self.signal_installed = False
 
     def __enter__(self):
         """Override the signal handlers with our delayed handler"""
         self.signal_received = False
-        self.handlers[signal.SIGINT] = signal.signal(signal.SIGINT, self.handler)
-        self.handlers[signal.SIGTERM] = signal.signal(signal.SIGTERM, self.handler)
+
+        try:
+            self.handlers[signal.SIGINT] = signal.signal(signal.SIGINT, self.handler)
+            self.handlers[signal.SIGTERM] = signal.signal(signal.SIGTERM, self.handler)
+            self.signal_installed = True
+
+        except ValueError:  # ValueError: signal only works in main thread
+            log.warning(
+                "SIGINT/SIGTERM protection hooks could not be installed because "
+                "Runner is executing inside a thread/subprocess, results could get lost "
+                "on interruptions"
+            )
+
         return self
 
     def handler(self, sig, frame):
@@ -65,6 +77,9 @@ def handler(self, sig, frame):
 
     def restore_handlers(self):
         """Restore old signal handlers"""
+        if not self.signal_installed:
+            return
+
         signal.signal(signal.SIGINT, self.handlers[signal.SIGINT])
         signal.signal(signal.SIGTERM, self.handlers[signal.SIGTERM])
 
@@ -268,7 +283,7 @@ def run(self):
     def should_sample(self):
         """Check if more trials could be generated"""
 
-        if self.is_broken or self.is_done:
+        if self.free_worker <= 0 or (self.is_broken or self.is_done):
             return 0
 
         pending = len(self.pending_trials) + self.trials
@@ -317,8 +332,9 @@ def gather(self):
 
         to_be_raised = None
         log.debug(f"Gathered new results {len(results)}")
-
         # register the results
+        # NOTE: For Ptera instrumentation
+        trials = 0  # pylint:disable=unused-variable
         for result in results:
             trial = self.pending_trials.pop(result.future)
 
@@ -327,6 +343,8 @@ def gather(self):
                     # NB: observe release the trial already
                     self.client.observe(trial, result.value)
                     self.trials += 1
+                    # NOTE: For Ptera instrumentation
+                    trials = self.trials  # pylint:disable=unused-variable
                 except InvalidResult as exception:
                     # stop the optimization process if we received `InvalidResult`
                     # as all the trials are assumed to be returning those
@@ -398,15 +416,19 @@ def _suggest_trials(self, count):
 
             # non critical errors
             except WaitingForTrials:
+                log.debug("Runner cannot sample because WaitingForTrials")
                 break
 
             except ReservationRaceCondition:
+                log.debug("Runner cannot sample because ReservationRaceCondition")
                 break
 
             except LockAcquisitionTimeout:
+                log.debug("Runner cannot sample because LockAcquisitionTimeout")
                 break
 
             except CompletedExperiment:
+                log.debug("Runner cannot sample because CompletedExperiment")
                 break
 
         return trials

src/orion/core/__init__.py

@@ -32,10 +32,10 @@
 __descr__ = "Asynchronous [black-box] Optimization"
 __version__ = VERSIONS["version"]
 __license__ = "BSD-3-Clause"
-__author__ = u"Epistímio"
-__author_short__ = u"Epistímio"
+__author__ = "Epistímio"
+__author_short__ = "Epistímio"
 __author_email__ = "[email protected]"
-__copyright__ = u"2017-2022, Epistímio"
+__copyright__ = "2017-2022, Epistímio"
 __url__ = "https://github.com/epistimio/orion"
 
 DIRS = AppDirs(__name__, __author_short__)
@@ -55,6 +55,7 @@ def define_config():
     define_experiment_config(config)
     define_worker_config(config)
     define_evc_config(config)
+    define_frontends_uri_config(config)
 
     config.add_option(
         "user_script_config",
@@ -73,6 +74,31 @@ def define_config():
     return config
 
 
+def define_frontends_uri_config(config):
+    """Create and define the field of frontends URI configuration."""
+
+    def parse_frontends_uri(data):
+        # Expect either a list of strings (URLs),
+        # or a string as comma-separated list of URLs
+        if isinstance(data, list):
+            return data
+        elif isinstance(data, str):
+            return [piece.strip() for piece in data.split(",")]
+        else:
+            raise RuntimeError(
+                f"frontends_uri: expected either a list of strings (URLs), "
+                f"or a string as comma-separated list of URLs, got {data}"
+            )
+
+    config.add_option(
+        "frontends_uri",
+        option_type=parse_frontends_uri,
+        default=[],
+        env_var="ORION_WEBAPI_FRONTENDS_URI",
+        help="List of frontends addresses allowed to send requests to Orion server.",
+    )
+
+
 def define_storage_config(config):
     """Create and define the fields of the storage configuration."""
     storage_config = Configuration()

src/orion/core/io/resolve_config.py

@@ -213,8 +213,9 @@ def fetch_config(args):
 
         local_config = unflatten(local_config)
 
+        backward_keys = ["storage", "experiment", "worker", "evc"]
         # For backward compatibility
-        for key in ["storage", "experiment", "worker", "evc"]:
+        for key in backward_keys:
             subkeys = list(global_config[key].keys())
 
             # Arguments that are only supported locally
@@ -241,6 +242,11 @@ def fetch_config(args):
                     local_config.setdefault(key, {})
                     local_config[key][subkey] = value
 
+        # Keep other keys parsed from config file
+        for key in tmp_config.keys():
+            if key not in backward_keys:
+                local_config[key] = tmp_config[key]
+
     return local_config
 
 

src/orion/executor/dask_backend.py

@@ -52,6 +52,16 @@ def successful(self):
 
 
 class Dask(BaseExecutor):
+    """Wrapper around the dask client.
+
+    .. warning::
+
+       The Dask executor can be pickled and used inside a subprocess,
+       the pickled client will use the main client that was spawned in the main process,
+       but you cannot spawn clients inside a subprocess.
+
+    """
+
     def __init__(self, n_workers=-1, client=None, **config):
         super(Dask, self).__init__(n_workers=n_workers)
 

src/orion/executor/multiprocess_backend.py

@@ -156,6 +156,10 @@ class PoolExecutor(BaseExecutor):
     backend: str
         Pool backend to use; thread or multiprocess, defaults to multiprocess
 
+    .. warning::
+
+       Pickling of the executor is not supported, see Dask for a backend that supports it
+
     """
 
     BACKENDS = dict(
@@ -173,6 +177,12 @@ def __init__(self, n_workers=-1, backend="multiprocess", **kwargs):
 
         self.pool = PoolExecutor.BACKENDS.get(backend, ThreadPool)(n_workers)
 
+    def __setstate__(self, state):
+        self.pool = state["pool"]
+
+    def __getstate__(self):
+        return dict(pool=self.pool)
+
     def __enter__(self):
         return self
 
@@ -188,13 +198,6 @@ def close(self):
         if hasattr(self, "pool"):
             self.pool.shutdown()
 
-    def __getstate__(self):
-        state = super(PoolExecutor, self).__getstate__()
-        return state
-
-    def __setstate__(self, state):
-        super(PoolExecutor, self).__setstate__(state)
-
     def submit(self, function, *args, **kwargs):
         try:
             return self._submit_cloudpickle(function, *args, **kwargs)

src/orion/serving/webapi.py

@@ -8,14 +8,74 @@
 
 """
 
+import logging
+
 import falcon
+from falcon_cors import CORS, CORSMiddleware
 
 from orion.serving.experiments_resource import ExperimentsResource
 from orion.serving.plots_resources import PlotsResource
 from orion.serving.runtime import RuntimeResource
 from orion.serving.trials_resource import TrialsResource
 from orion.storage.base import setup_storage
 
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+class MyCORSMiddleware(CORSMiddleware):
+    """Subclass of falcon-cors CORSMiddleware class.
+
+    Generate a HTTP 403 Forbidden response if request sender is not allowed
+    to access requested content.
+
+    Default middleware just prints a message in server side
+    (e.g. "Aborting response due to origin not allowed"), but still
+    sends content, so, a client ignoring headers might still access
+    data even if not allowed.
+
+    CORS middleware role is to add necessary "access-control-" headers to
+    response to mark it as allowed. So, a response lacking expected headers
+    after call to parent method `process_ressource()` can be considered
+    to not be delivered to request sender.
+
+    More info about CORS:
+    - https://developer.mozilla.org/fr/docs/Web/HTTP/CORS
+    - https://fr.wikipedia.org/wiki/Cross-origin_resource_sharing
+    """
+
+    def process_resource(self, req, resp, resource, *args):
+        """Generate a 403 Forbidden response if response is not allowed."""
+
+        cors_resp_headers_before = [
+            header
+            for header in resp.headers
+            if header.lower().startswith("access-control-")
+        ]
+        assert not cors_resp_headers_before, cors_resp_headers_before
+
+        super().process_resource(req, resp, resource, *args)
+
+        # We then verify if some access control headers were added to response.
+        # If not, reponse is not allowed.
+        # Special case: if request did not have an origin, it was certainly sent from
+        # a browser (ie. not another server), so CORS is not relevant.
+        cors_resp_headers_after = [
+            header
+            for header in resp.headers
+            if header.lower().startswith("access-control-")
+        ]
+        if not cors_resp_headers_after and req.get_header("origin"):
+            raise falcon.HTTPForbidden()
+
+
+class MyCORS(CORS):
+    """Subclass of falcon-cors CORS class to return a custom middleware."""
+
+    @property
+    def middleware(self):
+        return MyCORSMiddleware(self)
+
 
 class WebApi(falcon.API):
     """
@@ -24,7 +84,27 @@ class WebApi(falcon.API):
     """
 
     def __init__(self, config=None):
-        super(WebApi, self).__init__()
+        # By default, server will reject requests coming from a server
+        # with different origin. E.g., if server is hosted at
+        # http://myorionserver.com, it won't accept an API call
+        # coming from a server not hosted at same address
+        # (e.g. a local installation at http://localhost)
+        # This is a Cross-Origin Resource Sharing (CORS) security:
+        # https://developer.mozilla.org/fr/docs/Web/HTTP/CORS
+        # To make server accept CORS requests, we need to use
+        # falcon-cors package: https://github.com/lwcolton/falcon-cors
+        frontends_uri = (
+            config["frontends_uri"]
+            if "frontends_uri" in config
+            else ["http://localhost:3000"]
+        )
+        logger.info(
+            "allowed frontends: {}".format(
+                ", ".join(frontends_uri) if frontends_uri else "(none)"
+            )
+        )
+        cors = MyCORS(allow_origins_list=frontends_uri)
+        super(WebApi, self).__init__(middleware=[cors.middleware])
         self.config = config
 
         setup_storage(config.get("storage"))