Initial commit

.ci/installBackendsDependencies.sh

@@ -0,0 +1,34 @@
+apt-get update
+
+echo "dependencies for GPyopt"
+source ${BASH_SOURCE%/*}/installGPyOptDependencies.sh
+
+echo "dependencies for rbfopt"
+source ${BASH_SOURCE%/*}/installRBFOptDependencies.sh
+
+echo "dependencies for SMAC3"
+source ${BASH_SOURCE%/*}/installSMAC3Dependencies.sh
+
+if pip3 show ecabc; then :;
+else
+  git clone --depth=1 --branch _args_fix https://github.com/KOLANICH/ecabc.git
+  pip install --user --upgrade --pre ./ecabc
+fi;
+
+if [ -f $PYTHON_MODULES_DIR/hyperband.py ] ; then :;
+else
+  curl -O https://raw.githubusercontent.com/zygmuntz/hyperband/master/hyperband.py
+  2to3 -wn hyperband.py
+  mv hyperband.py $PYTHON_MODULES_DIR/
+fi;
+
+if [ -f $PYTHON_MODULES_DIR/diffevo.py ] ; then :;
+else
+  curl -O https://raw.githubusercontent.com/tiagoCuervo/EvoFuzzy/4cbfce4a432fd162d6f30017c8de0477b29e5f42/diffevo.py
+  2to3 -wn diffevo.py
+  mv diffevo.py $PYTHON_MODULES_DIR/
+fi;
+
+
+# RoBo -> george
+pip install --upgrade git+https://github.com/yaml/pyyaml.git

.ci/installCython.sh

@@ -0,0 +1,6 @@
+if pip3 show cython ; then :;
+else
+  apt-get -y install swig gcc g++;
+  git clone --depth=1 https://github.com/cython/cython.git;
+  pip3 install --upgrade --user --pre ./cython;
+fi;

.ci/installGPyOptDependencies.sh

@@ -0,0 +1,7 @@
+if pip3 show smac; then :;
+else
+  source ${BASH_SOURCE%/*}/installCython.sh
+  git clone --depth=1 https://github.com/SheffieldML/GPy.git
+  find ./GPy -name '*.pyx' -exec cython {} \;
+  pip3 install --upgrade --user --pre ./GPy
+fi;

.ci/installRBFOptDependencies.sh

@@ -0,0 +1,15 @@
+echo "dependencies for rbfopt";
+if [ -x $EXECUTABLE_DEPENDENCIES_DIR/bonmin ] && [ -x $EXECUTABLE_DEPENDENCIES_DIR/ipopt ]; then :;
+else
+  apt-get -y install p7zip-full;
+  curl -O https://ampl.com/dl/open/bonmin/bonmin-linux64.zip -O https://ampl.com/dl/open/ipopt/ipopt-linux64.zip;
+  7za x bonmin-linux64.zip;
+  7za x -y ipopt-linux64.zip;
+  mv ./bonmin ./ipopt $EXECUTABLE_DEPENDENCIES_DIR/;
+fi;
+if pip3 show pyutilib && pip3 show pyomo; then :;
+else
+  git clone --depth=1 https://github.com/PyUtilib/pyutilib.git;
+  git clone --depth=1 https://github.com/Pyomo/pyomo.git;
+  pip3 install --user --upgrade --pre ./pyutilib ./pyomo;
+fi;

.ci/installSMAC3Dependencies.sh

@@ -0,0 +1,7 @@
+apt-get -y install python3-pybind11;
+if pip3 show smac; then :;
+else
+  apt-get -y install pybind11-dev;
+  git clone --depth=1 --branch development https://github.com/automl/SMAC3.git;
+  pip3 install --user --upgrade --pre ./SMAC3;
+fi;

.ci/pythonStdlibFixes.sh

@@ -0,0 +1,12 @@
+if $( python -c "import sys;sys.exit(int(not (sys.version_info < (3, 5)) ))" ); then
+ curl -O https://raw.githubusercontent.com/python/cpython/3.6/Lib/typing.py;
+ curl -O https://raw.githubusercontent.com/python/cpython/3.5/Lib/linecache.py;
+ curl -O https://raw.githubusercontent.com/python/cpython/3.5/Lib/traceback.py;
+ #curl -O https://raw.githubusercontent.com/python/cpython/3.5/Lib/importlib/abc.py;
+ #curl -O https://raw.githubusercontent.com/python/cpython/3.5/Lib/importlib/_bootstrap_external.py;
+ mv ./typing.py ./linecache.py ./traceback.py $PYTHON_MODULES_DIR/
+fi;
+if $( python -c "import sys;sys.exit(int(not (sys.version_info < (3, 6)) ))" ); then
+curl -O https://raw.githubusercontent.com/python/cpython/3.7/Lib/enum.py;
+  mv ./enum.py $PYTHON_MODULES_DIR/
+fi;

.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+charset=utf-8
+indent_style = tab
+indent_size = 4
+insert_final_newline = false

.github/.templateMarker

@@ -0,0 +1 @@
+KOLANICH/python_project_boilerplate.py

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-type: "all"

.github/workflows/CI.yml

@@ -0,0 +1,15 @@
+name: CI
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: typical python workflow
+        uses: KOLANICH-GHActions/typical-python-workflow@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -0,0 +1,8 @@
+__pycache__
+*.py[co]
+/*.egg-info
+/build
+/dist
+/hyperband.py
+/shac
+/.eggs

.gitlab-ci.yml

@@ -0,0 +1,102 @@
+#image: pypy:latest
+image: registry.gitlab.com/kolanich/fixed_python:latest
+
+stages:
+  - dependencies
+  - build
+  - test
+  - tooling
+
+variables:
+  GIT_DEPTH: "1"
+  PYTHONUSERBASE: ${CI_PROJECT_DIR}/python_user_packages
+
+dependencies:
+  tags:
+    - shared
+  stage: dependencies
+  before_script:
+    - export PYTHON_MODULES_DIR=${PYTHONUSERBASE}/lib/python3.7/site-packages
+    - export EXECUTABLE_DEPENDENCIES_DIR=${PYTHONUSERBASE}/bin
+    - export PATH="$PATH:$EXECUTABLE_DEPENDENCIES_DIR" # don't move into `variables` any of them, it is unordered
+    #- python -c "import hyperband"
+  script:
+    - mkdir -p $EXECUTABLE_DEPENDENCIES_DIR $PYTHON_MODULES_DIR
+    - source ./.ci/installBackendsDependencies.sh
+
+  cache:
+    key: deps
+    paths:
+      - $PYTHONUSERBASE
+      - $EXECUTABLE_DEPENDENCIES_DIR
+
+build:
+  tags:
+    - shared
+  stage: build
+
+  before_script:
+    - export PYTHON_MODULES_DIR=${PYTHONUSERBASE}/lib/python3.7
+    - export EXECUTABLE_DEPENDENCIES_DIR=${PYTHONUSERBASE}/bin
+    - export PATH="$PATH:$EXECUTABLE_DEPENDENCIES_DIR" # don't move into `variables` any of them, it is unordered
+    - source ./.ci/installBackendsDependencies.sh # everything should be built at this moment, needed only to install the stuff installed by apt-get
+
+  script:
+    - python3 setup.py bdist_wheel
+    - mv ./dist/*.whl ./dist/UniOpt-0.CI-py3-none-any.whl
+    - pip3 install --user --upgrade --pre -e ./[hyperopt,hyperengine,SKOpt,SMAC,BeeColony,optunity,Yabox,PySHAC,RBFOpt,Bayessian,GPyOpt,SOpt,pySOT,BayTune,RoBo] #https://github.com/pypa/pip/issues/5903
+    - coverage run --source=UniOpt -m pytest --junitxml=./rspec.xml ./tests/tests.py
+    - coverage report -m
+    - coverage xml
+  cache:
+    key: deps
+    paths:
+      - $PYTHONUSERBASE
+
+  artifacts:
+    paths:
+      - dist
+    reports:
+      junit: ./rspec.xml
+      cobertura: ./coverage.xml
+
+checks:
+  stage: tooling
+  tags:
+    - shared
+  image: docker:latest
+  variables:
+    DOCKER_DRIVER: overlay2
+  allow_failure: true
+  services:
+    - docker:dind
+  script:
+    - docker run --env SAST_CONFIDENCE_LEVEL=5 --volume "$PWD:/code" --volume /var/run/docker.sock:/var/run/docker.sock "registry.gitlab.com/gitlab-org/security-products/sast:latest" /app/bin/run /code
+    #- docker run --env SOURCE_CODE="$PWD" --env CODECLIMATE_VERSION="latest" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock "registry.gitlab.com/gitlab-org/security-products/codequality:latest" /code
+    #- docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" --volume "$PWD:/code" --volume /var/run/docker.sock:/var/run/docker.sock "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:latest" /code
+
+  artifacts:
+    reports:
+      #codequality: gl-code-quality-report.json
+      sast: gl-sast-report.json
+      #dependency_scanning: gl-dependency-scanning-report.json 
+
+
+pages:
+  stage: tooling
+  tags:
+    - shared
+  image: alpine:latest
+  allow_failure: true
+  before_script:
+    - apk update
+    - apk add doxygen
+    - apk add ttf-freefont graphviz
+  script:
+    - doxygen ./Doxyfile
+    - mv ./docs/html ./public
+  artifacts:
+    paths:
+      - public
+  only:
+    - master

.travis.yml

@@ -0,0 +1,43 @@
+sudo: false
+dist: trusty
+language: python
+matrix:
+  include: # https://github.com/travis-ci/travis-ci/issues/8783
+    - os: linux
+      python: 3.4
+    - os: linux
+      python: 3.5
+    - os: linux
+      python: 3.6
+    - os: linux
+      python: 3.7-dev
+    - os: linux
+      python: nightly
+    - os: linux
+      python: pypy3
+    # https://github.com/travis-ci/travis-ci/issues/6865#issuecomment-345513305
+    #- os: linux
+    #  python: pypy3-nightly
+    - os: osx
+      python: nightly
+    #- os: osx
+    #  python: pypy3
+    #- os: osx
+    #  python: pypy3-nightly
+before_install:
+  - source ./.ci/pythonStdlibFixes.sh
+  - pip3 install --upgrade setuptools setuptools_scm
+  - pip3 install --upgrade git+https://github.com/pypa/pip.git git+https://github.com/pypa/wheel.git
+  - pip3 install --upgrade coveralls
+  - pip3 install --upgrade git+https://gitlab.com/KOLANICH/alternativez.py.git
+  - pip3 install --upgrade git+https://gitlab.com/KOLANICH/lazyImport.py.git
+  - pip3 install --upgrade git+https://gitlab.com/KOLANICH/Chassis.py.git
+  - pip3 install --upgrade https://gitlab.com/KOLANICH/pyxgboost/-/jobs/artifacts/master/raw/wheels/pyxgboost-0.CI-py3-none-any.whl?job=build
+install:
+  - python setup.py install
+script:
+  - coverage run --source=AutoXGBoost setup.py test
+after_success:
+  - coveralls
+  # - python3 setup.py bdist_wheel
+  # - python3 setup.py sdistz

Code_Of_Conduct.md

@@ -0,0 +1 @@
+No codes of conduct! Just do what you feel is right and say what you feel is right using the language you feel is right. If you feel that it is right to [make an own fork with a CoC and SJWs](https://en.wikipedia.org/wiki/Bender_Rodriguez), just do that. We here are doing the work, not accusing each other in violating codes of conduct.

Contributing.md

@@ -0,0 +1,79 @@
+Contributing guidelines
+=======================
+
+This project has a bit different policy than the rest of other projects have. Please read it carefully, otherwise it would be very surprising.
+
+Style guide
+-----------
+0. We don't follow PEP-8.
+1. Tabs and spaces are controlled by `.editorconfig` file. See https://editorconfig.org/ for more info about its format. [Install a plugin](https://editorconfig.org/#download) for your IDE/text editor if it doesn't support it out of the box.
+2. No manual line wrapping. Wrapping is done by text editor. Enable this feature.
+
+And some very opinionated things
+--------------------------------
+1. The joke about dead python is not going to be removed. It's the message that everyone should drop python 2 as soon as possible. If you find it inappropriate it's your own personal problem.
+2. We really DON'T support python 2, so don't send PRs fixing py2 support. I'm sorry, but this coup de grace is a necessity. With python 4 not so far we don't want to support a serpentarium of different python versions. Since python 3 works fine on a very ancient 1999 year hardware and 2001 year old OS there shouldn't be a serious reason to stay on py 2. If your org cannot or is afraid to migrate to later versions of python, it's your and your org problem, not ours. I advise you to start migrating as soon as possible, to make yourself ready for the moment py2 officially dropped by PSF, you have to migrate anyway sooner or later, if you are not dropping python at all. You can use the fact that many projects are dropping python 2 support to persuade your boss that you have to migrate. Fix your tests if you don't trust them, use 2to3, fix after it, make the code to pass the tests. You also may expect python 3 support being dropped not so far after python 4 release (it depends on if it requires to replace the hardware and the OS, for example pythons >3.5 have no support for Windows XP, and XP is needed for old (but good) hardware costing several millions of $ and having no drivers for Windows 7 or Linux).
+
+And now when the brief organizational FAQ is over, the docs about the architecture.
+
+Architecture guide
+==================
+
+As already mentioned, it is easy to add an own backend. Here is an approximate algorithm.
+
+0. Read his guide entirely and get yourself familiar to the conventions used in this library.
+	* `HyperparamVector` is a class encapsulating the most generic aspects of a hyperparam vector. Shouldn't be instantiated.
+		* `dict2native` transforming a dict into an object of native output type of a spec.
+		* `native2dict` transforming an object of native output type into a dict of a spec.
+	* `Spec` is a class storing and transforming generic search space specification into an optimizer-specific one. It must have some properties:
+		* `hyperparamsVectorType:HyperparamVector` is a type of a vector.
+		* `hyperparamsSpecType:HyperparamVector` is a type of a spec itself.
+	* `Spec`s are built via mixins. Because it is tricky to remember the right order of inheritance, and because we don't want to precreate them all (exponentially many from the count of mixins) `MSpec` is the function creating the spec class for you. See the detailed description later.
+		* `name` - a desired name, don't use: the names are generated automatically
+		* `isDummy` - do not transform spec items. Used for testing.
+	* `Optimizer` is a class doing optimization. It may get additional arguments. It must have some properties/methods:
+		* `specType` - Is a type of a spec.
+		* `prepareScoring` - a method setting up scoring. It is your chance to do something before progressbar appeared. It checks correctness, creates objects and returns a `tuple` of
+			0. count of function evaluations. You usually need `self.iters` here.
+			1. name of optimizer to display on the progressbar.
+			2. a context object. You should put there a maximally prepared evaluator.
+		* `invokeScoring` - receives a black-box function to optimize, a progressbar object and the context created by `prepareScoring`. This function is called in the context of progressbar. Progressbar object can be used to send messages to progressbar. Usually you wrap the black box function into own one, transforming its results to the format convenient to the optimizer.
+	* `ProgressReporter` is a class to report progress. Just install `tqdm` and it would pick that up.
+	* All the dependencies are imported with `lazyImport`. Imported package shouldn't be accessed in the main scope because this will cause actual import which will cause lags or errors, if the dependency is not available. If you need to do some preparative work, use `lazy_object_proxy.Proxy` for it.
+1. Find an example for the backend. Play with it a little. Determine the following:
+	* the format of a space spec:
+		* if it allows integers. This influences `integerMode:IntegerMetaMap` argument of `MSpec`:
+			* `supportsIntegers` - the optimizer supports specifying integers  and returns them as `int`. No action needed.
+			* `floatIntegers` - the optimizer supports integers, but returns them as `float`
+			* `noIntegers` - the optimizer doesn't support integers. We have to postprocess with rounding which drastically impacts performance.
+		* if it allows plugging variables not from uniform distribution. If it does, you need to define `HyperparamsSpecsConverters` in your class, or use `transformHyperDefItemUniversal`.
+		* if it is very dumb, allows only uniform distribution, disallows categories and scalars and if the optimizer-specific hyperparameter definition is just a sequence `(lower_bound, upper_bound)`. This is a very widespread situation, so we already have the classes for that. Find them in in `SpecOnlyBoxes` module.
+		* if it allows categorical variables. If it does, you need to define `_categorical` in `HyperparamsSpecsConverters`.
+		* if it allows scalars. This influences `scalarMode:ScalarMetaMap` argument of `MSpec`:
+			* `supportsScalars` - the optimizer deals with scalars in the spec itself.
+			* `degenerateCategory` - the optimizer doesn't support scalars but supports categorical variables. The lib puts scalars into a degenerate category. May affect performance, if the impact is low this way is preferred because optimizers may have side effects like saving info to the disk.
+			* `noScalars` - the optimizer doesn't support scalars and using categorical variables for them is infeasible: either not available or too big performance penalty. This causes scalars been saved into a separate dict and added back.
+		* `isArray` - `True` if the spec is `Iterable`-based.
+	* calling convention of a black-box function:
+		* whether it is a `Mapping` (`dict`) or an `Iterable` (`list`, `tuple`, `ndarray`) or something else.
+			* if it is a `Mapping`, you need specs with `hyperparamsVectorType` being derived from `HyperparamVector`
+			* if it is an `Iterable`, you need specs with `hyperparamsVectorType` being derived from `HyperparamArray`
+	* the argument controlling count of iterations. If there is no such a control, you can try to create a wrapper using an exception to break the loop.
+	* the way the optimizer prints messages. All the messages in `invokeScoring` should be print via the `ProgressReporter` object passed. You may need some hacks if a lib directly uses `print` or an io stream. Please don't redefine global builtins like `print`.
+2. Create a draft of a `Spec` using the info from the previous point, if it is needed. Inherit from the stuff resulted from call of `MSpec`.
+3. Now you are ready to write the code of the backend. Inherit `GenericOptimizer` and populate the following properties of the class:
+	* `specType` - is the type of your spec.
+	* `__init__` - here you can save additional parameters of an optimizer
+	* `prepareScoring` - here you can prepare your optimizer. You can save arbitrary context. This function returns a tuple `(countOfIterations, optimizerFriendlyName, context)`.
+		* `countOfIterations` is for the case you need additional iterations. Usually you need to return `self.iters`.
+		* `optimizerFriendlyName` is used in UI.
+		* `context` is your context.
+	* `invokeScoring(self, fn:typing.Callable, pb:ProgressReporter, context)` - actual optimization
+		* `fn` is a prepared function. Accepts either array or dict depending on `self.__class__.specType.hyperparamsVectorType`
+		* `pb` is a `ProgressReporter` object. You can use it for redirecting output and printing messages in the way not destructing the CLI progressbar.
+		* `context` is your context.
+		
+		You usually wanna wrap a `fn` into an own function returning only mean. But try to return the whole tuple first, if it works fine, return the whole tuple. Tuples are compared lexicographically, so this way the values with the same mean but lower variance gonna be preferred by the optimizer.
+
+4. Add it into `__init__.py`. Import the `Optimizer` subclass and add it to `Optimizers` class as a property, use a friendlier name if it is possible.
+3. To test it open `tests\tests.py`, disable all the unneeded tests with `@unittest.skip`, enable `OptimizersTests.testOptimizer` and replace the optimizer name there with a friendlier name for your backend.