Merge pull request #1579 from arenadata/develop

Release 2022.02.01

.dockerignore

@@ -2,6 +2,7 @@
 *
 !COPYRIGHT
 !LICENSE
+!assemble/app/build_venv.sh
 !conf
 !config.json
 !go/bin/runstatus

Makefile

@@ -2,8 +2,8 @@
 BRANCH_NAME ?= $(shell git rev-parse --abbrev-ref HEAD)
 
 ADCMBASE_IMAGE ?= hub.arenadata.io/adcm/base
-ADCMBASE_TAG ?= 20210927130343
-
+ADCMTEST_IMAGE ?= hub.arenadata.io/adcm/test
+ADCMBASE_TAG ?= 20220125085601
 APP_IMAGE ?= hub.adsw.io/adcm/adcm
 APP_TAG ?= $(subst /,_,$(BRANCH_NAME))
 
@@ -33,12 +33,8 @@ buildss: ## Build status server
 buildjs: ## Build client side js/html/css in directory wwwroot
 	@docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:12-alpine ./build.sh
 
-buildbase: ## Build base image for ADCM's container. That is alpine with all packages.
-	cd assemble/base && docker build --pull=true --no-cache=true \
-	-t $(ADCMBASE_IMAGE):$$(date '+%Y%m%d%H%M%S') -t $(ADCMBASE_IMAGE):latest \
-	.
-
 build: describe buildss buildjs ## Build final docker image and all depended targets except baseimage.
+	@docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)
 	@docker build --no-cache=true \
 	-f assemble/app/Dockerfile \
 	-t $(APP_IMAGE):$(APP_TAG) \
@@ -52,9 +48,14 @@ build: describe buildss buildjs ## Build final docker image and all depended tar
 testpyreqs: ## Install test prereqs into user's pip target dir
 	pip install --user -r requirements-test.txt
 
-unittests: ## Run unittests
+test_image:
 	docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)
-	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/tests/base $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /bin/sh -e ./run_test.sh
+
+basetests: test_image ## Run tests/base
+	docker run -i --rm -v $(CURDIR)/:/source -w /source/tests/base $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /venv.sh run default ./run_test.sh
+
+unittests: basetests test_image ## Run unittests
+	docker run -i --rm -v $(CURDIR)/:/source -w /source/ $(ADCMTEST_IMAGE):$(ADCMBASE_TAG) /venv.sh reqs_and_run default /source/requirements.txt /source/python/run_unit.sh
 
 pytest: ## Run functional tests
 	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster-x64
@@ -76,18 +77,25 @@ ng_tests: ## Run Angular tests
 	docker pull hub.adsw.io/library/functest:3.8.6.slim.buster-x64
 	docker run -i --rm -v $(CURDIR)/:/adcm -w /adcm/web hub.adsw.io/library/functest:3.8.6.slim.buster-x64 ./ng_test.sh
 
-linters : ## Run linters
-	docker pull hub.adsw.io/library/pr-builder:3-x64
-	docker run -i --rm -v $(CURDIR)/:/source -w /source hub.adsw.io/library/pr-builder:3-x64 \
-        /bin/bash -xeo pipefail -c "/linters.sh shellcheck pylint pep8 && \
-        /linters.sh -b ./tests -f ../tests pylint && \
-        /linters.sh -f ./tests black && \
-        /linters.sh -f ./tests/functional flake8_pytest_style && \
-        /linters.sh -f ./tests/ui_tests flake8_pytest_style"
+linters: test_image ## Run linters
+	docker run -i --rm -e PYTHONPATH="/source/tests" -v $(CURDIR)/:/source -w /source $(ADCMTEST_IMAGE):$(ADCMBASE_TAG) \
+        /bin/sh -eol pipefail -c "/linters.sh shellcheck && \
+			/venv.sh run default pip install -r requirements.txt -r requirements-test.txt && \
+			cd python && /venv.sh run default pylint_runner --rcfile ../pylintrc &&  cd .. \
+			/linters.sh -b ./tests -f ../tests pylint && \
+			/linters.sh -f ./tests black && \
+			/linters.sh -f ./tests/functional flake8_pytest_style && \
+			/linters.sh -f ./tests/ui_tests flake8_pytest_style"
 
 npm_check: ## Run npm-check
 	docker run -i --rm -v $(CURDIR)/wwwroot:/wwwroot -v $(CURDIR)/web:/code -w /code  node:12-alpine ./npm_check.sh
 
-django_tests : ## Run django tests.
-	docker pull $(ADCMBASE_IMAGE):$(ADCMBASE_TAG)
-	docker run -e DJANGO_SETTINGS_MODULE=adcm.test -i --rm -v $(CURDIR)/:/adcm -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) python python/manage.py test cm
+django_tests: test_image ## Run django tests.
+	docker run -e DJANGO_SETTINGS_MODULE=adcm.test -i --rm -v $(CURDIR)/python:/adcm/python -v $(CURDIR)/data:/adcm/data -v $(CURDIR)/requirements.txt:/adcm/requirements.txt -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /venv.sh reqs_and_run default /adcm/requirements.txt python python/manage.py test cm
+
+##################################################
+#                 U T I L S
+##################################################
+
+base_shell: ## Just mount a dir to base image and run bash on it over docker run
+	docker run -e DJANGO_SETTINGS_MODULE=adcm.test -it --rm -v $(CURDIR)/python:/adcm/python -v $(CURDIR)/data:/adcm/data -w /adcm/ $(ADCMBASE_IMAGE):$(ADCMBASE_TAG) /bin/bash -l

README.md

@@ -10,6 +10,17 @@ That is Arenadata Cluster Manager Project (aka Chapelnik)
 
 [ArenaData ADCM Documentation](http://docs.arenadata.io/adcm/)
 
+# Develop
+
+All standard Django commands are available.
+
+Run dev server for the first time with these commands:
+1. `manage.py migrate`
+2. `init_db.py`
+3. `manage.py upgraderole`
+4. `manage.py runserver --insecure`
+
+Re-run them when needed/applicable.
 
 # Sources
 

assemble/app/Dockerfile

@@ -3,13 +3,18 @@ ARG ADCMBASE_TAG
 FROM $ADCMBASE_IMAGE:$ADCMBASE_TAG
 
 COPY . /adcm/
+COPY assemble/app/build_venv.sh /
+
+RUN cp -r /adcm/os/* / && rm -rf /adcm/os; /build_venv.sh && rm -f /build_venv.sh && rm -rf /adcm/python/ansible && rmdir /var/log/nginx;
+
+RUN /venv.sh reqs default /adcm/requirements.txt
+RUN /venv.sh reqs 2.9 /adcm/requirements.txt
 
-RUN cp -r /adcm/os/* / && rm -rf /adcm/os; cp -r /adcm/python/ansible/* /usr/local/lib/python3.?/site-packages/ansible/ && rm -rf /adcm/python/ansible && rmdir /var/log/nginx;
 
 # Secret_key is mandatory for build_static procedure,
 # but should not be hardcoded in the image.
 # It will be generated on first start.
-RUN /adcm/web/build_static.sh && rm -f /adcm/web/build_static.sh
+RUN /venv.sh run default /adcm/web/build_static.sh && rm -f /adcm/web/build_static.sh
 
 EXPOSE 8000
 

assemble/app/build_venv.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+for d in $(find /adcm/venv -mindepth 1 -maxdepth 1 -type d); do
+    cp -r /adcm/python/ansible/* "${d}/lib/python3.10/site-packages/ansible/"
+done;

assemble/base/requirements-base.txt

@@ -9,6 +9,7 @@ djangorestframework
 drf-flex-fields==0.9.1
 drf-extensions
 django-background-tasks
+django-guardian
 social-auth-app-django
 git+git://github.com/arenadata/django-generate-secret-key.git
 # TODO jinja2 and markupsafe should be unpinned after https://arenadata.atlassian.net/browse/ADCM-2089 fixup
@@ -20,6 +21,7 @@ pyyaml
 ruyaml
 toml
 uwsgi
+jsonschema
 version_utils
 ruyaml
 yspec
@@ -29,4 +31,4 @@ jmespath
 lxml
 pycrypto
 --extra-index-url https://ci.arenadata.io/artifactory/api/pypi/python-packages/simple
-adwp-events==0.1.7
+adwp-events==0.1.8

os/etc/service/background_tasks/run

@@ -16,4 +16,6 @@ source /etc/adcmenv
 waitforwsgi
 
 echo "Run process_tasks ..."
+
+source "${adcmroot}/venv/default/bin/activate"
 python ${adcmroot}/python/manage.py process_tasks --log-std

os/etc/service/init/run

@@ -11,13 +11,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set +x
+
 source /etc/adcmenv
 
 echo "Application initialisation ..."
 
 mkdir -p "${adcmlog}"
-exec 1>"${adcmlog}/init.out"
-exec 2>"${adcmlog}/init.err"
+exec 1>>"${adcmlog}/init.out"
+exec 2>>"${adcmlog}/init.err"
 mkdir -p "${adcmrun}"
 mkdir -p "${adcmbundle}"
 mkdir -p "${adcmdownload}"
@@ -29,8 +31,12 @@ chmod 777 "${adcmtmp}"
 chmod a+t "${adcmtmp}"
 mkdir -p "${adcmuserconf}"
 mkdir -p "${adcmlog}/nginx"
-ln -s "${adcmlog}/nginx" /var/log/nginx
+[ ! -L  "/var/log/nginx" ] && ln -s "${adcmlog}/nginx" /var/log/
+
+## We got an issue on existing installation with folded log/nginx/nginx/nginx....
+[ -L "${adcmlog}/nginx/nginx" ] && rm -f "${adcmlog}/nginx/nginx"
 
+source "${adcmroot}/venv/default/bin/activate"
 
 echo "Root keys generation"
 key=/root/.ssh/id_rsa
@@ -51,6 +57,7 @@ fi
 "${adcmroot}"/python/backupdb.py
 "${adcmroot}"/python/manage.py migrate
 "${adcmroot}"/python/init_db.py
+"${adcmroot}"/python/manage.py upgraderole
 touch "${initreadyfile}"
 
 # That is sleep forever case because I have no idea how to force

os/etc/service/wsgi/run

@@ -16,8 +16,8 @@ source /etc/adcmenv
 waitforinit
 
 echo "Run main wsgi application ..."
-exec 1>"${adcmlog}/service_wsgi.out"
-exec 2>"${adcmlog}/service_wsgi.err"
+exec 1>>"${adcmlog}/service_wsgi.out"
+exec 2>>"${adcmlog}/service_wsgi.err"
 
 cd "${adcmroot}/python"
-uwsgi --socket "${wsgisocketfile}" --pidfile "/run/uwsgi.pid" --module adcm.wsgi --chmod-socket=777 --logger file:logfile="${adcmlog}/wsgi.log",maxsize=2000000
+uwsgi --venv "${adcmroot}/venv/default" --socket "${wsgisocketfile}" --pidfile "/run/uwsgi.pid" --module adcm.wsgi --chmod-socket=777 --logger file:logfile="${adcmlog}/wsgi.log",maxsize=2000000

pytest.sh

@@ -12,7 +12,7 @@
 # limitations under the License.
 
 
-pip3 install -U pip
+#pip3 install -U pip
 pip3 install -r requirements-test.txt
 
 find . -name "*.pyc" -type f -delete

python/adcm/settings.py

@@ -60,6 +60,7 @@
 # Application definition
 
 INSTALLED_APPS = [
+    'rbac',  # keep it above 'django.contrib.auth' in order to keep 'createsuperuser' working
     'django_generate_secret_key',
     'django_filters',
     'django.contrib.auth',
@@ -74,6 +75,7 @@
     'corsheaders',
     'rest_framework.authtoken',
     'social_django',
+    'guardian',
     'adwp_events',
     'cm.apps.CmConfig',
 ]
@@ -108,7 +110,7 @@
 ]
 
 WSGI_APPLICATION = 'adcm.wsgi.application'
-LOGIN_URL = '/admin/login/'
+LOGIN_URL = '/api/v1/auth/login/'
 
 REST_FRAMEWORK = {
     # Use Django's standard `django.contrib.auth` permissions,
@@ -128,6 +130,7 @@
     'DEFAULT_FILTER_BACKENDS': [
         'django_filters.rest_framework.DjangoFilterBackend',
         'rest_framework.filters.OrderingFilter',
+        'rest_framework.filters.SearchFilter',
     ],
     'EXCEPTION_HANDLER': 'cm.errors.custom_drf_exception_handler',
 }
@@ -144,11 +147,16 @@
         # 'HOST': 'localhost',
         # 'USER': 'adcm',
         # 'PASSWORD': 'adcm',
+        # 'TEST': {
+        #     'NAME': os.path.join(BASE_DIR, 'data/var/test.db'),
+        # },
         'OPTIONS': {
             'timeout': 20,
         },
-    }
+    },
 }
+# does not work for multi-table inherited model, but works fine as-is without user model swapping
+# AUTH_USER_MODEL = 'rbac.User'
 
 # Password validation
 # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
@@ -166,6 +174,7 @@
 
 AUTHENTICATION_BACKENDS = (
     'django.contrib.auth.backends.ModelBackend',
+    'guardian.backends.ObjectPermissionBackend',
     'social_core.backends.google.GoogleOAuth2',
 )
 
@@ -181,7 +190,6 @@
     'social_core.pipeline.user.user_details',
     'cm.views.get_token',
 )
-
 SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/'
 
 SOCIAL_AUTH_GOOGLE_OAUTH2_WHITELISTED_DOMAINS = []
@@ -252,7 +260,13 @@
             'level': 'DEBUG',
             'propagate': True,
         },
-        'adwp.events': {
+        'django.template': {
+            'level': 'ERROR',
+        },
+        'django.utils.autoreload': {
+            'level': 'INFO',
+        },
+        'adwp': {
             'handlers': ['adwp_file'],
             'level': 'DEBUG',
             'propagate': True,

python/adcm/urls.py

@@ -28,6 +28,7 @@
 from django.conf.urls import include
 
 import api.urls
+import api_ui.urls
 import cm.views
 import cm.adcm_config
 
@@ -39,7 +40,15 @@
     path('cm/error/<msg>/', cm.views.error),
     path('social/complete/google-oauth2/', cm.views.complete),
     path('social/', include('social_django.urls', namespace='social')),
-    path('api/v1/', include(api.urls)),
+    path(
+        'api/',
+        include(
+            [
+                path('v1/', include(api.urls)),
+                path('ui/', include(api_ui.urls)),
+            ]
+        ),
+    ),
 ]
 
 cm.adcm_config.load_social_auth()

python/api/action/views.py

@@ -10,6 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from rest_framework import permissions
 from rest_framework.response import Response
 
 from api.api_views import (
@@ -20,8 +21,11 @@
     create,
     check_obj,
     filter_actions,
+    permission_denied,
 )
 from api.job.serializers import RunTaskSerializer
+
+from cm.job import get_host_object
 from cm.models import (
     Host,
     Action,
@@ -136,12 +140,29 @@ def get(self, request, *args, **kwargs):
 class RunTask(GenericAPIPermView):
     queryset = TaskLog.objects.all()
     serializer_class = RunTaskSerializer
+    permission_classes = (permissions.IsAuthenticated,)
+
+    def has_action_perm(self, action, obj):
+        user = self.request.user
+
+        if user.has_perm('cm.add_task'):
+            return True
+
+        if action.host_action:
+            obj = get_host_object(action, obj.cluster)
+
+        return user.has_perm(f'cm.run_action_{action.display_name}', obj)
+
+    def check_action_perm(self, action, obj):
+        if not self.has_action_perm(action, obj):
+            permission_denied()
 
     def post(self, request, *args, **kwargs):
         """
         Ran specified action
         """
         obj, action_id = get_obj(**kwargs)
         action = check_obj(Action, {'id': action_id}, 'ACTION_NOT_FOUND')
+        self.check_action_perm(action, obj)
         serializer = self.serializer_class(data=request.data, context={'request': request})
         return create(serializer, action=action, task_object=obj)