From 53119839fa1d3e930c339c952046d133a32e01eb Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 21 Aug 2014 16:12:23 +0200 Subject: [PATCH 01/93] Fields for dpd/reprocessing request --- .../templates/prodtask/_requestform.html | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/atlas/prodtask/templates/prodtask/_requestform.html b/atlas/prodtask/templates/prodtask/_requestform.html index e7b15b96..ef6299ec 100644 --- a/atlas/prodtask/templates/prodtask/_requestform.html +++ b/atlas/prodtask/templates/prodtask/_requestform.html @@ -1,6 +1,7 @@ {% extends parent_template %} {% load url from future %} {% load static from staticfiles %} +{% load js %} {% block extra_css %} {{ block.super }} @@ -11,8 +12,74 @@ {% block subtitle %} {{pre_form_text|safe}} {% endblock %} +{% block base_js %} +{{ block.super }} + +{% django_js %} + + +{% endblock %} + +{% block extra_js %} +{{ block.super }} + + +{% endblock %} {% block body %} + + + {% if error_message %} ERROR: {{ error_message }} {% endif %} @@ -32,6 +99,86 @@ {{ form.as_table }} {% endif %} + + Add slice + 1 +
+
+ Slice #0 +
+
+ +
+
+
+
+ +
+
+
+
+ + + + + + + + + + + + + + + + + + + + +
Dataset Nameevents
mc14_8TeV.117050.PowhegPythia_P2011C_ttbar.merge.AOD.e1727_s1933_s1911_r5591_r562530000
mc14_8TeV.117050.PowhegPythia_P2011C_ttbar.merge.AOD.e1727_s1933_s1911_r5591_r562530000
+
+
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+ {% endblock %} From 585e5dfb74ae34d75be9807f001d86c97f1a36c4 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 21 Aug 2014 16:14:01 +0200 Subject: [PATCH 02/93] Adding form field for transmitting slices parameters --- atlas/prodtask/forms.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atlas/prodtask/forms.py b/atlas/prodtask/forms.py index 70f216db..9e7b171c 100644 --- a/atlas/prodtask/forms.py +++ b/atlas/prodtask/forms.py @@ -36,6 +36,7 @@ class TRequestMCCreateCloneForm(TRequestCreateCloneConfirmation): manager = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) + class Meta: model = TRequest exclude = ['reqid'] @@ -48,6 +49,7 @@ class TRequestDPDCreateCloneForm(TRequestCreateCloneConfirmation): cstatus = CharField(widget=forms.HiddenInput, required=False) request_type = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) + hidden_json_slices = CharField(required=False, label="Will be hidden") class Meta: model = TRequest From e95556a057f215775cd5ffd9b0835d201c47da23 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 21 Aug 2014 17:15:31 +0200 Subject: [PATCH 03/93] Fix step adding --- atlas/prodtask/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index c310e0d4..fda6a4fa 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -187,7 +187,10 @@ def create_steps(slice_steps, reqid, STEPS=StepExecution.STEPS, approve_level=99 #Create new step _logger.debug("Create step: %s execution for request: %i slice: %i "% (steps_status[index],int(reqid),input_list.slice)) - temp_priority = priority_obj.priority(STEPS[index], step_value['value']) + if(STEPS[index]): + temp_priority = priority_obj.priority(STEPS[index], step_value['value']) + else: + temp_priority = priority_obj.priority('Evgen', step_value['value']) # store input_vents only for first not skipped step, otherwise temp_input_events = -1 if still_skipped: From 63a9b1e0747d7a8dcac04202169d15cb43758ba6 Mon Sep 17 00:00:00 2001 From: stavrik Date: Tue, 26 Aug 2014 09:18:06 +0200 Subject: [PATCH 04/93] Bug with using .address fix --- atlas/prodtask/task_views.py | 2 +- .../templates/prodtask/_request_table.html | 2 +- .../templates/prodtask/_task_table.html | 49 ++++++++++++++----- 3 files changed, 40 insertions(+), 13 deletions(-) diff --git a/atlas/prodtask/task_views.py b/atlas/prodtask/task_views.py index f32e4663..71f7bce3 100644 --- a/atlas/prodtask/task_views.py +++ b/atlas/prodtask/task_views.py @@ -300,7 +300,7 @@ class Meta: fnServerParams = "taskServerParams" - fnDrawCallback = "taskDrawCallback" + fnPreDrawCallback = "taskDrawCallback" fnClientTransformData = "prepareData" diff --git a/atlas/prodtask/templates/prodtask/_request_table.html b/atlas/prodtask/templates/prodtask/_request_table.html index 8103ad25..03cac204 100644 --- a/atlas/prodtask/templates/prodtask/_request_table.html +++ b/atlas/prodtask/templates/prodtask/_request_table.html @@ -299,7 +299,7 @@ diff --git a/atlas/prodtask/templates/prodtask/_task_table.html b/atlas/prodtask/templates/prodtask/_task_table.html index 524db4ad..70d41042 100755 --- a/atlas/prodtask/templates/prodtask/_task_table.html +++ b/atlas/prodtask/templates/prodtask/_task_table.html @@ -412,9 +412,7 @@ } } - var changed_params = {}; - - function ParseParametersFromURL() + function LoadDefaultAndParametersFromURL() { var time_from = new Date(); var time_to = new Date(); @@ -447,7 +445,6 @@ default: $( "#"+parameters_list[i].id ).val( value ); - changed_params[ parameters_list[i].name ] = parameters_list[i]; break; } } @@ -456,7 +453,7 @@ switch( parameters_list[i].name ) { case 'task_type': - $("#task_type").val('all'); + // $("#task_type").val('all'); break; case 'time_from': $( "#time_from" ).datepicker( "setDate", time_from ); @@ -473,13 +470,43 @@ } } } - if( ! extern_parameter ) - $("#task_type").val('production'); + if( extern_parameter && ! $.address.parameter('task_type') ) + $("#task_type").val('all'); $( "#time_period" ).change(); } + function UpdateParametersFromURL() + { + for(var i in parameters_list) + { + var value = $.address.parameter( parameters_list[i].name ); + if( value && value != $( "#"+parameters_list[i].id ).val() ) + { + extern_parameter = 1; + switch( parameters_list[i].name ) + { + case 'time_from': + // time_from = new Date().parse( value ); + $( "#time_from" ).datepicker( "setDate", value ); + break; + case 'time_to': + // time_from = new Date().parse( value ); + $( "#time_to" ).datepicker( "setDate", value ); + break; + case 'time_period': + $("#time_period").val(value); + break; + + default: + $( "#"+parameters_list[i].id ).val( value ); + break; + } + } + } + } + $( "#time_from" ).datepicker({ defaultDate: "-1m", changeMonth: true, @@ -543,7 +570,9 @@ $('#task_stat_table').html(table); } - ParseParametersFromURL(); + LoadDefaultAndParametersFromURL(); + $("#update_task_table").button({ disabled: true }); + $("#update_task_table").tooltip( ); @@ -561,11 +590,9 @@ taskTable.fnSetColumnVis( 22, false, false ); taskTable.fnSetColumnVis( 23, false, false ); - $("#update_task_table").button({ disabled: true }); - $("#update_task_table").tooltip( ); $("#update_task_table").click(function(){ taskTable.fnDraw(); }); - $.address.change(function(){ ParseParametersFromURL(); taskTable.fnDraw(); }); + taskTable.one("draw", function(){ $.address.change(function(){ UpdateParametersFromURL(); taskTable.fnDraw(); }) }); for(var i in parameters_list) $('#'+parameters_list[i].id).click(function(){ $("#update_task_table").button( "enable" ); }); From d337153fd54558ceb9f23bd0338efc0a1f2b17eb Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 27 Aug 2014 12:20:25 +0200 Subject: [PATCH 05/93] VOMS interface, authentication backend and middleware (for Django), VOMS mappings --- atlas/auth/__init__.py | 1 + atlas/auth/voms/__init__.py | 1 + atlas/auth/voms/backends.py | 71 +++++++++++++++++++ atlas/auth/voms/collector.py | 47 ++++++++++++ atlas/auth/voms/interface.py | 62 ++++++++++++++++ atlas/auth/voms/management/__init__.py | 1 + .../auth/voms/management/commands/__init__.py | 1 + .../voms/management/commands/updatevomsmap.py | 14 ++++ atlas/auth/voms/middleware.py | 15 ++++ atlas/auth/voms/models.py | 21 ++++++ 10 files changed, 234 insertions(+) create mode 100644 atlas/auth/__init__.py create mode 100644 atlas/auth/voms/__init__.py create mode 100644 atlas/auth/voms/backends.py create mode 100644 atlas/auth/voms/collector.py create mode 100644 atlas/auth/voms/interface.py create mode 100644 atlas/auth/voms/management/__init__.py create mode 100644 atlas/auth/voms/management/commands/__init__.py create mode 100644 atlas/auth/voms/management/commands/updatevomsmap.py create mode 100644 atlas/auth/voms/middleware.py create mode 100644 atlas/auth/voms/models.py diff --git a/atlas/auth/__init__.py b/atlas/auth/__init__.py new file mode 100644 index 00000000..b4d39147 --- /dev/null +++ b/atlas/auth/__init__.py @@ -0,0 +1 @@ +__author__ = 'sbel' diff --git a/atlas/auth/voms/__init__.py b/atlas/auth/voms/__init__.py new file mode 100644 index 00000000..b4d39147 --- /dev/null +++ b/atlas/auth/voms/__init__.py @@ -0,0 +1 @@ +__author__ = 'sbel' diff --git a/atlas/auth/voms/backends.py b/atlas/auth/voms/backends.py new file mode 100644 index 00000000..bd5e3fa6 --- /dev/null +++ b/atlas/auth/voms/backends.py @@ -0,0 +1,71 @@ +import os + +from django.conf import settings +from django.contrib.auth.backends import ModelBackend +from django.contrib.auth.models import Group +from django.contrib.auth.models import User + +from .interface import VomsInterface +from .models import VomsUser + + +def add_group(group_name): + try: + group = Group.objects.get(name=group_name) + except Group.DoesNotExist: + group = Group(name=group_name) + group.save() + + return group + + +class VomsBackend(ModelBackend): + """ Adding VOMS-based authentication groups for user. """ + + def authenticate(self, request=None): + """ Chacking user against VOMS data and adding corresponding authentication groups. + + Parameters: + request: Http request (HttpRequest). + + Returns always None to pass further handling to ShibSSO module. + + """ + + username = request.META.get(settings.META_USERNAME) + + if not username: + return + + dn_map = {} + for record in VomsUser.objects.filter(username=username): + dn_map[record.dn] = record.ca + + if not dn_map: + return + + os.environ["X509_USER_PROXY"] = settings.VOMS_PROXY_CERT + + options = VomsInterface.get_identity_options() + options.update(settings.VOMS_OPTIONS) + voms = VomsInterface(options) + + voms_groups = [] + + for (dn, ca) in dn_map.items(): + vo_roles = voms.list_user_roles(dn, ca) or [] + vo_groups = voms.list_user_groups(dn, ca) or [] + vo_roles = ["vomsrole:" + x for x in vo_roles] + vo_groups = ["vomsgroup:" + x for x in vo_groups] + + voms_groups = vo_groups + vo_roles + + + for group in voms_groups: + add_group(group) + + meta_groups = request.META.get(settings.META_GROUP, '') + groups = ';'.join( [x for x in (voms_groups + [meta_groups]) if x] ) + request.META[settings.META_GROUP] = groups + + return diff --git a/atlas/auth/voms/collector.py b/atlas/auth/voms/collector.py new file mode 100644 index 00000000..b6199826 --- /dev/null +++ b/atlas/auth/voms/collector.py @@ -0,0 +1,47 @@ +from django.conf import settings + +from .interface import VomsInterface + +from .models import VomsUser + +def run(): + options = VomsInterface.get_identity_options() + options.update(settings.VOMS_OPTIONS) + + voms = VomsInterface(options) + + voms_users = {} + + for user_info in voms.list_users(): + dn = user_info["DN"] + ca = user_info["CA"] + nickname = voms.get_user_nickname(dn, ca) + + if not voms_users.get(nickname): + voms_users[nickname] = {} + voms_users[nickname].update({dn: ca}) + + for user in VomsUser.objects.all(): + info = voms_users.get(user.username) + if not info or not (dn in voms_users[user.username]): + user.delete() + continue + else: + del voms_users[user.username][user.dn] + + for (nickname, info) in voms_users.items(): + for (dn, ca) in info.items(): + user = VomsUser() + + try: + user = VomsUser.objects.get(username=nickname, dn=dn) + except: + user.username = nickname + user.dn = dn + user.ca = ca + user.save() + + +if __name__ == "__main__": + run() + diff --git a/atlas/auth/voms/interface.py b/atlas/auth/voms/interface.py new file mode 100644 index 00000000..a4c84949 --- /dev/null +++ b/atlas/auth/voms/interface.py @@ -0,0 +1,62 @@ +#!/bin/env python + +from VOMSAdmin.VOMSCommands import VOMSAdminProxy + +class VomsInterface: + """ + Retrieving information from VOMS server. + """ + + def __init__(self, options): + self.options = VomsInterface.get_identity_options() + self.options.update(options) + self.voms = VOMSAdminProxy(**options) + + def list_users(self): + vo_users = self.voms.listUsers() + users = [] + + for user in vo_users: + user_info = {} + for field in "CA,DN,mail".split(","): + user_info[field] = getattr(user, "_" + field) + users.append(user_info) + return users + + def list_user_attributes(self, dn, ca): + """ + Read user's attributes from VOMS server + :param dn: DN of user's certificate + :param ca: DN of the issuer of the certificate (CA) + :return: user's attributes as a dict + """ + result = {} + attrs = self.voms.call_method("list-user-attributes", dn, ca) + for item in attrs: + result[item._attributeClass._name] = item._value + return result + + def list_user_groups(self, dn, ca): + """ + Get user's groups in VO from VOMS server + :param dn: DN of user's certificate + :param ca: DN of the issuer of the certificate (CA) + :return: list of groups + """ + return self.voms.call_method("list-user-groups", dn, ca) + + def list_user_roles(self, dn, ca): + return self.voms.call_method("list-user-roles", dn, ca) + + def get_user_nickname(self, dn, ca): + attributes = self.list_user_attributes(dn, ca) + return attributes.get("nickname") + + @staticmethod + def get_identity_options(voms_admin_path="/usr/bin/voms-admin"): + import imp + voms_admin = imp.load_source("voms_admin", voms_admin_path) + voms_admin.vlog = lambda msg: None + voms_admin.setup_identity() + + return voms_admin.options diff --git a/atlas/auth/voms/management/__init__.py b/atlas/auth/voms/management/__init__.py new file mode 100644 index 00000000..1351973a --- /dev/null +++ b/atlas/auth/voms/management/__init__.py @@ -0,0 +1 @@ +__author__ = 'serge' diff --git a/atlas/auth/voms/management/commands/__init__.py b/atlas/auth/voms/management/commands/__init__.py new file mode 100644 index 00000000..1351973a --- /dev/null +++ b/atlas/auth/voms/management/commands/__init__.py @@ -0,0 +1 @@ +__author__ = 'serge' diff --git a/atlas/auth/voms/management/commands/updatevomsmap.py b/atlas/auth/voms/management/commands/updatevomsmap.py new file mode 100644 index 00000000..2e8e894f --- /dev/null +++ b/atlas/auth/voms/management/commands/updatevomsmap.py @@ -0,0 +1,14 @@ +__author__ = 'sbel' + + +from django.core.management.base import BaseCommand, CommandError +from atlas.auth.voms import collector + + +class Command(BaseCommand): + + def handle(self, *args, **options): + collector.run() + + + diff --git a/atlas/auth/voms/middleware.py b/atlas/auth/voms/middleware.py new file mode 100644 index 00000000..1881aafc --- /dev/null +++ b/atlas/auth/voms/middleware.py @@ -0,0 +1,15 @@ +from django.conf import settings +from django.contrib import auth + + +class VomsMiddleware(object): + """ + Updates user's groups once a session. + """ + + def process_request(self, request): + username = request.META.get(settings.META_USERNAME) + + if username and not request.session.get('VOMS_GROUPS_INITIALIZED'): + if auth.authenticate(request=request): + request.session['VOMS_GROUPS_INITIALIZED'] = 1 diff --git a/atlas/auth/voms/models.py b/atlas/auth/voms/models.py new file mode 100644 index 00000000..28c364ee --- /dev/null +++ b/atlas/auth/voms/models.py @@ -0,0 +1,21 @@ +__author__ = 'sbel' + +from django.db import models +from django.utils import timezone + +class VomsUser(models.Model): + username = models.CharField(max_length=60, db_column='username', primary_key=True) + dn = models.CharField(max_length=255, db_column='dn') + ca = models.CharField(max_length=255, db_column='ca') + added_on = models.DateTimeField(auto_now_add=True, db_column='added_on') + + def save(self, *args, **kwargs): + if not self.added_on: + self.added_on = timezone.now() + super(VomsUser, self).save(*args, **kwargs) + + + class Meta: + managed = True + db_table = u'"VOMS_USERS_MAP"' + unique_together = (("username", "dn"),) From 97c4dec10692daa14341ee11334c7b0077e18232 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 27 Aug 2014 15:13:59 +0200 Subject: [PATCH 06/93] Added ATLAS_DEFT as a database for VOMS mapping table --- atlas/auth/voms/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas/auth/voms/models.py b/atlas/auth/voms/models.py index 28c364ee..68d81b4c 100644 --- a/atlas/auth/voms/models.py +++ b/atlas/auth/voms/models.py @@ -17,5 +17,5 @@ def save(self, *args, **kwargs): class Meta: managed = True - db_table = u'"VOMS_USERS_MAP"' + db_table = u'"ATLAS_DEFT"."VOMS_USERS_MAP"' unique_together = (("username", "dn"),) From 7b33f4f943cc930ce1f121b461ea3a89580e86a8 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Wed, 27 Aug 2014 15:36:23 +0200 Subject: [PATCH 07/93] DPD LIST replacement interface --- atlas/prodtask/ddm_api.py | 36 +++++ atlas/prodtask/dpdconfparser.py | 13 +- atlas/prodtask/forms.py | 14 +- atlas/prodtask/request_views.py | 150 +++++++++++++----- .../templates/prodtask/_requestform.html | 87 +++++++--- atlas/prodtask/urls.py | 4 +- atlas/prodtask/views.py | 42 +---- 7 files changed, 238 insertions(+), 108 deletions(-) diff --git a/atlas/prodtask/ddm_api.py b/atlas/prodtask/ddm_api.py index 6544d70c..defee7d2 100644 --- a/atlas/prodtask/ddm_api.py +++ b/atlas/prodtask/ddm_api.py @@ -1,9 +1,45 @@ import logging import os +from ..getdatasets.models import ProductionDatasetsExec, TaskProdSys1 +from ..settings import dq2client as dq2_settings _logger = logging.getLogger('ddm_prodtask') + +def find_dataset_events(dataset_pattern): + return_list = [] + ddm = DDM(dq2_settings.PROXY_CERT,dq2_settings.RUCIO_ACCOUNT) + datasets_containers = ddm.find_dataset(dataset_pattern.replace('%','*')) + containers = [x for x in datasets_containers if x[-1] == '/' ] + datasets = [x for x in datasets_containers if x not in containers ] + for container in containers: + event_count = 0 + is_good = True + datasets_in_container = ddm.dataset_in_container(container) + for dataset_name in datasets_in_container: + if dataset_name in datasets: + datasets.remove(dataset_name) + try: + dataset = ProductionDatasetsExec.objects.get(name=dataset_name) + task = TaskProdSys1.objects.get(taskid=dataset.taskid) + if (task.status not in ['aborted','failed','lost']): + event_count += task.total_events + else: + is_good = False + except: + is_good = False + if is_good: + return_list.append({'dataset_name':container,'events':str(event_count)}) + for dataset_name in datasets: + try: + task = TaskProdSys1.objects.get(taskname=dataset_name) + if (task.status not in ['aborted','failed','lost']): + return_list.append({'dataset_name':dataset_name,'events':str(task.total_events)}) + except: + pass + return return_list + class DDM(object): """ Wrapper for atlas ddm systems: dq2/rucio diff --git a/atlas/prodtask/dpdconfparser.py b/atlas/prodtask/dpdconfparser.py index 233d8ab5..7c166448 100644 --- a/atlas/prodtask/dpdconfparser.py +++ b/atlas/prodtask/dpdconfparser.py @@ -16,8 +16,10 @@ class ConfigParser(object): COMMENT_CHAR = '#' OPTION_CHAR = ':' - def parse_config(self, open_file): - + def parse_config(self, open_file, count_list=[]): + if count_list!= []: + for key in count_list: + self.options[key+'_count_list'] = [] for line in open_file: # First, remove comments: if self.COMMENT_CHAR in line: @@ -25,11 +27,18 @@ def parse_config(self, open_file): line, comment = line.split(self.COMMENT_CHAR, 1) # Second, find lines with an option=value: if self.OPTION_CHAR in line: + # split on option char: option, value = line.split(self.OPTION_CHAR, 1) # strip spaces: option = option.strip() value = value.strip() + for key in count_list: + if key==option: + self.options[key+'_count_list'].append(0) + else: + if self.options[key+'_count_list']: + self.options[key+'_count_list'][-1]+=1 # store in dictionary: value_list = self.options.get(option, []) value_list.append(value) diff --git a/atlas/prodtask/forms.py b/atlas/prodtask/forms.py index 9e7b171c..07ba6731 100644 --- a/atlas/prodtask/forms.py +++ b/atlas/prodtask/forms.py @@ -23,6 +23,9 @@ class TRequestCreateCloneConfirmation(ModelForm): description = CharField(label='Short description', widget=Textarea, required=False) cstatus = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=True) + provenance = CharField(required=True) + phys_group = CharField(required=True, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + campaign = CharField(required=True) class Meta: model = TRequest @@ -35,7 +38,8 @@ class TRequestMCCreateCloneForm(TRequestCreateCloneConfirmation): excelfile = FileField(required=False, label="Spreadsheet File") manager = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) - + phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + campaign = CharField(required=False) class Meta: model = TRequest @@ -49,7 +53,9 @@ class TRequestDPDCreateCloneForm(TRequestCreateCloneConfirmation): cstatus = CharField(widget=forms.HiddenInput, required=False) request_type = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) - hidden_json_slices = CharField(required=False, label="Will be hidden") + hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden") + phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + campaign = CharField(required=False) class Meta: model = TRequest @@ -63,6 +69,8 @@ class TRequestHLTCreateCloneForm(TRequestCreateCloneConfirmation): cstatus = CharField(widget=forms.HiddenInput, required=False) request_type = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) + phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + campaign = CharField(required=False) class Meta: model = TRequest @@ -78,6 +86,8 @@ class TRequestReprocessingCreateCloneForm(TRequestCreateCloneConfirmation): # tag_hierarchy = CharField(help_text='tag hierarhy as python list with tuples as branches', # widget=Textarea, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) + phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + campaign = CharField(required=False) class Meta: model = TRequest diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index d01875b7..72f0b018 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -8,6 +8,8 @@ from django.template.response import TemplateResponse from django.utils import timezone from django.db.models import Count, Q +from django.views.decorators.csrf import csrf_protect +from ..prodtask.ddm_api import find_dataset_events import core.datatables as datatables import json import logging @@ -220,9 +222,39 @@ def hlt_form_prefill(form_data, request): _logger.debug('Gathered data: %s' % spreadsheet_dict) return spreadsheet_dict, eroor_message +def parse_json_slice_dict(json_string): + spreadsheet_dict = [] + input_dict = json.loads(json_string) + slice_index = 0 + for slice_number, slice in input_dict.items(): + for dataset in slice['datasets'].split(','): + if dataset: + irl = dict(slice=slice_index, brief=' ', comment='', dataset=dataset, + input_data='', + project_mode=slice['projectmode'], + priority=int(slice['priority']), + input_events=int(slice['totalevents'])) + slice_index += 1 + st_sexec_list = [] + if slice['ctag']: + task_config = {} + nEventsPerJob = slice['eventsperjob'] + task_config.update({'nEventsPerJob':dict((step,nEventsPerJob) for step in StepExecution.STEPS)}) + task_config.update({'project_mode':slice['projectmode']}) + step_name = step_from_tag(slice['ctag']) + sexec = dict(status='NotChecked', priority=int(slice['priority']), + input_events=int(slice['totalevents'])) + st_sexec_list.append({'step_name': step_name, 'tag': slice['ctag'], 'step_exec': sexec, + 'memory': slice['ram'], + 'formats': slice['formats'], + 'task_config':task_config}) + spreadsheet_dict.append({'input_dict': irl, 'step_exec_dict': st_sexec_list}) + return spreadsheet_dict def dpd_form_prefill(form_data, request): spreadsheet_dict = [] + output_dict = {} + error_message = '' try: if form_data.get('excellink'): _logger.debug('Try to read data from %s' % form_data.get('excellink')) @@ -232,9 +264,12 @@ def dpd_form_prefill(form_data, request): if form_data.get('excelfile'): file_obj = request.FILES['excelfile'].read().split('\n') _logger.debug('Try to read data from %s' % form_data.get('excelfile')) + elif form_data.get('hidden_json_slices'): + spreadsheet_dict = parse_json_slice_dict(form_data.get('hidden_json_slices')) + if not spreadsheet_dict: + conf_parser = ConfigParser() + output_dict = conf_parser.parse_config(file_obj,['formats']) - conf_parser = ConfigParser() - output_dict = conf_parser.parse_config(file_obj) except Exception, e: _logger.error('Problem with data gathering %s' % e) error_message = str(e) @@ -258,39 +293,38 @@ def dpd_form_prefill(form_data, request): form_data['energy_gev'] = 8000 if not form_data.get('provenance'): form_data['provenance'] = 'test' + if not spreadsheet_dict: + task_config = {} + if 'events_per_job' in output_dict: + nEventsPerJob = output_dict['events_per_job'][0] + task_config.update({'nEventsPerJob':dict((step,nEventsPerJob) for step in StepExecution.STEPS)}) + if 'project_mode' in output_dict: + project_mode = output_dict['project_mode'][0] + task_config.update({'project_mode':project_mode}) + if 'ds' in output_dict: + formats = [] + for index,formats_count in enumerate(output_dict.get('formats_count_list', [None])): + formats+=[output_dict['formats'][index]]*formats_count + if len(formats)!=len(output_dict['ds']): + error_message = 'ds and format lenght do not match' + return {}, error_message + for slice_index, ds in enumerate(output_dict['ds']): + st_sexec_list = [] + irl = dict(slice=slice_index, brief=' ', comment=output_dict.get('comment', [''])[0], dataset=ds, + input_data=output_dict.get('joboptions', [''])[0], + project_mode=output_dict.get('project_mode', [''])[0], + priority=int(output_dict.get('priority', [0])[0]), + input_events=int(output_dict.get('total_num_genev', [-1])[0])) + if 'tag' in output_dict: + step_name = step_from_tag(output_dict['tag'][0]) + sexec = dict(status='NotChecked', priority=int(output_dict.get('priority', [0])[0]), + input_events=int(output_dict.get('total_num_genev', [-1])[0])) + st_sexec_list.append({'step_name': step_name, 'tag': output_dict['tag'][0], 'step_exec': sexec, + 'memory': output_dict.get('ram', [None])[0], + 'formats': formats[slice_index], + 'task_config':task_config}) + spreadsheet_dict.append({'input_dict': irl, 'step_exec_dict': st_sexec_list}) - task_config = {} - if 'events_per_job' in output_dict: - nEventsPerJob = output_dict['events_per_job'][0] - task_config.update({'nEventsPerJob':dict((step,nEventsPerJob) for step in StepExecution.STEPS)}) - if 'project_mode' in output_dict: - project_mode = output_dict['project_mode'][0] - task_config.update({'project_mode':project_mode}) - if 'ds' in output_dict: - if len(output_dict.get('formats', [None]))>1 and len(output_dict.get('formats', [None]))!=len(output_dict['ds']): - error_message = 'ds and format lenght do not match' - return {}, error_message - if len(output_dict.get('formats', [None]))==1: - formats = [(output_dict.get('formats', [None])[0])]*len(output_dict['ds']) - else: - formats = output_dict.get('formats', [None]) - for slice_index, ds in enumerate(output_dict['ds']): - st_sexec_list = [] - irl = dict(slice=slice_index, brief=' ', comment=output_dict.get('comment', [''])[0], dataset=ds, - input_data=output_dict.get('joboptions', [''])[0], - project_mode=output_dict.get('project_mode', [''])[0], - priority=int(output_dict.get('priority', [0])[0]), - input_events=int(output_dict.get('total_num_genev', [-1])[0])) - if 'tag' in output_dict: - step_name = step_from_tag(output_dict['tag'][0]) - sexec = dict(status='NotChecked', priority=int(output_dict.get('priority', [0])[0]), - input_events=int(output_dict.get('total_num_genev', [-1])[0])) - st_sexec_list.append({'step_name': step_name, 'tag': output_dict['tag'][0], 'step_exec': sexec, - 'memory': output_dict.get('ram', [None])[0], - 'formats': formats[slice_index], - 'task_config':task_config}) - spreadsheet_dict.append({'input_dict': irl, 'step_exec_dict': st_sexec_list}) - error_message = '' if not spreadsheet_dict: error_message= 'No "ds" data founnd in file.' _logger.debug('Gathered data: %s' % spreadsheet_dict) @@ -305,11 +339,17 @@ def reprocessing_form_prefill(form_data, request): file_name = open_tempfile_from_url(form_data['excellink'], 'txt') with open(file_name) as open_file: file_obj = open_file.read().split('\n') - if form_data.get('excelfile'): + conf_parser = ConfigParser() + output_dict = conf_parser.parse_config(file_obj) + elif form_data.get('excelfile'): file_obj = request.FILES['excelfile'].read().split('\n') _logger.debug('Try to read data from %s' % form_data.get('excelfile')) - conf_parser = ConfigParser() - output_dict = conf_parser.parse_config(file_obj) + conf_parser = ConfigParser() + output_dict = conf_parser.parse_config(file_obj) + elif form_data.get('hidden_json_slices'): + output_dict = parse_json_slice_dict(form_data.get('hidden_json_slices')) + + except Exception, e: _logger.error('Problem with data gathering %s' % e) eroor_message = str(e) @@ -420,7 +460,19 @@ def recursive_string_tag_tree_parsing(rest_list, current_parent, current_positio return tag_tree_dict, current_parent, current_position - +@csrf_protect +def find_datasets_by_pattern(request): + if request.method == 'POST': + data = request.body + input_dict = json.loads(data) + dataset_pattern = input_dict['datasetPattern'] + if dataset_pattern[-1] != '*' or dataset_pattern[-1] != '/': + dataset_pattern+='*' + return_list = find_dataset_events(dataset_pattern) + results = {} + results.update({'success':True,'data':return_list}) + return HttpResponse(json.dumps(results), content_type='application/json') + pass #TODO: Change it to real dataset workflow @@ -473,7 +525,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone if form.is_valid(): # Process the data from request prefill form - if form.cleaned_data.get('excellink') or form.cleaned_data.get('excelfile'): + if (form.cleaned_data.get('excellink') or form.cleaned_data.get('excelfile')) or form.cleaned_data.get('hidden_json_slices'): file_dict, error_message = form_prefill(form.cleaned_data, request) if error_message != '': # recreate prefill form with error message @@ -513,11 +565,24 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone #TODO: Waiting message #TODO: One commission file_dict = request.session['file_dict'] + form2 = TRequestCreateCloneConfirmation(request.POST, request.FILES) + if not form2.is_valid(): + inputlists = [x['input_dict'] for x in file_dict] + # store data from prefill form to http request + return render(request, 'prodtask/_previewreq.html', { + 'active_app': 'mcprod', + 'pre_form_text': title, + 'form': form2, + 'submit_url': submit_url, + 'url_args': rid, + 'parent_template': 'prodtask/_index.html', + 'inputLists': inputlists + }) del request.session['file_dict'] longdesc = form.cleaned_data.get('long_description', '') cc = form.cleaned_data.get('cc', '') del form.cleaned_data['long_description'], form.cleaned_data['cc'], form.cleaned_data['excellink'], \ - form.cleaned_data['excelfile'] + form.cleaned_data['excelfile'], form.cleaned_data['hidden_json_slices'] if 'reqid' in form.cleaned_data: del form.cleaned_data['reqid'] # if 'tag_hierarchy' in form.cleaned_data: @@ -595,7 +660,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone return HttpResponseRedirect(reverse('prodtask:request_table')) return HttpResponseRedirect(reverse('prodtask:input_list_approve',args=(req.reqid,))) else: - return render(request, 'prodtask/_form.html', { + return render(request, 'prodtask/_requestform.html', { 'active_app': 'mcprod', 'pre_form_text': title, 'form': form, @@ -603,6 +668,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone 'url_args': rid, 'parent_template': 'prodtask/_index.html', }) + # GET request else: @@ -620,7 +686,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone _logger.debug("Start request creation ") form = TRequestCreateCloneForm() # Create prefill form - return render(request, 'prodtask/_form.html', { + return render(request, 'prodtask/_requestform.html', { 'active_app': 'mcprod', 'pre_form_text': title, 'form': form, diff --git a/atlas/prodtask/templates/prodtask/_requestform.html b/atlas/prodtask/templates/prodtask/_requestform.html index ef6299ec..7bcba0cd 100644 --- a/atlas/prodtask/templates/prodtask/_requestform.html +++ b/atlas/prodtask/templates/prodtask/_requestform.html @@ -5,6 +5,9 @@ {% block extra_css %} {{ block.super }} + {% endblock %} @@ -33,12 +36,32 @@ event.preventDefault(); var nextNumber = parseInt($('#sliceNumber').html()); var cloneForm = $('#form0').clone().prop({'id':"form"+nextNumber.toString()}); + var idForm = "form"+nextNumber.toString(); cloneForm.children("legend").html("Slice #"+(nextNumber).toString()); $('#formSet').append(cloneForm); $('#sliceNumber').html((nextNumber+1).toString()); + $("#"+idForm+" #searchDatasetButton0").prop({'id':"searchDatasetButton"+nextNumber.toString()}); + $("#" +idForm+ " .datasetTBody").html(""); + $("#" +idForm+ " .datasetTBody").prop({'id':"datasetsTable"+nextNumber.toString()}); + $("#" +idForm+ " #datasetPattern0").val(''); + $("#" +idForm+ " #datasetPattern0").prop({'id':"datasetPattern"+nextNumber.toString()}); bindOnChangeToFieldSet(); fillSlicesField(); } + var csrftoken = $.cookie('csrftoken'); + function csrfSafeMethod(method) { + // these HTTP methods do not require CSRF protection + return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method)); + } + $.ajaxSetup({ + crossDomain: false, // obviates need for sameOrigin test + beforeSend: function(xhr, settings) { + if (!csrfSafeMethod(settings.type)) { + xhr.setRequestHeader("X-CSRFToken", csrftoken); + } + } + }); + $(document).ready(function(){ bindOnChangeToFieldSet(); @@ -47,6 +70,7 @@ $('.storeInput').bind('change',function(){fillSlicesField();}); } + function fillSlicesField(){ var resultObject = {}; $(".sliceFiledset").each(function(){ @@ -66,20 +90,42 @@ } resultObject[sliceNumber] = sliceObject; }); - console.log(resultObject); $("#{{ form.hidden_json_slices.auto_id }}").val($.toJSON(resultObject)); } function findDatasets(currentButton){ - console.log(currentButton); - console.log($(currentButton).parentsUntil(".sliceFiledset")); - $(currentButton).parentsUntil(".sliceFiledset").find("datasetTBody").html(''); + var sliceNumber = currentButton.slice("searchDatasetButton".length); + sendData = {'datasetPattern':$("#datasetPattern" + sliceNumber).val()} + $( "#waitingDialog" ).dialog(); + $.ajax({ + url: "{% url 'prodtask:find_datasets_by_pattern' %}", + type: 'POST', + contentType: 'application/json; charset=utf-8', + data: $.toJSON(sendData), + dataType: 'text', + success: function(data,status) { + $( "#waitingDialog" ).dialog('close'); + if(status){ + var datasets = $.evalJSON(data).data; + $("#datasetsTable" + sliceNumber).html(''); + for(var i=0;i'+ + datasets[i].dataset_name+''+datasets[i].events+ + ' '); + } + bindOnChangeToFieldSet(); + } + } + }); + return false; } - + {% if error_message %} ERROR: {{ error_message }} {% endif %} @@ -108,13 +154,13 @@
-
-
- +
+
+ Find datasets
@@ -127,17 +173,7 @@ - - - mc14_8TeV.117050.PowhegPythia_P2011C_ttbar.merge.AOD.e1727_s1933_s1911_r5591_r5625 - 30000 - - - - mc14_8TeV.117050.PowhegPythia_P2011C_ttbar.merge.AOD.e1727_s1933_s1911_r5591_r5625 - 30000 - - +
@@ -153,7 +189,7 @@
-
+
@@ -162,12 +198,17 @@ +
+
+
-
diff --git a/atlas/prodtask/urls.py b/atlas/prodtask/urls.py index 651b3479..de6bb878 100644 --- a/atlas/prodtask/urls.py +++ b/atlas/prodtask/urls.py @@ -26,8 +26,8 @@ url(r'^request_clone/(?P\d+)/$', 'atlas.prodtask.request_views.request_clone', name='request_clone'), url(r'^request_update/(?P\d+)/$', 'atlas.prodtask.request_views.request_update', name='request_update'), url(r'^request_create/$', 'atlas.prodtask.request_views.request_create', name='request_create'), - - + url(r'^find_datasets_by_pattern/$', 'atlas.prodtask.request_views.find_datasets_by_pattern', name='find_datasets_by_pattern'), + url(r'^dpd_request_create/$', 'atlas.prodtask.request_views.dpd_request_create', name='dpd_request_create'), url(r'^reprocessing_request_create/$', 'atlas.prodtask.request_views.reprocessing_request_create', name='reprocessing_request_create'), diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index fda6a4fa..1586cec4 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -9,10 +9,9 @@ from django.views.decorators.csrf import csrf_protect from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse -from .ddm_api import DDM +from ..prodtask.ddm_api import find_dataset_events from .request_views import fill_dataset -from ..getdatasets.models import ProductionDatasetsExec, TaskProdSys1 -from ..settings import dq2client as dq2_settings + import core.datatables as datatables @@ -290,6 +289,7 @@ def form_skipped_slice(slice, reqid): return {} return {} + @csrf_protect def find_input_datasets(request, reqid): if request.method == 'POST': @@ -382,43 +382,11 @@ def find_skipped_dataset(DSID,job_option,tags,data_type): for base_value in ['mc','valid']: dataset_pattern = base_value+"%"+str(DSID)+"%"+job_option+"%"+data_type+"%"+"%".join(tags)+"%" _logger.debug("Search dataset by pattern %s"%dataset_pattern) - # datasets = ProductionDatasetsExec.objects.extra(where=['name like %s'], params=[dataset_pattern]).exclude(status__iexact = u'deleted') - # for dataset in datasets: - # task = TaskProdSys1.objects.get(taskid=dataset.taskid) - # return_list.append({'dataset_name':dataset.name,'events':str(task.total_events)}) - # _logger.debug("Find dataset: %s"%str(return_list[-1])) - ddm = DDM(dq2_settings.PROXY_CERT,dq2_settings.RUCIO_ACCOUNT) - datasets_containers = ddm.find_dataset(dataset_pattern.replace('%','*')) - containers = [x for x in datasets_containers if x[-1] == '/' ] - datasets = [x for x in datasets_containers if x not in containers ] - for container in containers: - event_count = 0 - is_good = True - datasets_in_container = ddm.dataset_in_container(container) - for dataset_name in datasets_in_container: - if dataset_name in datasets: - datasets.remove(dataset_name) - try: - dataset = ProductionDatasetsExec.objects.get(name=dataset_name) - task = TaskProdSys1.objects.get(taskid=dataset.taskid) - if (task.status not in ['aborted','failed','lost']): - event_count += task.total_events - else: - is_good = False - except: - is_good = False - if is_good and (event_count>0): - return_list.append({'dataset_name':container,'events':str(event_count)}) - for dataset_name in datasets: - try: - task = TaskProdSys1.objects.get(taskname=dataset_name) - if (task.status not in ['aborted','failed','lost']): - return_list.append({'dataset_name':dataset_name,'events':str(task.total_events)}) - except: - pass + return_list.append(find_dataset_events(dataset_pattern)) return return_list + def find_old_double_trf(tags): double_trf_tags = set() for tag in tags: From 76c36e5aea33a6bb05f5cd62c9d92b7abd77af33 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 28 Aug 2014 16:07:48 +0200 Subject: [PATCH 08/93] Add prodsys2 dataset to search --- atlas/prodtask/views.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index 8b61c440..7d9e73d7 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -412,7 +412,12 @@ def find_skipped_dataset(DSID,job_option,tags,data_type): if (task.status not in ['aborted','failed','lost']): return_list.append({'dataset_name':dataset_name,'events':str(task.total_events)}) except: - pass + try: + dataset_in_db = ProductionDataset.objects.get(name=dataset_name) + if dataset_in_db.status == 'done': + return_list.append({'dataset_name':dataset_name,'events':str(-1)}) + except: + pass return return_list From 5da5f74c0a2354a3ebc7bcc7dddeb86f1a81f2ea Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 28 Aug 2014 23:40:43 +0200 Subject: [PATCH 09/93] fix for dataset search --- atlas/prodtask/ddm_api.py | 8 +++++++- atlas/prodtask/views.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/atlas/prodtask/ddm_api.py b/atlas/prodtask/ddm_api.py index defee7d2..22d74908 100644 --- a/atlas/prodtask/ddm_api.py +++ b/atlas/prodtask/ddm_api.py @@ -1,6 +1,7 @@ import logging import os +from ..prodtask.models import ProductionDataset from ..getdatasets.models import ProductionDatasetsExec, TaskProdSys1 from ..settings import dq2client as dq2_settings @@ -37,7 +38,12 @@ def find_dataset_events(dataset_pattern): if (task.status not in ['aborted','failed','lost']): return_list.append({'dataset_name':dataset_name,'events':str(task.total_events)}) except: - pass + try: + dataset_in_db = ProductionDataset.objects.get(name=dataset_name) + if dataset_in_db.status == 'done': + return_list.append({'dataset_name':dataset_name,'events':str(-1)}) + except: + pass return return_list class DDM(object): diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index 3a21b926..560444c2 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -382,7 +382,7 @@ def find_skipped_dataset(DSID,job_option,tags,data_type): for base_value in ['mc','valid']: dataset_pattern = base_value+"%"+str(DSID)+"%"+job_option+"%"+data_type+"%"+"%".join(tags)+"%" _logger.debug("Search dataset by pattern %s"%dataset_pattern) - return_list.append(find_dataset_events(dataset_pattern)) + return_list += find_dataset_events(dataset_pattern) return return_list From 77865c720291dfa5f48bde35c4b0b0b0e76b910a Mon Sep 17 00:00:00 2001 From: Alexei Klimentov Date: Fri, 29 Aug 2014 13:32:57 +0200 Subject: [PATCH 10/93] minor changes --- atlas/prodsys2/deft/deft_handling.py | 219 ++++++++++++++++++++++----- 1 file changed, 178 insertions(+), 41 deletions(-) diff --git a/atlas/prodsys2/deft/deft_handling.py b/atlas/prodsys2/deft/deft_handling.py index bde68012..8c057e6e 100644 --- a/atlas/prodsys2/deft/deft_handling.py +++ b/atlas/prodsys2/deft/deft_handling.py @@ -17,8 +17,11 @@ # June 26, 2014. SSO is modified (DG) # Changes in project name insert part # July 17, 2014. Task submission info (e-mails) +# July 28, 2014. Add link to monitoring page and modify tasks grouping in the report +# Aug 5, 2014. e-mail notification, final version +# Aug 11, 2014. GP daily reports # -# Last Edit : July 17, 2014 ak +# Last Edit : Aug 11, 2014 ak # import re @@ -112,6 +115,9 @@ def usage() : print usage.__doc__ + +#--- service subs + def execCmd(cmd,flag) : (s,o) = commands.getstatusoutput(cmd) if s != 0 and flag == 1: @@ -247,6 +253,24 @@ def JediTaskCmd(cmd,task_id,priority) : return status + +def sendEmail(email,subject, buf) : + + msg = "To:%s \n"%(email) + msg += "Subject : %s"%(subject) + msg += "\n--- begin \n" + msg += "This message was generated automatically. Pls do not reply" + msg += buf + msg += "Timestamp : %s"%time.ctime() + msg += "\n--- ends here \n" + MAIL = "/usr/lib/sendmail" + print msg + p = os.popen("%s -t" % MAIL,'w') + p.write(msg) + exitcode = p.close() + +#--- end of service subs + def obsoleteTaskState(task_id, dbupdate) : # set task state to 'obsolete' and update datasets states accordingly error = 0 @@ -336,24 +360,38 @@ def generateProdSysReport(flag) : # # get information about ProdSys2 tasks and send report # - TR_ID_MIN = 4000000 - MC_flag = 'MC' - Repro_flag= 'Repro' - GP_flag = 'GP' - + TR_ID_MIN = 4000000 + MC_flag = 'MC' + Repro_flag = 'Repro' + GP_flag = 'GP' + send_email_flag = 1 + header_msg = '' + info_msg = '' + warning_msg = '' + alarm_msg = '' + timenow = int(time.time()) findProcess('deft_handling','generate-prodsys-report','Quit') + sql_provenance = '' + if flag == MC_flag : + sql_provenance = \ + "AND provenance ='%s' AND (project LIKE '%s' OR project LIKE '%s') "%('AP','mc%','valid%') + elif flag == GP_flag : + sql_provenance = "AND provenance = '%s' AND project NOT LIKE '%s' "%('GP','user%') + else : + print "ERROR. Unknown report flag : ",flag,". Quit" + sys.exit(1) # connect to Oracle (pdb,dbcur,deftDB) = connectDEFT('R') # get information about tasks submitted during last 24h t_table_DEFT = "%s.%s"%(deftDB,deft_conf.daemon['t_production_task']) - sql = "SELECT taskname, taskid, status, provenance, priority, submit_time,timestamp FROM %s "%(t_table_DEFT) + sql = "SELECT taskname, taskid, status, provenance, priority, submit_time,timestamp FROM %s "%\ + (t_table_DEFT) sql+= "WHERE (submit_time > current_timestamp -1 OR timestamp > current_timestamp - 1) " - if flag == 'MC' : - sql += "AND provenance ='%s' AND (project LIKE '%s' OR project LIKE '%s') "%('AP','mc%','valid%') + sql+= sql_provenance sql+= "ORDER by taskid" - #--print sql + print sql tids = DButils.QueryAll(pdb,sql) t_tid_MIN = 400000*1000 @@ -362,6 +400,7 @@ def generateProdSysReport(flag) : t_start_LAST= -1 ntasks = len(tids) + print "Total tasks : ",ntasks for t in tids : t_name = t[0] t_id = t[1] @@ -382,35 +421,40 @@ def generateProdSysReport(flag) : if t_start_LAST < t_submit : t_start_LAST = t_submit fmt = '%Y-%m-%d %H:%M:%S %Z' - t_start_LAST = t_start_LAST.strftime(fmt) - t_time_LAST = t_time_LAST.strftime(fmt) + if ntasks == 0 : + t_start_last = "NO submission in the last 24h" + t_time_last = "NO database update within last 24h" + else : + t_start_LAST = t_start_LAST.strftime(fmt) + t_time_LAST = t_time_LAST.strftime(fmt) - print "Production Activity : ",time.ctime(timenow-24*60*60),'-',time.ctime(timenow) - print "Tasks states changed: ",ntasks - print "Tasks IDs : ",t_tid_MIN,'-',t_tid_MAX - print "Latest task submit time : ",t_start_LAST - print "Latest database update time : ",t_time_LAST + header_msg = "Production Activity : %s - %s \n"%(time.ctime(timenow-24*60*60),time.ctime(timenow)) + header_msg+= "Tasks states changed: %s \n"%(ntasks) + header_msg+= "Tasks IDs : %s - %s \n"%(t_tid_MIN,t_tid_MAX) + header_msg+= "Latest task submit time : %s \n"%(t_start_LAST) + header_msg+= "Latest database update time : %s \n"%(t_time_LAST) + print header_msg # get information about potentially problematic tasks sql = "SELECT taskname, taskid, status, provenance, priority, submit_time,timestamp FROM %s "%(t_table_DEFT) sql+= "WHERE taskid > %s "%(TR_ID_MIN) sql+= "AND status IN ('assigning','registered','pending') " - if flag == 'MC' : - sql += "AND provenance ='%s' AND (project LIKE '%s' OR project LIKE '%s') "%('AP','mc%','valid%') + sql+= sql_provenance sql+= "ORDER by taskid" #--print sql wtids = DButils.QueryAll(pdb,sql) - print " " - print " " - print "Tasks taken by JEDI, but not running yet. #Tasks : ",len(wtids) - + task_info_msg = '' + task_info_msg = "%s %s"%('\n','\n') + task_warning_msg = task_info_msg + task_alarm_msg = task_info_msg + task_info_msg += "%s : %s \n"%( "Tasks taken by JEDI, but not running yet. #Tasks",len(wtids)) + print task_info_msg sql = "SELECT taskname, taskid, status, provenance, priority, submit_time,timestamp FROM %s "%(t_table_DEFT) sql+= "WHERE (submit_time > current_timestamp -1 OR timestamp > current_timestamp - 1) " sql+= "AND status IN ('broken','failed') " - if flag == 'MC' : - sql += "AND provenance ='%s' AND (project LIKE '%s' OR project LIKE '%s') "%('AP','mc%','valid%') + sql+= sql_provenance sql+= "ORDER by taskid" #--print sql ftids = DButils.QueryAll(pdb,sql) @@ -419,8 +463,13 @@ def generateProdSysReport(flag) : # get info from JEDI for problematic tasks (pdb,dbcur,deftDB) = connectJEDI('R','pandamon') t_table_PANDAMON = "%s"%(deft_conf.daemon['t_jedi_tasks']) - - print '%-8s %6s %-10s %20s'%('Task ID','Prio','Status','JEDI Info') + task_url_prefix = 'http://bigpanda.cern.ch/task/' + JEDI_waiting_tasks = [] + header_line = '%-60s %8s %5s %12s %-40s %10s \n'%\ + ('Task Name', 'Task ID','Prio','Status','URL','JEDI Info') + print header_line + task_waiting_list = '' + task_jedi_warning_list = '' for t in wtids : t_name = t[0] t_id = t[1] @@ -429,18 +478,53 @@ def generateProdSysReport(flag) : t_priority = t[4] t_submit = t[5] t_time = t[6] + jj = t_name.split('.') + t_name_short = '' + i = 0 + for j in jj : + if i == 2 : + t_name_short += "*." + else : + t_name_short += "%s."%(j) + i += 1 + t_name_short = t_name_short[0:(len(t_name_short)-1)] + task_url ="%s%s"%(task_url_prefix,t_id) sql = "SELECT errordialog FROM %s WHERE jeditaskid=%s"%(t_table_PANDAMON,t_id) #--print sql msg = DButils.QueryAll(pdb,sql) - print t_name - print '%-8s %6s %-10s %20s'%(t_id,t_priority,t_status,msg[0][0]) + if msg[0][0] != None : + str = '%-60s %8s %5s %12s %-40s %s \n'%\ + (t_name_short,t_id,t_priority,t_status,task_url,msg[0][0]) + task_jedi_warning_list += str + print str + else : + # JEDI msg == None ; just waiting + str = "%-60s %8s %5s %12s %-40s %5s \n"%\ + (t_name_short, t_id,t_priority,t_status,task_url,msg[0][0]) + JEDI_waiting_tasks.append(str) + task_waiting_list += str + task_info_msg += header_line + task_info_msg += task_waiting_list + task_info_msg += "\n \n" + + + str = "Tasks not Processed by JEDI yet" + task_warning_msg += str + header_line = '%-60s %8s %4s %10s %-40s %10s \n'%\ + ('Task Name', 'Task ID','Prio','Status','URL','JEDI Info') + task_warning_msg += header_line + task_warning_msg += task_jedi_warning_list + task_warning_msg += "\n \n" + print task_jedi_warning_list + #for s in JEDI_waiting_tasks : + # print s + #print " " + #print " " + task_alarm_msg = "Tasks broken/failed (last 24h). #Tasks : %s \n"%(len(ftids)) + task_alarm_msg += '%-60s %8s %4s %10s %-40s %10s'%\ + ('Task Name', 'Task ID','Prio','Status','URL','JEDI Info') + print task_alarm_msg - - print " " - print " " - print "Tasks broken/failed (last 24h). #Tasks : ",len(ftids) - - print '%-8s %6s %-10s %20s'%('Task ID','Prio','Status','JEDI Info') for t in ftids : t_name = t[0] t_id = t[1] @@ -452,11 +536,57 @@ def generateProdSysReport(flag) : sql = "SELECT errordialog FROM %s WHERE jeditaskid=%s"%(t_table_PANDAMON,t_id) #--print sql msg = DButils.QueryAll(pdb,sql) - print t_name - print '%-8s %6s %-10s %20s '%(t_id,t_priority,t_status,msg[0][0]) + t_name_short = '' + i = 0 + for j in jj : + if i == 2 : + t_name_short += "*." + else : + t_name_short += "%s."%(j) + i += 1 + t_name_short = t_name_short[0:(len(t_name_short)-1)] + task_url ="%s%s"%(task_url_prefix,t_id) + str= '%-60s,%8s %4s %10s %-40s %s '%(t_name_short,t_id,t_priority,t_status,task_url,msg[0][0]) + task_alarm_msg += str + print str DButils.closeDB(pdb,dbcur) - + # send e-mail + email_addr = 'XYZ' + if flag == MC_flag : email_addr = "jose.enrique.garcia@cern.ch" + if flag == GP_flag : email_addr = "nurcan@uta.edu" + + if send_email_flag == 1 : + if email_addr == 'XYZ' : + print "WARNING : unknown e-mail address or flag" + else : + email_addr += ",alexei.klimentov@cern.ch" + timestamp_email = int(time.time()) + subject = "ProdSys2 report (%s). "%(flag) + # info e-mail + subject+= "Info" + subject = "%s %s"%(subject,time.ctime(timestamp_email)) + mail_body = "\n You receive this e-mail because you are listed as %s Production Manager \n"%(flag) + mail_body += "\n\n" + mail_body += task_info_msg + sendEmail(email_addr,subject,mail_body) + # warning e-mail + subject+= "Warning" + subject = "%s %s"%(subject,time.ctime(timestamp_email)) + mail_body = "\n You receive this e-mail because you are listed as %s Production Manager \n"%(flag) + mail_body += "\n\n" + mail_body += task_warning_msg + sendEmail(email_addr,subject,mail_body) + #alarm e-mail + subject+= "Alarm" + subject = "%s %s"%(subject,time.ctime(timestamp_email)) + mail_body = "\n You receive this e-mail because you are listed as %s Production Manager \n"%(flag) + mail_body += "\n\n" + mail_body += task_alarm_msg + # + print mail_body + # + sendEmail(email_addr,subject,mail_body) def checkAbortedTasks() : # @@ -580,7 +710,7 @@ def insertJediTasksJSON(user_task_list): jedi_task_names = ['userName','taskName','taskPriority','vo'] # connect to Oracle - (pdb,dbcur,deftDB) = connectJEDI('R') + (pdb,dbcur,deftDB) = connectJEDI('R','jedi') t_table_DEFT = "%s.%s"%(deftDB,deft_conf.daemon['t_production_task']) t_table_JEDI = "%s.%s"%(deftDB,deft_conf.daemon['t_task']) @@ -890,7 +1020,7 @@ def synchronizeJediDeftTasks() : DButils.closeDB(pdb,dbcur) print "%s DEFT tasks match to the criteria"%(len(tasksDEFT)) - (pdb,dbcur,deftDB) = connectJEDI('R') + (pdb,dbcur,deftDB) = connectJEDI('R','jedi') sql_select = "SELECT taskid, status,total_done_jobs,submit_time, start_time, prodsourcelabel," sql_select+= "priority,current_priority, taskname, total_req_jobs, total_events " sql = sql_select @@ -1207,6 +1337,7 @@ def main() : task_id = -1 # Task ID task_prio = -1 # Task priority task_state='unknown' # Task state + report_flag = 'XYZ' # generate report for 'MC', 'GP' et al for o, a in opts: if o in ("-h","--help") : @@ -1222,6 +1353,8 @@ def main() : task_id = int(a.strip()) elif o == "-p" : task_prio = int(a.strip()) + elif o == '-f' : + report_flag = a.strip() elif o == "--change-task-state" : changeTaskStateF = True elif o == '--finish-task' : @@ -1280,8 +1413,12 @@ def main() : status = checkAbortedTasks() if generateProdSysReportF == True : + if report_flag == 'XYZ' : + print "ERROR : unknown provenance. Report flag : ", report_flag + else : findProcess('deft_handling','generate-prodsys-report','Quit') - status = generateProdSysReport('MC') + print "Generate daily report for ",report_flag,' production' + status = generateProdSysReport(report_flag) if finishTaskF == True : if task_id < 0 : From a40f2090aa475cf75afccb073ac2a8bf7edf54f3 Mon Sep 17 00:00:00 2001 From: alexei Date: Fri, 29 Aug 2014 13:53:44 +0200 Subject: [PATCH 11/93] minor changes --- atlas/prodsys2/deft/add_containter.py | 260 ++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 atlas/prodsys2/deft/add_containter.py diff --git a/atlas/prodsys2/deft/add_containter.py b/atlas/prodsys2/deft/add_containter.py new file mode 100644 index 00000000..72a7ed16 --- /dev/null +++ b/atlas/prodsys2/deft/add_containter.py @@ -0,0 +1,260 @@ +# A.Klimentov May 13, 2014 +# +# register productiuon containers and add datasets +# +# Last Edit : Jun 11, 2014 ak +# + +import re +import sys +import os +import getopt +import commands +import datetime +import time +import cx_Oracle + +import deft_conf +import deft_pass +import deft_user_list + +import DButils + +# +from dq2.clientapi.DQ2 import DQ2 + +from dq2.common.DQConstants import DatasetState +from dq2.common.DQException import DQBackendException + +from dq2.common.client.DQClientException import DQInternalServerException + + +from dq2.common.dao.DQDaoException import * + +from dq2.common import get_dict_item + +from dq2.location.DQLocationConstants import LocationState +from dq2.clientapi.cli.SetMetaDataAttribute import SetMetaDataAttribute + +# +os.environ['RUCIO_ACCOUNT'] = 'alexei' +# +dq2api = DQ2() + + +verbose = False + +task_finish_states = 'done,finish,failed,obsolete,aborted' +task_aborted_states = 'aborted,failed,obsolete' +datasets_deletion_states = 'toBeDeleted,Deleted' + +user_task_label = 'user' # JEDI prodsourcelabel parameter + +JEDI_datasets_final_statesL = ['aborted','broken','done','failed','partial','ready'] +DEFT_datasets_done_statesL = ['done'] +DEFT_datasets_final_statesL = \ + ['aborted','broken','done','failed','deleted,toBeDeleted,toBeErased,waitErased,toBeCleaned,waitCleaned'] +DEFT_datasets_postproduction_states = 'deleted,toBeErased,waitErased,toBeCleaned,waitCleaned' +DEFT_tasks_abort_statesL = ['aborted','broken','failed'] +Request_update_statesL = ['registered','running','done','finished','failed'] + +MIN_DEFT_TASK_ID = 4000000 + +#synch intervals (hours) +REQUEST_SYNCH_INTERVAL = 12000 +TASK_SYNCH_INTERVAL = 12000 +DATASET_SYNCH_INTERVAL = 72 + + + +def connectDEFT(flag) : + +# +# connect in RO mode if flag = 'R' +# connect in Update mode = 'W' + + error = 0 + dbname = deft_conf.daemon['deftDB_INTR'] + deftDB = deft_conf.daemon['deftDB_host'] + if flag == 'W' : + dbuser = deft_conf.daemon['deftDB_writer'] + dbpwd = deft_pass.deft_pass_intr['atlas_deft_w'] + else : + dbuser = deft_conf.daemon['deftDB_reader'] + dbpwd = deft_pass.deft_pass_intr['atlas_deft_r'] + (pdb,dbcur) = DButils.connectDEFT(dbname,dbuser,dbpwd) + + return pdb, dbcur, deftDB + + + + + +def addTidDatasetToContainer(): +# +# get list of TID datasets +# register container if it isn't registered yet +# add TID dataset to Container +# + + timeInterval = DATASET_SYNCH_INTERVAL # hours + nDQ2ErrorsInRow = 0 + nContainers = 0 + nDatasets = 0 + dbupdate = True + minTaskID = MIN_DEFT_TASK_ID + maxTaskID = minTaskID*10 + + DEFT_datasets_final_states = '' + for s in DEFT_datasets_done_statesL : + DEFT_datasets_final_states += "'%s',"%(s) + DEFT_datasets_final_states = DEFT_datasets_final_states[0:(len(DEFT_datasets_final_states)-1)] + + T0 = int(time.time()) + # connect to Oracle + (pdb,dbcur,deftDB) = connectDEFT('R') + + t_table_datasets_DEFT = "%s.%s"%(deftDB,deft_conf.daemon['t_production_dataset']) + t_table_containers_DEFT = "%s.%s"%(deftDB,deft_conf.daemon['t_production_container']) + + # get list of datasets + sql = "SELECT name, taskid, pr_id, status, phys_group, timestamp FROM %s "%(t_table_datasets_DEFT) + sql+= "WHERE TIMESTAMP > current_timestamp - %s AND taskid >= %s "%(timeInterval,MIN_DEFT_TASK_ID) + sql+= "AND (name NOT LIKE '%s' AND name NOT LIKE '%s') "%('user%','%.log.%') + sql+= "AND status in (%s) "%(DEFT_datasets_final_states) + sql+= "ORDER BY taskid" + print sql + dsets = DButils.QueryAll(pdb,sql) + # get min and max task ID + sql = "SELECT min(parent_tid), max(parent_tid) FROM %s "%(t_table_datasets_DEFT) + sql+= "WHERE TIMESTAMP > current_timestamp - %s AND taskid >= %s "%(timeInterval,MIN_DEFT_TASK_ID) + sql+= "AND (name NOT LIKE '%s' AND name NOT like '%s') "%('user%','%.log.%') + sql+= "AND status in (%s) "%(DEFT_datasets_final_states) + print sql + mimax = DButils.QueryAll(pdb,sql) + if len(mimax) : + for i in mimax : + minTaskID = i[0] + maxTaskID = i[1] + else : + print "Warning. Cannot find information for query" + # get list of containers + sql = "SELECT name, parent_tid, status, c_time FROM %s "%(t_table_containers_DEFT) + sql+= "WHERE parent_tid >= %s and parent_tid <= %s "%(minTaskID, maxTaskID) + sql+= "ORDER by parent_tid" + print sql + cntrs = DButils.QueryAll(pdb,sql) + + DButils.closeDB(pdb,dbcur) + + # prepare a list of containers to be registered + sql_update = [] + for ds in dsets : + dsname = ds[0] + d_tid = ds[1] + d_rid = ds[2] + d_status = ds[3] + d_phgroup= ds[4] + if dsname.find('.log') < 0 : + junk = dsname.split('_tid') + top_dsname = junk[0].strip() + cnt_name = "%s/"%(top_dsname) + print "Check containers list" + cnt_list_flag = 0 # container registered in database + ddm_list_flag = 0 # in DDM + reg_dset_flag = 0 # new dataset(s) added to the container + for cn in cntrs : + cname = cn[0] + c_tid = cn[1] + c_time= cn[3] + if c_tid == d_tid : + if cname == cnt_name : + print "Container %s found in database (task id = %s, registration time : %s)"%\ + (cname,c_tid,c_time) + cnt_list_flag = 1 + if c_tid > d_tid : + break + if cnt_list_flag != 1 : + print "Container %s NOT found in database "%(cnt_name) + print "Check DDM catalog : %s"%(cnt_name) + error = 0 + try : + ret = dq2api.listDatasets(cnt_name) + print ret + if len(ret) > 0 : ddm_list_flag = 1 + except : + print "ERROR - cannot execute : %s%s%s "%("ret = dq2api.listDatasets(",cnt_name,")") + error = 1 + if ddm_list_flag == 1 and error == 0 : + msg = "Container %s exists in DDM "%(cnt_name) + if cnt_list_flag == 0 : msg += ". Get meta-info and add it to the database" + if cnt_list_flag == 1 : msg += "and in database. Do nothing. Proceed to datasets registration in the container" + print msg + if ddm_list_flag == 0 : + print "Register container : %s (Container already registered : %s)"%(cnt_name,nContainers) + try : + dq2api.registerContainer(cnt_name) + nContainers += 1 + except : + error = 1 + print "Error dq2api.registerContainer(%s)"%(cnt_name) + print "do no update database (%s)"%(cnt_name) + if error == 1 : + print "Error in DDM part. quit" + sys.exit(1) + if error == 0 : + # get creation date + creationdate = dq2api.getMetaDataAttribute(cnt_name, ['creationdate',]) + c_time = creationdate['creationdate'] + ELEMENTS = [] + print "Register new elements in %s (%s)"%(cnt_name,dsname) + ELEMENTS.append(dsname) + try : + ret = dq2api.registerDatasetsInContainer(cnt_name,ELEMENTS) + nDatasets += len(ELEMENTS) + reg_dset_flag = 1 + except (DQException): + print "Warning : %s already has dataset : %s"%(cnt_name,dsname) + reg_dset_flag = 0 + except : + """fatal error... I increment the error but you can exit if you want""" + nDQ2ErrorsInRow +=1 + error = 1 + print "Fatal error in registerDatasetsInContainer. Quit " + sys.exit(1) + # form sql statetment + sql ='' + if d_rid == None or d_rid == 'None' : d_rid=-1 + if cnt_list_flag == 1 : + if reg_dset_flag == 1 : + sql = "UPDATE %s SET d_time = current_timestamp WHERE name = '%s'"%(t_table_containers_DEFT,cnt_name) + sql_update.append(sql) + else : + sql = "INSERT INTO %s VALUES"%(t_table_containers_DEFT) + if reg_dset_flag == 0 : + sql+= "('%s',%s,%s,'%s','%s',to_timestamp('%s','YYYY-MM-DD HH24:MI:SS'),to_timestamp('%s','YYYY-MM-DD HH24:MI:SS'),current_timestamp)"%(cnt_name,d_tid,d_rid,'registered',d_phgroup,c_time,c_time) + else : + sql+= "('%s',%s,%s,'%s','%s',to_timestamp('%s','YYYY-MM-DD HH24:MI:SS'),current_timestamp,current_timestamp)"%\ + (cnt_name,d_tid,d_rid,'registered',d_phgroup,c_time) + if len(sql) : sql_update.append(sql) + if dbupdate : + (pdb,dbcur,deftDB) = connectDEFT('W') + for sql in sql_update : + print "SQL update : ",sql + DButils.QueryUpdate(pdb,sql) + DButils.QueryCommit(pdb) + DButils.closeDB(pdb,dbcur) + else : + print "No database update" + Tf = int(time.time()) + dT = Tf - T0 + print time.ctime() + print "addTiddatasets. Container registered : %s, Datasets registered : %s"%(nContainers, nDatasets) + print "addTidDatasets. Total time : %s sec"%(dT) + +def main() : + + addTidDatasetToContainer() + +main() + From 4a78ba6c9f9acf236da57d4daf8fa303b5aa2a67 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 1 Sep 2014 15:44:04 +0200 Subject: [PATCH 12/93] fix forms for mc --- atlas/prodtask/forms.py | 2 ++ atlas/prodtask/request_views.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/forms.py b/atlas/prodtask/forms.py index 07ba6731..7cc3e1f4 100644 --- a/atlas/prodtask/forms.py +++ b/atlas/prodtask/forms.py @@ -40,6 +40,7 @@ class TRequestMCCreateCloneForm(TRequestCreateCloneConfirmation): project = ModelChoiceField(queryset=TProject.objects.all(),required=False) phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) campaign = CharField(required=False) + provenance = CharField(widget=forms.HiddenInput, required=False) class Meta: model = TRequest @@ -57,6 +58,7 @@ class TRequestDPDCreateCloneForm(TRequestCreateCloneConfirmation): phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) campaign = CharField(required=False) + class Meta: model = TRequest exclude = ['reqid'] diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index 5d0cc977..ee5474fb 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -582,7 +582,9 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone longdesc = form.cleaned_data.get('long_description', '') cc = form.cleaned_data.get('cc', '') del form.cleaned_data['long_description'], form.cleaned_data['cc'], form.cleaned_data['excellink'], \ - form.cleaned_data['excelfile'], form.cleaned_data['hidden_json_slices'] + form.cleaned_data['excelfile'] + if form.cleaned_data.get('hidden_json_slices'): + del form.cleaned_data['hidden_json_slices'] if 'reqid' in form.cleaned_data: del form.cleaned_data['reqid'] # if 'tag_hierarchy' in form.cleaned_data: From e91f6782152013f92ebf71a1afebd588bd1ca9a6 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 08:25:36 +0200 Subject: [PATCH 13/93] Added settings for proxy certificate --- atlas/auth/voms/backends.py | 8 ++------ atlas/auth/voms/interface.py | 7 +++++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/atlas/auth/voms/backends.py b/atlas/auth/voms/backends.py index bd5e3fa6..9185c25f 100644 --- a/atlas/auth/voms/backends.py +++ b/atlas/auth/voms/backends.py @@ -1,5 +1,3 @@ -import os - from django.conf import settings from django.contrib.auth.backends import ModelBackend from django.contrib.auth.models import Group @@ -23,7 +21,7 @@ class VomsBackend(ModelBackend): """ Adding VOMS-based authentication groups for user. """ def authenticate(self, request=None): - """ Chacking user against VOMS data and adding corresponding authentication groups. + """ Checking user against VOMS data and adding corresponding authentication groups. Parameters: request: Http request (HttpRequest). @@ -44,9 +42,7 @@ def authenticate(self, request=None): if not dn_map: return - os.environ["X509_USER_PROXY"] = settings.VOMS_PROXY_CERT - - options = VomsInterface.get_identity_options() + options = VomsInterface.get_identity_options(proxy_cert=settings.VOMS_PROXY_CERT) options.update(settings.VOMS_OPTIONS) voms = VomsInterface(options) diff --git a/atlas/auth/voms/interface.py b/atlas/auth/voms/interface.py index a4c84949..57eb6bc8 100644 --- a/atlas/auth/voms/interface.py +++ b/atlas/auth/voms/interface.py @@ -1,4 +1,4 @@ -#!/bin/env python +import os from VOMSAdmin.VOMSCommands import VOMSAdminProxy @@ -53,10 +53,13 @@ def get_user_nickname(self, dn, ca): return attributes.get("nickname") @staticmethod - def get_identity_options(voms_admin_path="/usr/bin/voms-admin"): + def get_identity_options(voms_admin_path="/usr/bin/voms-admin", proxy_cert=None): import imp voms_admin = imp.load_source("voms_admin", voms_admin_path) voms_admin.vlog = lambda msg: None + + if proxy_cert: + os.environ["X509_USER_PROXY"] = proxy_cert voms_admin.setup_identity() return voms_admin.options From c37671b27a1d8ae2b185ed56bb38a00bbb35d8b5 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 08:29:48 +0200 Subject: [PATCH 14/93] Model changed to work both with Oracle and Sqlite --- atlas/auth/voms/models.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/atlas/auth/voms/models.py b/atlas/auth/voms/models.py index 68d81b4c..a98f6bdc 100644 --- a/atlas/auth/voms/models.py +++ b/atlas/auth/voms/models.py @@ -4,10 +4,11 @@ from django.utils import timezone class VomsUser(models.Model): - username = models.CharField(max_length=60, db_column='username', primary_key=True) - dn = models.CharField(max_length=255, db_column='dn') - ca = models.CharField(max_length=255, db_column='ca') - added_on = models.DateTimeField(auto_now_add=True, db_column='added_on') + id = models.AutoField(db_column='ID', primary_key=True) + username = models.CharField(max_length=60, db_column='USERNAME', db_index=True) + dn = models.CharField(max_length=255, db_column='DN', db_index=True) + ca = models.CharField(max_length=255, db_column='CA') + added_on = models.DateTimeField(auto_now_add=True, db_column='ADDED_ON') def save(self, *args, **kwargs): if not self.added_on: @@ -17,5 +18,6 @@ def save(self, *args, **kwargs): class Meta: managed = True - db_table = u'"ATLAS_DEFT"."VOMS_USERS_MAP"' + db_table = u'VOMS_USERS_MAP' + app_label = 'auth' unique_together = (("username", "dn"),) From e25614652478431877137b779ec481df7bde916e Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 08:30:51 +0200 Subject: [PATCH 15/93] Added handling for possible wrong data from VOMS --- atlas/auth/voms/collector.py | 52 ++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/atlas/auth/voms/collector.py b/atlas/auth/voms/collector.py index b6199826..91492ff2 100644 --- a/atlas/auth/voms/collector.py +++ b/atlas/auth/voms/collector.py @@ -1,11 +1,18 @@ +#!/bin/env python + +import re from django.conf import settings from .interface import VomsInterface - from .models import VomsUser def run(): - options = VomsInterface.get_identity_options() + """ + Update VOMS mapping nickname <-> (DN, CA) in the database + :return: dictionary, 'added', 'removed' - records added/removed, + 'details' - more information on operations performed + """ + options = VomsInterface.get_identity_options(proxy_cert=settings.VOMS_PROXY_CERT) options.update(settings.VOMS_OPTIONS) voms = VomsInterface(options) @@ -15,16 +22,35 @@ def run(): for user_info in voms.list_users(): dn = user_info["DN"] ca = user_info["CA"] - nickname = voms.get_user_nickname(dn, ca) - + try: + nickname = voms.get_user_nickname(dn, ca) + except: + # TODO: log the error (e.g. user was removed during data collection) + continue + if not re.match(r"^\w+$", nickname): + # TODO: log the warning + continue if not voms_users.get(nickname): voms_users[nickname] = {} voms_users[nickname].update({dn: ca}) + result = {'added': 0, 'removed': 0, 'detailed': [],} + for user in VomsUser.objects.all(): info = voms_users.get(user.username) - if not info or not (dn in voms_users[user.username]): - user.delete() + if not info or not dn in voms_users[user.username]: + try: + user.delete() + except: + # TODO: log the error + continue + + result['removed'] += 1 + result['detailed'].append({ + 'action': 'remove', 'username': user.username, + 'dn': user.dn, 'ca': user.ca, + }) + # TODO: log operation continue else: del voms_users[user.username][user.dn] @@ -39,9 +65,19 @@ def run(): user.username = nickname user.dn = dn user.ca = ca - user.save() + try: + user.save() + except: + # TODO: log the error + continue + result['added'] += 1 + result['detailed'].append({ + 'action': 'add', 'username': user.username, + 'dn': user.dn, 'ca': user.ca, + }) + + return result if __name__ == "__main__": run() - From b555c174ff4d19b88aba4be9edae32bac2d45f5a Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 08:32:51 +0200 Subject: [PATCH 16/93] Added simple output for the command execution results --- atlas/auth/voms/management/commands/updatevomsmap.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/atlas/auth/voms/management/commands/updatevomsmap.py b/atlas/auth/voms/management/commands/updatevomsmap.py index 2e8e894f..ba8fe654 100644 --- a/atlas/auth/voms/management/commands/updatevomsmap.py +++ b/atlas/auth/voms/management/commands/updatevomsmap.py @@ -8,7 +8,9 @@ class Command(BaseCommand): def handle(self, *args, **options): - collector.run() - - + # TODO: add verbose and quiet mode + info = collector.run() + # TODO: use logging here + print "Added records: %s" % (info['added']) + print "Removed records: %s" % (info['removed']) From bcb1519b28d2b040094b8412dc6e5768f6514d9b Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 08:51:59 +0200 Subject: [PATCH 17/93] DEFT API client (to be replaced with subtree import) --- atlas/deftcore/__init__.py | 0 atlas/deftcore/api/__init__.py | 0 atlas/deftcore/api/client/__init__.py | 128 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 atlas/deftcore/__init__.py create mode 100644 atlas/deftcore/api/__init__.py create mode 100644 atlas/deftcore/api/client/__init__.py diff --git a/atlas/deftcore/__init__.py b/atlas/deftcore/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/atlas/deftcore/api/__init__.py b/atlas/deftcore/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/atlas/deftcore/api/client/__init__.py b/atlas/deftcore/api/client/__init__.py new file mode 100644 index 00000000..1618f2d1 --- /dev/null +++ b/atlas/deftcore/api/client/__init__.py @@ -0,0 +1,128 @@ +__author__ = 'Dmitry Golubkov' +__email__ = 'dmitry.v.golubkov@cern.ch' + +import json +import requests +import urllib + + +class Client(object): + BASE_URL = 'https://aipanda015.cern.ch' + + def __init__(self, auth_user, auth_key, verify_ssl_cert=False): + self.verify_ssl_cert = verify_ssl_cert + self.base_url = self.BASE_URL + self.api_url = '/api/v1/request/' + self.api_search_task_url = '/api/v1/task/' + self.headers = {'Content-Type': 'application/json', + 'Authorization': "ApiKey %s:%s" % (auth_user, auth_key)} + + def _get_action_list(self): + url = "%s%sactions/" % (self.base_url, self.api_url) + response = requests.get(url, headers=self.headers, verify=self.verify_ssl_cert) + if response.status_code == requests.codes.ok: + return json.loads(response.content)['result'] + else: + raise Exception("Invalid HTTP response code: %d" % response.status_code) + + def _create_request(self, action, owner, body): + action_list = self._get_action_list() + if not action in action_list: + raise Exception("Invalid action: %s (%s)" % (action, str(action_list))) + + url = "%s%s" % (self.base_url, self.api_url) + + data = {'action': action, 'owner': owner, 'body': "%s" % json.dumps(body)} + + response = requests.post(url, headers=self.headers, data=json.dumps(data), verify=self.verify_ssl_cert) + + if response.status_code == requests.codes.created: + api_request_object = json.loads(response.content) + return api_request_object['id'] + elif response.status_code == requests.codes.unauthorized: + raise Exception("Access denied") + else: + raise Exception("Invalid HTTP response code: %d" % response.status_code) + + def get_status(self, request_id): + url = "%s%s%s/" % (self.base_url, self.api_url, request_id) + response = requests.get(url, headers=self.headers, verify=self.verify_ssl_cert) + if response.status_code == requests.codes.ok: + status_string = json.loads(response.content)['status'] + if status_string: + return json.loads(status_string) + elif response.status_code == requests.codes.unauthorized: + raise Exception("Access denied") + else: + raise Exception("Invalid HTTP response code: %d" % response.status_code) + + def create_task_chain(self, owner, step_id, max_number_of_steps=None, debug_mode=False): + body = {'step_id': step_id, 'max_number_of_steps': max_number_of_steps, 'debug_mode': debug_mode} + return self._create_request('create_task_chain', owner, body) + + def clone_task(self, owner, task_id): + body = {'task_id': task_id} + return self._create_request('clone_task', owner, body) + + def abort_task(self, owner, task_id): + body = {'task_id': task_id} + return self._create_request('abort_task', owner, body) + + def finish_task(self, owner, task_id): + body = {'task_id': task_id} + return self._create_request('finish_task', owner, body) + + def reassign_task_to_site(self, owner, task_id, site): + body = {'task_id': task_id, 'site': site, 'cloud': None} + return self._create_request('reassign_task', owner, body) + + def reassign_task_to_cloud(self, owner, task_id, cloud): + body = {'task_id': task_id, 'site': None, 'cloud': cloud} + return self._create_request('reassign_task', owner, body) + + def change_task_priority(self, owner, task_id, priority): + body = {'task_id': task_id, 'priority': priority} + return self._create_request('change_task_priority', owner, body) + + def retry_task(self, owner, task_id): + body = {'task_id': task_id} + return self._create_request('retry_task', owner, body) + + def add_task_comment(self, owner, task_id, comment): + body = {'task_id': task_id, 'comment_body': comment} + return self._create_request('add_task_comment', owner, body) + + def _search_task(self, filter_dict): + if len(filter_dict.keys()): + filter_dict.update({'limit': 0}) + filter_string = urllib.urlencode(filter_dict) + url = "%s%s?%s" % (self.base_url, self.api_search_task_url, filter_string) + response = requests.get(url, headers=self.headers, verify=self.verify_ssl_cert) + if response.status_code == requests.codes.ok: + return json.loads(response.content) + else: + raise Exception("Invalid HTTP response code: %d" % response.status_code) + + def search_task_by_id(self, task_id): + filter_dict = dict() + if not task_id is None: + filter_dict.update({'id': task_id}) + return self._search_task(filter_dict) + + def search_task_by_parent_id(self, parent_id): + filter_dict = dict() + if not parent_id is None: + filter_dict.update({'parent_id': parent_id}) + return self._search_task(filter_dict) + + def search_task_by_chain_id(self, chain_id): + filter_dict = dict() + if not chain_id is None: + filter_dict.update({'chain_id': chain_id}) + return self._search_task(filter_dict) + + def search_task_by_name(self, taskname): + filter_dict = dict() + if not taskname is None: + filter_dict.update({'name__icontains': taskname}) + return self._search_task(filter_dict) From 4a278c884c8e50ef9268ca97517b05b3eac91dfa Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 08:52:53 +0200 Subject: [PATCH 18/93] Added settings for DEFT API access --- atlas/settings/local.py-example-template | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/atlas/settings/local.py-example-template b/atlas/settings/local.py-example-template index 9c95bac2..9165425a 100644 --- a/atlas/settings/local.py-example-template +++ b/atlas/settings/local.py-example-template @@ -59,3 +59,7 @@ defaultDatetimeFormat = defaultDatetimeFormatOracle LOG_ROOT = "/data/bigpandamon_virtualhosts/atlas/logs" +# DEFT API settings + +DEFT_AUTH_USER = 'FIXME' +DEFT_AUTH_KEY = 'FIXME' From c3e89cdfe2206d2d8978595eb38a847bd6d0160e Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Wed, 3 Sep 2014 21:26:15 +0200 Subject: [PATCH 19/93] Fixed nickname/dn pairs comparison --- atlas/auth/voms/collector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/atlas/auth/voms/collector.py b/atlas/auth/voms/collector.py index 91492ff2..f5b853d1 100644 --- a/atlas/auth/voms/collector.py +++ b/atlas/auth/voms/collector.py @@ -30,15 +30,16 @@ def run(): if not re.match(r"^\w+$", nickname): # TODO: log the warning continue + if not voms_users.get(nickname): voms_users[nickname] = {} voms_users[nickname].update({dn: ca}) - result = {'added': 0, 'removed': 0, 'detailed': [],} + result = {'added': 0, 'removed': 0, 'detailed': []} for user in VomsUser.objects.all(): info = voms_users.get(user.username) - if not info or not dn in voms_users[user.username]: + if not info or not user.dn in info: try: user.delete() except: From 1a58d7f2d5eef9bc8ee18b71daaf4b37332cb96f Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Fri, 5 Sep 2014 12:01:59 +0200 Subject: [PATCH 20/93] Add Validation group --- atlas/prodtask/models.py | 2 +- atlas/prodtask/request_views.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/models.py b/atlas/prodtask/models.py index 40dd7fc1..b3e094e1 100644 --- a/atlas/prodtask/models.py +++ b/atlas/prodtask/models.py @@ -40,7 +40,7 @@ class Meta: class TRequest(models.Model): PHYS_GROUPS=[(x,x) for x in ['physics','BPhysics','Btagging','DPC','Detector','EGamma','Exotics','HI','Higgs', 'InDet','JetMet','LAr','MuDet','Muon','SM','Susy','Tau','Top','Trigger','TrackingPerf', - 'reprocessing','trig-hlt']] + 'reprocessing','trig-hlt','Validation']] REQUEST_TYPE = [(x,x) for x in ['MC','GROUP','REPROCESSING','ANALYSIS','HLT']] PROVENANCE_TYPE = [(x,x) for x in ['AP','GP','XP']] diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index ee5474fb..6306240d 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -583,6 +583,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone cc = form.cleaned_data.get('cc', '') del form.cleaned_data['long_description'], form.cleaned_data['cc'], form.cleaned_data['excellink'], \ form.cleaned_data['excelfile'] + form.cleaned_data['hidden_json_slices'] = 'a' if form.cleaned_data.get('hidden_json_slices'): del form.cleaned_data['hidden_json_slices'] if 'reqid' in form.cleaned_data: From d8771b7b1a9cac06abdb54af28bb39193b0a170b Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 5 Sep 2014 13:17:30 +0200 Subject: [PATCH 21/93] Code refactored. Fixed tasks management with DEFT API. --- atlas/prodtask/task_actions.py | 269 +++++++++++++--------------- atlas/prodtask/task_manage_views.py | 53 +++--- atlas/settings/config.py | 6 + 3 files changed, 156 insertions(+), 172 deletions(-) diff --git a/atlas/prodtask/task_actions.py b/atlas/prodtask/task_actions.py index c0b133fd..c135cf92 100644 --- a/atlas/prodtask/task_actions.py +++ b/atlas/prodtask/task_actions.py @@ -1,162 +1,159 @@ -""" -Temporary placeholder to exec Jedi client remotely. -To be replaced with upcoming DEFT API functions -""" - -import ast -import os.path -import re -import subprocess +import json + from django.utils import timezone from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings -import atlas.settings -from .models import ProductionTask +from .models import ProductionTask, MCPriority +import atlas.deftcore.api.client as deft -RSA_KEY_FILE = "%s/%s" % (os.path.dirname(os.path.abspath(atlas.settings.__file__)), - "jediclient-ssh/id_rsa") +_deft_client = deft.Client(settings.DEFT_AUTH_USER, settings.DEFT_AUTH_KEY) -def _exec_jedi_command(task_id, command, *params): - """ - Perform JEDI command for the task. - :param task_id: task ID - :param command: command to perform - :param params: additional command parameters for JEDI client - :return: dict containing keys 'accepted', 'registered', 'jedi_message', 'jedi_status_code' - """ - # TODO: add logging and permissions checking - jedi_commands = ['killTask', 'finishTask', 'retryTask', 'changeTaskPriority', - 'reassignTaskToSite', 'reassignTaskToCloud'] - - if not command in jedi_commands: - raise ValueError("JEDI command not supported: '%s'" % (command)) - - (task_id, params) = (str(task_id), [str(x) for x in params]) - action_call = "import jedi.client as jc; print jc.%s(%s)" % (command, ",".join(list([task_id])+params)) - - proc = subprocess.Popen(['ssh', '-q', - '-i', RSA_KEY_FILE, - '-o', 'StrictHostKeyChecking=no', - '-o', 'UserKnownHostsFile=/dev/null', - '-o', 'LogLevel=QUIET', - 'sbelov@aipanda015', - 'PYTHONPATH=/mnt/atlswing/site-packages/ python -c "%s"' % (action_call), - '2>/dev/null' - ], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out = proc.communicate() - bash_lines = re.compile("^bash:") - # removing shell warnings if any - out = [x.rstrip() for x in out if not bash_lines.match(x)] - out = filter(None, out) # remove empty lines - - result = dict(task=task_id) - - if not out: - return result +# Mapping between task actions and DEFT task actions +_deft_actions = { + 'kill': 'abort_task', + 'finish': 'finish_task', + 'change_priority': 'change_task_priority', + 'reassign_to_site': 'reassign_task_to_site', + 'reassign_to_cloud': 'reassign_task_to_cloud', + 'retry': 'retry_task', +} - jedi_response = tuple(ast.literal_eval(out[0])) - if not jedi_response: - return result +supported_actions = _deft_actions.keys() +supported_actions.extend(['obsolete', 'increase_priority', 'decrease_priority']) - (accepted, registered, message) = (False, False, '') - (status_code, return_code) = (0, 0) - status_code = jedi_response[0] - if status_code == 0: - accepted = True +def do_action(owner, task_id, action, *args): + result = dict(owner=owner, task=task_id, action=action, args=args, + status=None, accepted=False, registered=False, + exception=None, exception_source=None) - if command == "changeTaskPriority": - return_code = jedi_response[1] - else: - (return_code, message) = jedi_response[1] - registered = bool(return_code) - result['jedi_return_code'] = return_code - else: - message = jedi_response[1] + if not action in supported_actions: + result['exception'] = "Action '%s' is not supported" % action + return result - result.update(accepted=accepted, registered=registered, - jedi_message=message, jedi_status_code=status_code) + if action in _deft_actions: + result.update(_do_deft_action(owner, task_id, action, *args)) + elif action == 'increase_priority': + result.update(increase_task_priority(owner, task_id, *args)) + elif action == 'decrease_priority': + result.update(decrease_task_priority(owner, task_id, *args)) + elif action == 'obsolete': + result.update(obsolete_task(owner, task_id)) return result -def kill_task(task_id): +def _do_deft_action(owner, task_id, action, *args): """ - Kill task with all it jobs. - :param task_id: ID of task to kill - :return: dict with action status + Perform task action using DEFT API + :param owner: username form which task action will be performed + :param task_id: task ID + :param action: action name + :param args: additional arguments for the action (if needed) + :return: dictionary with action execution details """ - return _exec_jedi_command(task_id, "killTask") + result = dict(owner=owner, task=task_id, action=action, args=args, + status=None, accepted=False, registered=False, + exception=None, exception_source=None) -def finish_task(task_id): - """ - Finish task with all it jobs. - :param task_id: task ID - :return: dict with action status - """ - return _exec_jedi_command(task_id, "finishTask") + if not action in _deft_actions: + result['exception'] = "Action '%s' is not supported" % action + return result + try: + func = getattr(_deft_client, _deft_actions[action]) + except AttributeError as e: + result.update(exception=str(e)) + return result + + try: + request_id = func(owner, task_id, *args) + except Exception as e: + result.update(exception=str(e), + exception_source=_deft_client.__class__.__name__) + return result + + result['accepted'] = True -def obsolete_task(task_id): + try: + status = _deft_client.get_status(request_id) + except Exception as e: + result.update(exception=str(e), + exception_source=_deft_client.__class__.__name__) + return result + + result.update(registered=True, status=status) + + return result + + +def obsolete_task(owner, task_id): """ Mark task as 'obsolete' + :param owner: username form which task action will be performed :param task_id: task ID :return: dict with action status """ - # TODO: add logging and permissions checking + result = dict(owner=owner, task_id=task_id, + accepted=True, registered=False, exception=None) + + # TODO: add logging + # TODO: add logging with PandaLog (using DEFT API) try: task = ProductionTask.objects.get(id=task_id) except ObjectDoesNotExist: - return dict(accepted=True, registered=False, message="Task %s does not exist") + result['exception'] = "Task '%s' does not exist" % task_id + return result except Exception as error: - return dict(accepted=True, registered=False, message=error) + result['exception'] = str(error) + return result if task.status not in ['done', 'finished']: - return {} + result['exception'] = "Task '%s' is in the state '%s', not 'done' or 'finished'" % (task_id, task.status) + return result #TODO: log action ProductionTask.objects.filter(id=task_id).update(status='obsolete', timestamp=timezone.now()) - return dict(accepted=True, registered=True) + result['registered'] = True + return result -def change_task_priority(task_id, priority): +def change_task_priority(owner, task_id, priority): """ Set task JEDI priority. - :param task_id: + :param task_id: task ID :param priority: JEDI task priority :return: dict with action status """ - return _exec_jedi_command(task_id, "changeTaskPriority", priority) + # TODO: add status checking and logging + return _do_deft_action(owner, task_id, 'change_priority', priority) def get_task_priority_levels(task_id): """ - Get task priority level (if any) and - :param task_id: - :return: + Get task priority levels (if any) for the task + :param task_id: task ID + :return: dict containing available levels, current level and task priority """ def get_priority_levels(): """ Get JEDI priority levels from the step template - :param priority_key: priority level (<100) - :param step_name: name of the step in question - :return: value of JEDI priority of the step { name: {level: priority, ...}, ...} + :return: dict of JEDI priority of the step { name: {level: priority, ...}, ...} """ - levels = {} + levels_ = {} for prio in MCPriority.objects.all(): try: named_priorities = json.loads(prio.priority_dict) except: continue for name, priority in named_priorities.items(): - if not levels.get(name): - levels[name] = {} - levels[name][int(prio.priority_key)] = priority - - return levels + if not levels_.get(name): + levels_[name] = {} + levels_[name][int(prio.priority_key)] = priority + return levels_ result = dict(id=task_id, current_level=None, levels={}, current_priority=None) @@ -180,12 +177,14 @@ def get_priority_levels(): return result -def shift_task_priority(task_id, level_shift, priority_shift=None): +def shift_task_priority(owner, task_id, level_shift, priority_shift=None): """ Shifting task priority up or down - :param task_id: Task ID - :param level_shift: if > 0, increasing the priority, otherwise decreasing. Has precedence over priority_shift. - :param priority_shift: + :param owner: username form which task action will be performed + :param task_id: task ID + :param level_shift: if > 0, increasing the priority, otherwise decreasing. + Has precedence over priority_shift. + :param priority_shift: value of priority shift to apply :return: """ @@ -196,10 +195,10 @@ def shift_task_priority(task_id, level_shift, priority_shift=None): result = dict() if levels: - levels = levels.values + levels = levels.values() if not levels and (priority_shift is not None): - return change_task_priority(task_id, current_prio+priority_shift) + return change_task_priority(owner, task_id, current_prio+priority_shift) if level_shift > 0: next_priorities = sorted([x for x in levels if x > current_prio]) @@ -210,48 +209,34 @@ def shift_task_priority(task_id, level_shift, priority_shift=None): return result new_priority = next_priorities[0] - return change_task_priority(task_id, new_priority) + return change_task_priority(owner, task_id, new_priority) -def increase_task_priority(task_id, delta=None): - if isinstance(delta, int): - return shift_task_priority(task_id=task_id, level_shift=-1, priority_shift=delta) - else: - return shift_task_priority(task_id=task_id, level_shift=-1) - - -def decrease_task_priority(task_id, delta=None): - if isinstance(delta, int): - return shift_task_priority(task_id=task_id, level_shift=1, priority_shift=-delta) - else: - return shift_task_priority(task_id=task_id, level_shift=1) - - - -def reassign_task_to_site(task_id, site): +def increase_task_priority(owner, task_id, delta=None): """ - Reassign task to specified site. + Increase task priority for one level or specified value + :param owner: username form which task action will be performed :param task_id: task ID - :param site: site name - :return: dict with action status + :param delta: value to change priority on + :return: """ - return _exec_jedi_command(task_id, "reassignTaskToSite", site) + if isinstance(delta, int): + return shift_task_priority(owner=owner, task_id=task_id, level_shift=-1, priority_shift=delta) + else: + return shift_task_priority(owner=owner, task_id=task_id, level_shift=-1) -def reassign_task_to_cloud(task_id, cloud): +def decrease_task_priority(owner, task_id, delta=None): """ - Reassign task to specified cloud - :param task_id: Task ID - :param cloud: cloud name - :return: dict with action status + Decrease task priority for one level or specified value + :param owner: username form which task action will be performed + :param task_id: task ID + :param delta: value to change priority on + :return: """ - return _exec_jedi_command(task_id, "reassignTaskToCloud", cloud) + if isinstance(delta, int): + return shift_task_priority(owner=owner, task_id=task_id, level_shift=1, priority_shift=-delta) + else: + return shift_task_priority(owner=owner, task_id=task_id, level_shift=1) -def retry_task(task_id): - """ - Resubmit specified task - :param task_id: Task ID - :return: dict with action status - """ - _exec_jedi_command(task_id, "retryTask") \ No newline at end of file diff --git a/atlas/prodtask/task_manage_views.py b/atlas/prodtask/task_manage_views.py index 5360943e..a6d48c72 100644 --- a/atlas/prodtask/task_manage_views.py +++ b/atlas/prodtask/task_manage_views.py @@ -1,9 +1,11 @@ +import json from django.http import HttpResponse from django.template.response import TemplateResponse from django.views.decorators.cache import never_cache from django.views.decorators.csrf import csrf_protect, csrf_exempt, ensure_csrf_cookie from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings import core.datatables as datatables @@ -11,42 +13,25 @@ from .task_views import ProductionTaskTable, Parameters, get_clouds, get_sites -from .task_actions import kill_task, finish_task, obsolete_task,\ - change_task_priority, increase_task_priority, decrease_task_priority, \ - reassign_task_to_site, reassign_task_to_cloud, retry_task - - -import json - +from .task_actions import do_action, supported_actions -_task_actions = { - 'kill': kill_task, - 'finish': finish_task, - 'obsolete': obsolete_task, - 'change_priority': change_task_priority, - 'increase_priority': increase_task_priority, - 'decrease_priority': decrease_task_priority, - 'reassign_to_site': reassign_task_to_site, - 'reassign_to_cloud': reassign_task_to_cloud, - 'retry': retry_task, -} - -def do_tasks_action(tasks, action, *args): +def do_tasks_action(owner, tasks, action, *args): """ - Performing task actions - :param tasks: list of tasks affected - :param action: name of action + Performing tasks actions + :param tasks: list of tasks IDs affected + :param action: name of the action :param args: additional arguments - :return: array of actions' statuses + :return: array of per-task actions' statuses """ - if (not tasks) or not (action in _task_actions): + # TODO: add logging + # TODO: + if (not tasks) or not (action in supported_actions): return result = [] for task in tasks: - response = _task_actions[action](task, *args) - req_info = dict(task_id=task, action=action, response=response) + req_info = do_action(owner, task, action, *args) result.append(req_info) return result @@ -61,7 +46,16 @@ def tasks_action(request, action): """ empty_response = HttpResponse('') - if request.method != 'POST' or not (action in _task_actions): + if request.method != 'POST' or not (action in supported_actions): + return empty_response + + # TODO: return comprehensible response anytime + # TODO: rewrite with django auth system + #if not request.user.groups.filter(name='vomsrole:/atlas/Role=production'): + # return empty_response + + owner = request.META.get(settings.META_USERNAME) + if not owner: return empty_response data_json = request.body @@ -74,7 +68,7 @@ def tasks_action(request, action): return empty_response params = data.get("parameters", []) - response = do_tasks_action(tasks, action, *params) + response = do_tasks_action(owner, tasks, action, *params) return HttpResponse(json.dumps(response)) @@ -119,7 +113,6 @@ def get_same_slice_tasks(request): return HttpResponse(json.dumps(tasks_slices)) - @ensure_csrf_cookie @csrf_protect @never_cache diff --git a/atlas/settings/config.py b/atlas/settings/config.py index 0dbb1a7b..85791777 100644 --- a/atlas/settings/config.py +++ b/atlas/settings/config.py @@ -79,3 +79,9 @@ ## more details on how to customize your logging configuration. from .logconfig import LOGGING +# Settings for authentication variables +META_EMAIL = 'ADFS_EMAIL' +META_FIRSTNAME = 'ADFS_FIRSTNAME' +META_GROUP = 'ADFS_GROUP' +META_LASTNAME = 'ADFS_LASTNAME' +META_USERNAME = 'ADFS_LOGIN' \ No newline at end of file From eec98528bc2bfd9e7fc06303b995ff9ec9932aa4 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 5 Sep 2014 17:47:16 +0200 Subject: [PATCH 22/93] VOMS interface changed to take all options only from parameters given. Method 'get_identity_options' fixed to have no potential side effects on environment variables. --- atlas/auth/voms/interface.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/atlas/auth/voms/interface.py b/atlas/auth/voms/interface.py index 57eb6bc8..ba6e39de 100644 --- a/atlas/auth/voms/interface.py +++ b/atlas/auth/voms/interface.py @@ -2,14 +2,14 @@ from VOMSAdmin.VOMSCommands import VOMSAdminProxy + class VomsInterface: """ Retrieving information from VOMS server. """ def __init__(self, options): - self.options = VomsInterface.get_identity_options() - self.options.update(options) + self.options = dict(options) self.voms = VOMSAdminProxy(**options) def list_users(self): @@ -59,7 +59,15 @@ def get_identity_options(voms_admin_path="/usr/bin/voms-admin", proxy_cert=None) voms_admin.vlog = lambda msg: None if proxy_cert: + old_proxy = os.environ.get("X509_USER_PROXY") os.environ["X509_USER_PROXY"] = proxy_cert - voms_admin.setup_identity() + voms_admin.setup_identity() + # Restore initial environment + if old_proxy is None: + del os.environ["X509_USER_PROXY"] + else: + os.environ["X509_USER_PROXY"] = old_proxy + else: + voms_admin.setup_identity() return voms_admin.options From ae30dc83a27a0d9e676cb3fac3b2ff04e434da8e Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 5 Sep 2014 17:52:04 +0200 Subject: [PATCH 23/93] Initialize VOMS interface with options from setting s only. --- atlas/auth/voms/backends.py | 5 +---- atlas/auth/voms/collector.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/atlas/auth/voms/backends.py b/atlas/auth/voms/backends.py index 9185c25f..b9e0565d 100644 --- a/atlas/auth/voms/backends.py +++ b/atlas/auth/voms/backends.py @@ -42,10 +42,7 @@ def authenticate(self, request=None): if not dn_map: return - options = VomsInterface.get_identity_options(proxy_cert=settings.VOMS_PROXY_CERT) - options.update(settings.VOMS_OPTIONS) - voms = VomsInterface(options) - + voms = VomsInterface(settings.VOMS_OPTIONS) voms_groups = [] for (dn, ca) in dn_map.items(): diff --git a/atlas/auth/voms/collector.py b/atlas/auth/voms/collector.py index f5b853d1..3798c283 100644 --- a/atlas/auth/voms/collector.py +++ b/atlas/auth/voms/collector.py @@ -12,11 +12,8 @@ def run(): :return: dictionary, 'added', 'removed' - records added/removed, 'details' - more information on operations performed """ - options = VomsInterface.get_identity_options(proxy_cert=settings.VOMS_PROXY_CERT) - options.update(settings.VOMS_OPTIONS) - - voms = VomsInterface(options) + voms = VomsInterface(settings.VOMS_OPTIONS) voms_users = {} for user_info in voms.list_users(): From 9aa24e9078754e43ebca25d2a168b1415db2e883 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 12 Sep 2014 11:47:12 +0200 Subject: [PATCH 24/93] PAckage 'python-requests' added to dependencies (needed for DEFT API) --- setup.cfg | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index efd6ee7d..24024ceb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,9 @@ release_type = dev provides = bigpandamon-atlas release = 1 packager = Panda Team -requires = python +requires = python + python-requests bigpandamon-core >= 0.0.11 dq2-clients myproxy + From be9c7ab736a4bd85f52d86797f4c9b385516411a Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 12 Sep 2014 11:50:16 +0200 Subject: [PATCH 25/93] Removed middleware file (not needed anymore). --- atlas/auth/voms/middleware.py | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 atlas/auth/voms/middleware.py diff --git a/atlas/auth/voms/middleware.py b/atlas/auth/voms/middleware.py deleted file mode 100644 index 1881aafc..00000000 --- a/atlas/auth/voms/middleware.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.conf import settings -from django.contrib import auth - - -class VomsMiddleware(object): - """ - Updates user's groups once a session. - """ - - def process_request(self, request): - username = request.META.get(settings.META_USERNAME) - - if username and not request.session.get('VOMS_GROUPS_INITIALIZED'): - if auth.authenticate(request=request): - request.session['VOMS_GROUPS_INITIALIZED'] = 1 From f7d969676c7afeaa0420d09c6daad2839e8815cb Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 12 Sep 2014 12:07:13 +0200 Subject: [PATCH 26/93] Settings files changed to enable authentication backends --- atlas/settings/base.py | 15 ++++++++++++++- atlas/settings/config.py | 28 +++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/atlas/settings/base.py b/atlas/settings/base.py index a39ca9bc..b564d4a1 100644 --- a/atlas/settings/base.py +++ b/atlas/settings/base.py @@ -24,6 +24,15 @@ ) INSTALLED_APPS_BIGPANDAMON_ATLAS = ( + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.admin', + 'shibsso', + 'atlas.auth.voms', + ### BigPanDAmon core 'core.common', 'core.table', @@ -42,10 +51,14 @@ ) INSTALLED_APPS = COMMON_INSTALLED_APPS + INSTALLED_APPS_BIGPANDAMON_ATLAS JS_I18N_APPS_EXCLUDE = common.settings.base.JS_I18N_APPS_EXCLUDE + ('django_tables2',) + TEMPLATE_CONTEXT_PROCESSORS = ( - 'django.core.context_processors.request', #django-tables2 + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django.core.context_processors.request', #django-tables2 ) + ROOT_URLCONF = 'atlas.urls' SITE_ID = 2 diff --git a/atlas/settings/config.py b/atlas/settings/config.py index 85791777..3e1723dc 100644 --- a/atlas/settings/config.py +++ b/atlas/settings/config.py @@ -79,9 +79,35 @@ ## more details on how to customize your logging configuration. from .logconfig import LOGGING + +MIDDLEWARE_CLASSES = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'shibsso.middleware.ShibSSOMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', +) + + +AUTHENTICATION_BACKENDS = ( +# 'django.contrib.auth.backends.ModelBackend', + 'atlas.auth.voms.backends.VomsBackend', + 'shibsso.backends.ShibSSOBackend', +) + + # Settings for authentication variables + +SHIB_SSO_ADMIN = True +SHIB_SSO_CREATE_ACTIVE = True +SHIB_SSO_CREATE_STAFF = False +SHIB_SSO_CREATE_SUPERUSER = False +SHIB_LOGIN_PATH = '/Shibboleth.sso/?target=' +SHIB_LOGOUT_URL = '/' META_EMAIL = 'ADFS_EMAIL' META_FIRSTNAME = 'ADFS_FIRSTNAME' META_GROUP = 'ADFS_GROUP' META_LASTNAME = 'ADFS_LASTNAME' -META_USERNAME = 'ADFS_LOGIN' \ No newline at end of file +META_USERNAME = 'ADFS_LOGIN' +LOGIN_REDIRECT_URL = '/' From ec07856f366c3b0a9ecd5a69d8bf4530b7dd44f1 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Fri, 12 Sep 2014 12:18:35 +0200 Subject: [PATCH 27/93] Add formats for MC Reco steps --- atlas/prodtask/request_views.py | 2 ++ atlas/prodtask/spdstodb.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index 6306240d..5155a81c 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -630,6 +630,8 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone task_config.update({'nEventsPerInputFile':int(step['task_config']['nEventsPerJob'].get(step['step_name'],-1))}) if 'project_mode' in step['task_config']: task_config.update({'project_mode':step['task_config']['project_mode']}) + if 'input_format' in step['task_config']: + task_config.update({'input_format':step['task_config']['input_format']}) step['step_exec']['request'] = req step['step_exec']['slice'] = irl step['step_exec']['step_template'] = st diff --git a/atlas/prodtask/spdstodb.py b/atlas/prodtask/spdstodb.py index 001a5e7e..98c8df15 100644 --- a/atlas/prodtask/spdstodb.py +++ b/atlas/prodtask/spdstodb.py @@ -125,6 +125,7 @@ def translate_excl_to_dict(excel_dict): input_events=int(input_events)) index += 1 + reduce_input_format = False for currentstep in StepExecution.STEPS: if translated_row.get(currentstep): st = currentstep @@ -137,11 +138,20 @@ def translate_excl_to_dict(excel_dict): else: sexec = dict(status='NotChecked', input_events=-1) formats = None + task_config = {'nEventsPerJob':get_default_nEventsPerJob_dict(), + 'project_mode':get_default_project_mode_dict().get(st,'')} + + if reduce_input_format: + task_config.update({'input_format':'AOD'}) + reduce_input_format = False if currentstep == 'Reco': - formats = 'AOD' + if translated_row.get('format', ''): + formats = 'AOD'+'.'+translated_row.get('format', '') + reduce_input_format = True + else: + formats = 'AOD' st_sexec_list.append({'step_name' :st, 'tag': tag, 'formats': formats, 'step_exec': sexec, - 'task_config':{'nEventsPerJob':get_default_nEventsPerJob_dict(), - 'project_mode':get_default_project_mode_dict().get(st,'')}}) + 'task_config':task_config}) return_list.append({'input_dict':irl, 'step_exec_dict':st_sexec_list}) return return_list From 93491b2c81e5e86b2a12549846072ad2bfd3bd79 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Fri, 12 Sep 2014 14:13:54 +0200 Subject: [PATCH 28/93] Added login and logout urls --- atlas/prodtask/urls.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/atlas/prodtask/urls.py b/atlas/prodtask/urls.py index 7b0350d7..9eb3b214 100644 --- a/atlas/prodtask/urls.py +++ b/atlas/prodtask/urls.py @@ -75,6 +75,9 @@ url(r'^$', 'atlas.prodtask.views.home', name='home'), + url(r'^login/$', 'shibsso.views.login', name='login'), + url(r'^logout/$', 'shibsso.views.logout', name='logout'), + # Uncomment the admin/doc line below to enable admin documentation: # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), From e18a48390113f2ba4fd8f12a65b684b623e4aff2 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 15 Sep 2014 12:12:30 +0200 Subject: [PATCH 29/93] add dataset list textarea --- .../prodtask/templates/prodtask/_reqdatatable.html | 2 +- atlas/prodtask/templates/prodtask/_requestform.html | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index 976d418f..eb379c8a 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -766,7 +766,7 @@
With pattern: {{ pattern }}
{% else %} diff --git a/atlas/prodtask/templates/prodtask/_requestform.html b/atlas/prodtask/templates/prodtask/_requestform.html index 7bcba0cd..3f70f802 100644 --- a/atlas/prodtask/templates/prodtask/_requestform.html +++ b/atlas/prodtask/templates/prodtask/_requestform.html @@ -85,9 +85,13 @@ datasets += $(this).html() + ','; } }); - if (datasets!=''){ + sliceObject['datasets']=''; + if (datasets&&(datasets!='')){ sliceObject['datasets'] = datasets; } + if (sliceObject['datasetList']&&(sliceObject['datasetList']!='')){ + sliceObject['datasets'] += sliceObject['datasetList'].replace(/(\r\n|\n|\r)/gm,","); + } resultObject[sliceNumber] = sliceObject; }); $("#{{ form.hidden_json_slices.auto_id }}").val($.toJSON(resultObject)); @@ -178,6 +182,13 @@
+
+
+ +
+
+ +{#
#} +{#

#} +{#
#} +{# {% for step in step_list %}#} +{#
{{step.name}}
#} +{# {% endfor %}#} +{#
#} +{#
#} +{# {% for step in step_list %}#} +{#
#} +{#

This is the {{forloop.counter0}} panel of the basic tab example.

#} +{#
#} +{# {% endfor %}#} +{#
#} +{# ×#} +{#
#} +
+

Dataset:

+

job options:

+
+ {% for step in step_list %} +
+

{{step.name}}

+
+
+
+
+ +
+
+ + +
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+ +
+
+ +
+
+
+
+
+ {% endfor %} + +
+ × +
{% endblock %} \ No newline at end of file From 30bf861c0b6ea12ef4ac1821d9e294dab3876c38 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Mon, 22 Sep 2014 18:54:28 +0200 Subject: [PATCH 49/93] User name to be taken from request.user object --- atlas/prodtask/task_manage_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas/prodtask/task_manage_views.py b/atlas/prodtask/task_manage_views.py index a6d48c72..1c9a9b8d 100644 --- a/atlas/prodtask/task_manage_views.py +++ b/atlas/prodtask/task_manage_views.py @@ -54,7 +54,7 @@ def tasks_action(request, action): #if not request.user.groups.filter(name='vomsrole:/atlas/Role=production'): # return empty_response - owner = request.META.get(settings.META_USERNAME) + owner = request.user.username if not owner: return empty_response From 71900e089e10b137b060317de64657835e681326 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 03:02:47 +0200 Subject: [PATCH 50/93] Added user's account info page --- .../templates/prodtask/_userinfo.html | 75 +++++++++++++++++++ atlas/prodtask/views.py | 12 +++ 2 files changed, 87 insertions(+) create mode 100644 atlas/prodtask/templates/prodtask/_userinfo.html diff --git a/atlas/prodtask/templates/prodtask/_userinfo.html b/atlas/prodtask/templates/prodtask/_userinfo.html new file mode 100644 index 00000000..0e716843 --- /dev/null +++ b/atlas/prodtask/templates/prodtask/_userinfo.html @@ -0,0 +1,75 @@ +{% extends parent_template %} +{% load url from future %} +{% load static from staticfiles %} + + +{% block extra_js %} +{{ block.super }} + + + + + + +{% endblock %} + + +{% block subtitle %} +User's details on {{ user.first_name|safe }} {{ user.last_name|safe }} ({{ user.username|safe }}) +{% endblock %} + + +{% block body %} + +{{ block.super }} + + + + + + + + + + + + + + + + + + + + + + + + {% if user.is_superuser %} + + + + + {% endif %} + + + + + + + + + + +
{{ user.username|safe }}
{{ user.first_name|safe }} {{ user.last_name|safe }}
{{ user.email|safe }}
+ {% if user.groups %} + {% for group in user.groups.all %} + {{ group|safe }}
+ {% endfor %} + {% endif %} +
{{ user.is_active }}
Yes
{{ user.date_joined }}
{{ user.last_login }}
+ + +{% endblock %} \ No newline at end of file diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index 560444c2..fdcf8aa1 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -6,6 +6,7 @@ from django.template import Context, Template, RequestContext from django.template.loader import get_template from django.template.response import TemplateResponse +from django.views.decorators.cache import never_cache from django.views.decorators.csrf import csrf_protect, ensure_csrf_cookie from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse @@ -1026,3 +1027,14 @@ def production_dataset_table(request): return TemplateResponse(request, 'prodtask/_dataset_table.html', { 'title': 'Aborted and Obsolete Production Dataset Status Table', 'active_app' : 'prodtask', 'table': request.fct, 'parent_template': 'prodtask/_index.html'}) + + +@never_cache +def userinfo(request): + return TemplateResponse(request, "prodtask/_userinfo.html", + { + 'title': 'User info', + 'active_app' : 'prodtask', + 'parent_template': 'prodtask/_index.html', + }) + From d5b680f3c1bccfa7d06240aef09d8a3f033762ed Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 03:04:07 +0200 Subject: [PATCH 51/93] Account info page added to URLs list --- atlas/prodtask/urls.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atlas/prodtask/urls.py b/atlas/prodtask/urls.py index 60dc6891..0f8a2d92 100644 --- a/atlas/prodtask/urls.py +++ b/atlas/prodtask/urls.py @@ -77,6 +77,8 @@ url(r'^login/$', 'shibsso.views.login', name='login'), url(r'^logout/$', 'shibsso.views.logout', name='logout'), + url(r'^userinfo/$', 'atlas.prodtask.views.userinfo', name='userinfo'), + # Uncomment the admin/doc line below to enable admin documentation: # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), From 491bc4a6d86da861fc9206fec449f66cd35bbe5a Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 03:09:20 +0200 Subject: [PATCH 52/93] User info in dropdown menu replaced with link to the account info page --- atlas/templates/_base_bigpandamon_atlas.html | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/atlas/templates/_base_bigpandamon_atlas.html b/atlas/templates/_base_bigpandamon_atlas.html index a27ba7ce..47a4e125 100644 --- a/atlas/templates/_base_bigpandamon_atlas.html +++ b/atlas/templates/_base_bigpandamon_atlas.html @@ -143,13 +143,7 @@
  • Logged as: {{ user.first_name.0 }}. {{ user.last_name }}
    • From 235d1518cc795761fb0b58c438f4ec99ebb96651 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 03:13:37 +0200 Subject: [PATCH 53/93] Fixed code indentation --- atlas/templates/_base_bigpandamon_atlas.html | 53 ++++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/atlas/templates/_base_bigpandamon_atlas.html b/atlas/templates/_base_bigpandamon_atlas.html index 47a4e125..7f9104c9 100644 --- a/atlas/templates/_base_bigpandamon_atlas.html +++ b/atlas/templates/_base_bigpandamon_atlas.html @@ -128,33 +128,32 @@ {% endblock %} - {% block top_menu_all_6 %} -
  • - {% block top_menu_6 %} - Help - - {% endblock %} -
    • - - {% if user.is_authenticated %} -
  • - Logged as: {{ user.first_name.0 }}. {{ user.last_name }} - -
    • +{% block top_menu_all_6 %} +
  • + {% block top_menu_6 %} + Help + + {% endblock %} +
    • + +{% if user.is_authenticated %} +
  • + Logged as: {{ user.first_name.0 }}. {{ user.last_name }} + +
    • + +{% else %} +
  • + Login +
    • +{% endif %} - {% else %} -
  • - Login -
    • - {% endif %} - - - {% endblock %} +{% endblock %} From 8280bf01e9bbacb36eb63b6262ce52def7300ad0 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 04:05:05 +0200 Subject: [PATCH 54/93] Changed menu item title for logged in user --- atlas/templates/_base_bigpandamon_atlas.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas/templates/_base_bigpandamon_atlas.html b/atlas/templates/_base_bigpandamon_atlas.html index 7f9104c9..a5a5c2fa 100644 --- a/atlas/templates/_base_bigpandamon_atlas.html +++ b/atlas/templates/_base_bigpandamon_atlas.html @@ -141,7 +141,7 @@ {% if user.is_authenticated %}
  • - Logged as: {{ user.first_name.0 }}. {{ user.last_name }} + {{ user.first_name.0 }}. {{ user.last_name }}
    • Account info ({{ user.username }})
    • Logout
      • From c3105e6b7e7818454ad6b2e29085f2909c7fd9b6 Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 04:15:04 +0200 Subject: [PATCH 55/93] Fix: removed duplicated string from installed apps --- atlas/settings/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/atlas/settings/base.py b/atlas/settings/base.py index b564d4a1..79d13b13 100644 --- a/atlas/settings/base.py +++ b/atlas/settings/base.py @@ -24,7 +24,6 @@ ) INSTALLED_APPS_BIGPANDAMON_ATLAS = ( - 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', From 70312cb0b763e2eb6225a2fef6426151419d1dce Mon Sep 17 00:00:00 2001 From: Sergey Belov Date: Tue, 23 Sep 2014 04:28:39 +0200 Subject: [PATCH 56/93] Merged from 'devel' --- atlas/prodtask/forms.py | 3 +- atlas/prodtask/request_views.py | 102 +++++++++++++--- atlas/prodtask/spdstodb.py | 9 +- atlas/prodtask/step_manage_views.py | 39 +++++++ .../templates/prodtask/_previewreq.html | 2 + .../templates/prodtask/_reqdatatable.html | 31 +++++ .../templates/prodtask/_requestform.html | 109 +++++++++++++++--- atlas/prodtask/urls.py | 2 + 8 files changed, 259 insertions(+), 38 deletions(-) diff --git a/atlas/prodtask/forms.py b/atlas/prodtask/forms.py index 7cc3e1f4..42a8eef6 100644 --- a/atlas/prodtask/forms.py +++ b/atlas/prodtask/forms.py @@ -85,11 +85,10 @@ class TRequestReprocessingCreateCloneForm(TRequestCreateCloneConfirmation): provenance = CharField(widget=forms.HiddenInput, required=False) cstatus = CharField(widget=forms.HiddenInput, required=False) request_type = CharField(widget=forms.HiddenInput, required=False) - # tag_hierarchy = CharField(help_text='tag hierarhy as python list with tuples as branches', - # widget=Textarea, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) campaign = CharField(required=False) + hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden") class Meta: model = TRequest diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index 5155a81c..838f8dcc 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -1,3 +1,4 @@ +import copy from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail from django.core.urlresolvers import reverse @@ -222,18 +223,49 @@ def hlt_form_prefill(form_data, request): _logger.debug('Gathered data: %s' % spreadsheet_dict) return spreadsheet_dict, eroor_message + def parse_json_slice_dict(json_string): spreadsheet_dict = [] input_dict = json.loads(json_string) slice_index = 0 - for slice_number, slice in input_dict.items(): - for dataset in slice['datasets'].split(','): + slices_dict = {} + + + for slice_step in input_dict.keys(): + # prepare input + current_step_dict = {} + for key,item in input_dict[slice_step].items(): + current_step_dict[key] = str(item).strip() + current_slice,current_step = slice_step.split('_') + if int(current_slice) not in slices_dict.keys(): + slices_dict[int(current_slice)] = {'steps':{}} + if current_step == '0': + slices_dict[int(current_slice)].update(current_step_dict) + slices_dict[int(current_slice)].update({'step_order':slice_step}) + else: + slices_dict[int(current_slice)]['steps'][int(current_step)] = current_step_dict + slices_dict[int(current_slice)]['steps'][int(current_step)].update({'step_order':slice_step}) + for slice_number in range(len(slices_dict.keys())): + slice = slices_dict[slice_number] + if (slice['step_order'] != slice['parentstepshort']): + datasets = ['foreign'] * len(slices_dict[int(slice['parentstepshort'].split('_')[0])]['datasets'].split(',')) + slice['datasets'] = ','.join(datasets) + else: + datasets = [x.strip() for x in slice['datasets'].split(',') if x] + for prefix,dataset in enumerate(datasets): if dataset: - irl = dict(slice=slice_index, brief=' ', comment='', dataset=dataset, - input_data='', - project_mode=slice['projectmode'], - priority=int(slice['priority']), - input_events=int(slice['totalevents'])) + if dataset == 'foreign': + irl = dict(slice=slice_index, brief=' ', comment='', + input_data='', + project_mode=slice['projectmode'], + priority=int(slice['priority']), + input_events=int(slice['totalevents'])) + else: + irl = dict(slice=slice_index, brief=' ', comment='', dataset=dataset, + input_data='', + project_mode=slice['projectmode'], + priority=int(slice['priority']), + input_events=int(slice['totalevents'])) slice_index += 1 st_sexec_list = [] if slice['ctag']: @@ -245,10 +277,31 @@ def parse_json_slice_dict(json_string): sexec = dict(status='NotChecked', priority=int(slice['priority']), input_events=int(slice['totalevents'])) st_sexec_list.append({'step_name': step_name, 'tag': slice['ctag'], 'step_exec': sexec, - 'memory': slice['ram'], + 'memory': slice['ram'], 'step_order':str(prefix)+'_'+slice['step_order'], + 'step_parent': str(prefix)+'_'+slice['parentstepshort'], 'formats': slice['formats'], 'task_config':task_config}) + for step_number in range(1,len(slice['steps'])+1): + step = slice['steps'][step_number] + if step['ctag']: + task_config = {} + nEventsPerJob = step['eventsperjob'] + task_config.update({'nEventsPerJob':dict((x,nEventsPerJob) for x in StepExecution.STEPS)}) + task_config.update({'project_mode':step['projectmode']}) + if step['inputFormat']: + task_config.update({'input_format':step['inputFormat']}) + step_name = step_from_tag(step['ctag']) + sexec = dict(status='NotChecked', priority=int(step['priority']), + input_events=int(step['totalevents'])) + st_sexec_list.append({'step_name': step_name, 'tag': step['ctag'], 'step_exec': sexec, + 'memory': step['ram'],'step_order':str(prefix)+'_'+step['step_order'], + 'step_parent': str(prefix)+'_'+step['parentstepshort'], + 'formats': step['formats'], + 'task_config':task_config}) + else: + break spreadsheet_dict.append({'input_dict': irl, 'step_exec_dict': st_sexec_list}) + return spreadsheet_dict def dpd_form_prefill(form_data, request): @@ -261,7 +314,7 @@ def dpd_form_prefill(form_data, request): file_name = open_tempfile_from_url(form_data['excellink'], 'txt') with open(file_name) as open_file: file_obj = open_file.read().split('\n') - if form_data.get('excelfile'): + elif form_data.get('excelfile'): file_obj = request.FILES['excelfile'].read().split('\n') _logger.debug('Try to read data from %s' % form_data.get('excelfile')) elif form_data.get('hidden_json_slices'): @@ -333,21 +386,23 @@ def dpd_form_prefill(form_data, request): def reprocessing_form_prefill(form_data, request): spreadsheet_dict = [] + output_dict = {} + error_message = '' + try: if form_data.get('excellink'): _logger.debug('Try to read data from %s' % form_data.get('excellink')) file_name = open_tempfile_from_url(form_data['excellink'], 'txt') with open(file_name) as open_file: file_obj = open_file.read().split('\n') - conf_parser = ConfigParser() - output_dict = conf_parser.parse_config(file_obj) elif form_data.get('excelfile'): file_obj = request.FILES['excelfile'].read().split('\n') _logger.debug('Try to read data from %s' % form_data.get('excelfile')) - conf_parser = ConfigParser() - output_dict = conf_parser.parse_config(file_obj) elif form_data.get('hidden_json_slices'): - output_dict = parse_json_slice_dict(form_data.get('hidden_json_slices')) + spreadsheet_dict = parse_json_slice_dict(form_data.get('hidden_json_slices')) + if not spreadsheet_dict: + conf_parser = ConfigParser() + output_dict = conf_parser.parse_config(file_obj,['formats']) except Exception, e: @@ -527,6 +582,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone # Process the data from request prefill form if (form.cleaned_data.get('excellink') or form.cleaned_data.get('excelfile')) or form.cleaned_data.get('hidden_json_slices'): file_dict, error_message = form_prefill(form.cleaned_data, request) + if error_message != '': # recreate prefill form with error message return render(request, 'prodtask/_requestform.html', { @@ -544,7 +600,14 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone # del form.cleaned_data['tag_hierarchy'] try: form = TRequestCreateCloneConfirmation(form.cleaned_data) - inputlists = [x['input_dict'] for x in file_dict] + inputlists = [] + for slices in file_dict: + slice = slices['input_dict'] + tags = [] + for step in slices.get('step_exec_dict'): + tags.append(step.get('tag')) + inputlists.append(copy.deepcopy(slice)) + inputlists[-1].update({'tags':','.join(tags)}) # store data from prefill form to http request request.session['file_dict'] = file_dict # create request creation form @@ -608,6 +671,7 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone APP_SETTINGS['prodtask.default.email.list'] + cc.replace(';', ',').split(','), fail_silently=True) # Saving slices->steps + step_parent_dict = {} for current_slice in file_dict: input_data = current_slice["input_dict"] input_data['request'] = req @@ -617,7 +681,6 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone _logger.debug("Filling input data: %s" % input_data) irl = InputRequestList(**input_data) irl.save() - step_parent_dict = {} for step in current_slice.get('step_exec_dict'): st = fill_template(step['step_name'], step['tag'], input_data['priority'], step.get('formats', None), step.get('memory', None)) @@ -640,9 +703,12 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone st_exec = StepExecution(**step['step_exec']) if step_parent_dict: if ('step_parent' in step) and ('step_order' in step): - st_exec.step_parent = step_parent_dict[step['step_parent']] + if (step['step_parent']==step['step_order']): + upadte_after = True + else: + st_exec.step_parent = step_parent_dict[step['step_parent']] else: - st_exec.step_parent = step_parent_dict[0] + upadte_after = True else: upadte_after = True if task_config: diff --git a/atlas/prodtask/spdstodb.py b/atlas/prodtask/spdstodb.py index 98c8df15..be3283e2 100644 --- a/atlas/prodtask/spdstodb.py +++ b/atlas/prodtask/spdstodb.py @@ -126,6 +126,7 @@ def translate_excl_to_dict(excel_dict): index += 1 reduce_input_format = False + step_index = 0 for currentstep in StepExecution.STEPS: if translated_row.get(currentstep): st = currentstep @@ -150,8 +151,14 @@ def translate_excl_to_dict(excel_dict): reduce_input_format = True else: formats = 'AOD' + if step_index != 0: + step_index_parent = step_index - 1 + else: + step_index_parent = 0 st_sexec_list.append({'step_name' :st, 'tag': tag, 'formats': formats, 'step_exec': sexec, - 'task_config':task_config}) + 'task_config':task_config,'step_order':str(index)+'_'+str(step_index), + 'step_parent':str(index)+'_'+str(step_index_parent)}) + step_index += 1 return_list.append({'input_dict':irl, 'step_exec_dict':st_sexec_list}) return return_list diff --git a/atlas/prodtask/step_manage_views.py b/atlas/prodtask/step_manage_views.py index 5b3072ff..d786e88d 100644 --- a/atlas/prodtask/step_manage_views.py +++ b/atlas/prodtask/step_manage_views.py @@ -24,6 +24,45 @@ def tag_info(request, tag_name): pass return HttpResponse(json.dumps(results), content_type='application/json') +@csrf_protect +def clone_slices_in_req(request, reqid): + if request.method == 'POST': + results = {'success':False} + try: + data = request.body + input_dict = json.loads(data) + slices = input_dict + #form levels from input text lines + #create chains for each input + new_slice_number = InputRequestList.objects.filter(request=reqid).count() + for slice_number in slices: + current_slice = InputRequestList.objects.filter(request=reqid,slice=int(slice_number)) + new_slice = current_slice.values()[0] + new_slice['slice'] = new_slice_number + new_slice_number += 1 + del new_slice['id'] + new_input_data = InputRequestList(**new_slice) + new_input_data.save() + step_execs = StepExecution.objects.filter(slice=current_slice) + ordered_existed_steps, parent_step = form_existed_step_list(step_execs) + for step in ordered_existed_steps: + step.id = None + step.slice = new_input_data + if step.status == 'Skipped': + step.status = 'NotCheckedSkipped' + elif step.status == 'Approved': + step.status = 'NotChecked' + step.save_with_current_time() + if parent_step: + step.step_parent = parent_step + else: + step.step_parent = step + step.save() + parent_step = step + except Exception,e: + pass + return HttpResponse(json.dumps(results), content_type='application/json') + @csrf_protect def step_params_from_tag(request, reqid): if request.method == 'POST': diff --git a/atlas/prodtask/templates/prodtask/_previewreq.html b/atlas/prodtask/templates/prodtask/_previewreq.html index 7be960f0..c194f505 100644 --- a/atlas/prodtask/templates/prodtask/_previewreq.html +++ b/atlas/prodtask/templates/prodtask/_previewreq.html @@ -30,6 +30,7 @@

      Input List

      Dataset: Job Option: Brief: + Tags: {% for inputList in inputLists %} @@ -37,6 +38,7 @@

      Input List

      {{ inputList.dataset }} {{ inputList.input_data }} {{ inputList.brief }} + {{ inputList.tags }} {% endfor %} diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index eb379c8a..aa4d09df 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -607,6 +607,33 @@ }); return false; }; + function cloneSlicesClick(event){ + event.preventDefault(); + var sliceList=[]; + var i=0; + $(".checkboxSlice:checked").each(function(){ + {# 8 = "checkbox".length#} + var sliceID = $(this).attr("id").slice(8); + sliceList.push(sliceID); + }); + sendData = sliceList; + $( "#reloadDialog" ).dialog(); + $.ajax({ + url: "{% url 'prodtask:clone_slices_in_req' pr_id %}", + type: 'POST', + contentType: 'application/json; charset=utf-8', + data: $.toJSON(sendData), + dataType: 'text', + success: function(data,status) { + $( "#reloadDialog" ).dialog('close'); + if(status){ + location.reload(); + } + } + } + ); + return false; + };
    • -
    -
    - -
    -
    -
    diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index fa6d31c7..69ab167f 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -729,6 +729,9 @@ def form_step_obj(step,task,foreign=False): edit_mode = True slice_steps = [x[1] for x in slice_steps_list] + [form_step_obj({},{})]*(len(STEPS_LIST)-len(slice_steps_list)) + approved = get_approve_status(slice_steps[:len(slice_steps_list)]) + if (approved == 'approved')or(approved == 'partially_approved'): + approved_count += 1 if another_chain_step: input_lists.append((slice, slice_steps, get_approve_status(slice_steps), show_task, another_chain_step.id, approve_level(slice_steps))) From 32ed319c9816337d5fee965580e6774ed2f09f13 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 16 Oct 2014 17:26:29 +0400 Subject: [PATCH 75/93] Change default values --- atlas/prodtask/request_views.py | 11 ++++++++--- atlas/prodtask/templates/prodtask/_requestform.html | 11 ++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index e79a355d..b0822468 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -291,8 +291,10 @@ def parse_json_slice_dict(json_string): for merge_option in merge_options: if slice[merge_option]: task_config.update({merge_option:slice[merge_option]}) - - task_config.update({'project_mode':'cmtconfig='+slice['cmtconfig']+';'+slice['projectmode']}) + if slice['cmtconfig']: + task_config.update({'project_mode':'cmtconfig='+slice['cmtconfig']+';'+slice['projectmode']}) + else: + task_config.update({'project_mode':slice['projectmode']}) if slice['token']: task_config.update({'token':'dst:'+slice['token'].replace('dst:','')}) @@ -310,7 +312,10 @@ def parse_json_slice_dict(json_string): task_config = {} nEventsPerJob = step['eventsperjob'] task_config.update({'nEventsPerJob':dict((x,nEventsPerJob) for x in StepExecution.STEPS)}) - task_config.update({'project_mode':'cmtconfig='+step['cmtconfig']+';'+step['projectmode']}) + if step['cmtconfig']: + task_config.update({'project_mode':'cmtconfig='+step['cmtconfig']+';'+step['projectmode']}) + else: + task_config.update({'project_mode':step['projectmode']}) if step['jediTag']: task_config.update({'merging_tag':step['jediTag']}) for merge_option in merge_options: diff --git a/atlas/prodtask/templates/prodtask/_requestform.html b/atlas/prodtask/templates/prodtask/_requestform.html index 6aac9bf3..a24c7480 100644 --- a/atlas/prodtask/templates/prodtask/_requestform.html +++ b/atlas/prodtask/templates/prodtask/_requestform.html @@ -278,17 +278,17 @@
    @@ -305,8 +305,9 @@
    From df2f567e40ab461db10cf2908569e057ebb7e817 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 16 Oct 2014 17:34:51 +0400 Subject: [PATCH 76/93] change default value --- atlas/prodtask/templates/prodtask/_requestform.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atlas/prodtask/templates/prodtask/_requestform.html b/atlas/prodtask/templates/prodtask/_requestform.html index a24c7480..40be3d99 100644 --- a/atlas/prodtask/templates/prodtask/_requestform.html +++ b/atlas/prodtask/templates/prodtask/_requestform.html @@ -282,8 +282,8 @@
    -
    From 783b6779e7b3db1980192a8c9dbc9eaa42f25cb9 Mon Sep 17 00:00:00 2001 From: stavrik Date: Thu, 16 Oct 2014 20:55:30 +0700 Subject: [PATCH 77/93] CTag filtering --- atlas/prodtask/task_views.py | 2 ++ .../prodtask/templates/prodtask/_task_table.html | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/atlas/prodtask/task_views.py b/atlas/prodtask/task_views.py index af95c181..eabfcbba 100644 --- a/atlas/prodtask/task_views.py +++ b/atlas/prodtask/task_views.py @@ -357,6 +357,8 @@ class Parameters(datatables.Parametrized): step_name = datatables.Parameter(label='Step Name', model_field='step__step_template__step') step_output_format = datatables.Parameter(label='Step output format', get_Q=lambda v: Q( **{ 'step__step_template__output_formats__iexact' : v } ) ) + ctag = datatables.Parameter(label='CTag exact', get_Q=lambda v: Q( **{ 'step__step_template__ctag__iexact' : v } ) ) + task_name = datatables.Parameter(label='Task name', name='taskname', id='taskname', get_Q=lambda v: Q( **{ 'name__iregex' : v } ) ) class Meta: diff --git a/atlas/prodtask/templates/prodtask/_task_table.html b/atlas/prodtask/templates/prodtask/_task_table.html index 0717efde..aed6fee1 100755 --- a/atlas/prodtask/templates/prodtask/_task_table.html +++ b/atlas/prodtask/templates/prodtask/_task_table.html @@ -134,6 +134,13 @@ + + Chain: + + + + + @@ -142,16 +149,16 @@ - + - + From e2c3f2b6f20e30238be57b38c95ee23372fabc25 Mon Sep 17 00:00:00 2001 From: stavrik Date: Thu, 16 Oct 2014 21:34:14 +0700 Subject: [PATCH 78/93] Relabel CTag to AMI tag --- atlas/prodtask/task_views.py | 2 +- atlas/prodtask/templates/prodtask/_task_table.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/atlas/prodtask/task_views.py b/atlas/prodtask/task_views.py index eabfcbba..b67a365c 100644 --- a/atlas/prodtask/task_views.py +++ b/atlas/prodtask/task_views.py @@ -357,7 +357,7 @@ class Parameters(datatables.Parametrized): step_name = datatables.Parameter(label='Step Name', model_field='step__step_template__step') step_output_format = datatables.Parameter(label='Step output format', get_Q=lambda v: Q( **{ 'step__step_template__output_formats__iexact' : v } ) ) - ctag = datatables.Parameter(label='CTag exact', get_Q=lambda v: Q( **{ 'step__step_template__ctag__iexact' : v } ) ) + ctag = datatables.Parameter(label='AMI tag exact', get_Q=lambda v: Q( **{ 'step__step_template__ctag__iexact' : v } ) ) task_name = datatables.Parameter(label='Task name', name='taskname', id='taskname', get_Q=lambda v: Q( **{ 'name__iregex' : v } ) ) diff --git a/atlas/prodtask/templates/prodtask/_task_table.html b/atlas/prodtask/templates/prodtask/_task_table.html index aed6fee1..354dad75 100755 --- a/atlas/prodtask/templates/prodtask/_task_table.html +++ b/atlas/prodtask/templates/prodtask/_task_table.html @@ -156,7 +156,7 @@ - + From 83b6c9c09f3e54ab5bfa323efd16b265f2594a69 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Fri, 17 Oct 2014 12:59:34 +0400 Subject: [PATCH 79/93] fix input format --- atlas/prodtask/request_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index b0822468..30a54674 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -297,7 +297,8 @@ def parse_json_slice_dict(json_string): task_config.update({'project_mode':slice['projectmode']}) if slice['token']: task_config.update({'token':'dst:'+slice['token'].replace('dst:','')}) - + if slice['inputFormat']: + task_config.update({'input_format':slice['inputFormat']}) step_name = step_from_tag(slice['ctag']) sexec = dict(status='NotChecked', priority=int(slice['priority']), input_events=int(slice['totalevents'])) From d9c6570f81d747b236a540ae15bf59ca9f457d76 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 20 Oct 2014 14:04:42 +0400 Subject: [PATCH 80/93] Add step reject. --- atlas/prodtask/request_views.py | 10 +++++-- atlas/prodtask/step_manage_views.py | 27 +++++++++++++++++- .../templates/prodtask/_reqdatatable.html | 28 +++++++++++++++++++ atlas/prodtask/urls.py | 1 + 4 files changed, 63 insertions(+), 3 deletions(-) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index 30a54674..cc9bcb17 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -705,8 +705,14 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone req = TRequest(**form.cleaned_data) req.save() - #TODO:Take owner from sso cookies - request_status = RequestStatus(request=req,comment='Request created by WebUI',owner='default', + owner='' + try: + owner = request.user.username + except: + pass + if not owner: + owner = 'default' + request_status = RequestStatus(request=req,comment='Request created by WebUI',owner=owner, status='waiting') request_status.save_with_current_time() current_uri = request.build_absolute_uri(reverse('prodtask:input_list_approve',args=(req.reqid,))) diff --git a/atlas/prodtask/step_manage_views.py b/atlas/prodtask/step_manage_views.py index 36f0141c..31720128 100644 --- a/atlas/prodtask/step_manage_views.py +++ b/atlas/prodtask/step_manage_views.py @@ -6,7 +6,7 @@ from django.views.decorators.csrf import csrf_protect from .views import form_existed_step_list -from .models import StepExecution, InputRequestList, TRequest, Ttrfconfig +from .models import StepExecution, InputRequestList, TRequest, Ttrfconfig, ProductionTask _logger = logging.getLogger('prodtaskwebui') @@ -68,6 +68,31 @@ def clone_slices_in_req(request, reqid): pass return HttpResponse(json.dumps(results), content_type='application/json') +@csrf_protect +def reject_slices_in_req(request, reqid): + if request.method == 'POST': + results = {'success':False} + try: + data = request.body + input_dict = json.loads(data) + slices = input_dict + for slice_number in slices: + current_slice = InputRequestList.objects.filter(request=reqid,slice=int(slice_number)) + new_slice = current_slice.values()[0] + step_execs = StepExecution.objects.filter(slice=current_slice) + ordered_existed_steps, parent_step = form_existed_step_list(step_execs) + for step in ordered_existed_steps: + if ProductionTask.objects.filter(step=step).count() == 0: + step.step_appr_time = None + if step.status == 'Skipped': + step.status = 'NotCheckedSkipped' + elif step.status == 'Approved': + step.status = 'NotChecked' + step.save() + except Exception,e: + pass + return HttpResponse(json.dumps(results), content_type='application/json') + @csrf_protect def step_params_from_tag(request, reqid): if request.method == 'POST': diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index 3230638a..b546c41c 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -806,6 +806,33 @@ ); return false; }; + + function rejectSlicesClick(event){ + var sliceList=[]; + var i=0; + $(".checkboxSlice:checked").each(function(){ + {# 8 = "checkbox".length#} + var sliceID = $(this).attr("id").slice(8); + sliceList.push(sliceID); + }); + sendData = sliceList; + $( "#reloadDialog" ).dialog(); + $.ajax({ + url: "{% url 'prodtask:reject_slices_in_req' pr_id %}", + type: 'POST', + contentType: 'application/json; charset=utf-8', + data: $.toJSON(sendData), + dataType: 'text', + success: function(data,status) { + $( "#reloadDialog" ).dialog('close'); + if(status){ + location.reload(); + } + } + } + ); + return false; + }; - +
    Chain:Campaign: - +
    Campaign:Ctag exact: - +
    Ctag exact:AMI tag exact:
    {{ task.update_owner }}
    {{ task.comments }}
    {{ task.input_dataset }}
    {{ task.output_dataset }}
    {% for od in output_datasets %}{{ od.name }}
    {% endfor %}
    {{ task.physics_tag }}
    From 68bf4403627985a1ade8bbd61e07ccaa9cebf851 Mon Sep 17 00:00:00 2001 From: stavrik Date: Mon, 20 Oct 2014 17:51:23 +0700 Subject: [PATCH 82/93] Physics group list --- .../templates/prodtask/_task_table.html | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/atlas/prodtask/templates/prodtask/_task_table.html b/atlas/prodtask/templates/prodtask/_task_table.html index 354dad75..5fe442c0 100755 --- a/atlas/prodtask/templates/prodtask/_task_table.html +++ b/atlas/prodtask/templates/prodtask/_task_table.html @@ -216,17 +216,34 @@ From e58306bc954fd16977d8e44d48960bae12b598e0 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 20 Oct 2014 19:29:31 +0400 Subject: [PATCH 83/93] optimize request work page --- .../templates/prodtask/_reqdatatable.html | 6 +- atlas/prodtask/views.py | 74 ++++++++++++------- 2 files changed, 52 insertions(+), 28 deletions(-) diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index b546c41c..d4153ff7 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -946,17 +946,17 @@
    With pattern: {{ pattern }}
    - {% if inputList.input_data|length < 10 %} + {% if not_use_input_date_for_pattern %} {% else %} diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index 69ab167f..44ff85a6 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -10,6 +10,7 @@ from django.views.decorators.csrf import csrf_protect, ensure_csrf_cookie from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse +import time from ..prodtask.ddm_api import find_dataset_events from .request_views import fill_dataset @@ -595,7 +596,21 @@ def form_step_obj(step,task,foreign=False): if request.method == 'GET': try: + cur_request = TRequest.objects.get(reqid=rid) + #steps_db = + + steps_db = list(StepExecution.objects.filter(request=rid)) + steps = {} + for current_step in steps_db: + steps[current_step.slice.id] = steps.get(current_step.slice.id,[])+[current_step] + tasks = {} + tasks_db = list(ProductionTask.objects.filter(request=rid).order_by('-submit_time')) + for current_task in tasks_db: + tasks[current_task.step.id] = tasks.get(current_task.step.id,[]) + [current_task] + + # for current_step in steps_db: + # steps[current_step.slice] = steps.get(current_step.slice,[]).append(current_step) if cur_request.request_type != 'MC': STEPS_LIST = [str(x) for x in range(10)] pattern_list_name = [('Empty', ['' for step in STEPS_LIST])] @@ -610,6 +625,7 @@ def form_step_obj(step,task,foreign=False): show_reprocessing = (cur_request.request_type == 'REPROCESSING') or (cur_request.request_type == 'HLT') input_lists_pre = InputRequestList.objects.filter(request=cur_request).order_by('slice') + input_list_count = InputRequestList.objects.filter(request=cur_request).count() # input_lists - list of tuples for end to form. # tuple format: # first element - InputRequestList object @@ -626,40 +642,46 @@ def form_step_obj(step,task,foreign=False): edit_mode = True else: # choose how to form input data pattern: from jobOption or from input dataset + slice_pattern = '*' use_input_date_for_pattern = True if not input_lists_pre[0].input_data: - use_input_date_for_pattern = False - if use_input_date_for_pattern: - slice_pattern = input_lists_pre[0].input_data.split('.') - else: - slice_pattern = input_lists_pre[0].dataset.name.split('.') + use_input_date_for_pattern = False + if input_list_count < 50: + if use_input_date_for_pattern: + slice_pattern = input_lists_pre[0].input_data.split('.') + else: + slice_pattern = input_lists_pre[0].dataset.name.split('.') + for slice in input_lists_pre: - step_execs = StepExecution.objects.filter(slice=slice) + #step_execs = StepExecution.objects.filter(slice=slice) + + #step_execs = [x for x in steps if x.slice == slice] + step_execs = steps[slice.id] slice_steps = {} total_slice += 1 show_task = False # creating a pattern - if use_input_date_for_pattern: - if slice.input_data: - current_slice_pattern = slice.input_data.split('.') - else: - current_slice_pattern='' - else: - if slice.dataset: - current_slice_pattern = slice.dataset.name.split('.') + if input_list_count < 50: + if use_input_date_for_pattern: + if slice.input_data: + current_slice_pattern = slice.input_data.split('.') + else: + current_slice_pattern='' else: - current_slice_pattern='' - - if current_slice_pattern: - for index,token in enumerate(current_slice_pattern): - if index >= len(slice_pattern): - slice_pattern.append(token) + if slice.dataset: + current_slice_pattern = slice.dataset.name.split('.') else: - if token!=slice_pattern[index]: - slice_pattern[index] = os.path.commonprefix([token,slice_pattern[index]]) - slice_pattern[index] += '*' + current_slice_pattern='' + if current_slice_pattern: + for index,token in enumerate(current_slice_pattern): + if index >= len(slice_pattern): + slice_pattern.append(token) + else: + if token!=slice_pattern[index]: + slice_pattern[index] = os.path.commonprefix([token,slice_pattern[index]]) + slice_pattern[index] += '*' # Creating step dict slice_steps_list = [] temp_step_list = [] @@ -667,7 +689,7 @@ def form_step_obj(step,task,foreign=False): for step in step_execs: step_task = {} try: - step_task = ProductionTask.objects.filter(step = step).order_by('-submit_time')[0] + step_task = tasks[step.id][0] except Exception,e: step_task = {} @@ -718,6 +740,7 @@ def form_step_obj(step,task,foreign=False): slice_steps_list.append((current_step[0].id,form_step_obj(current_step[0],current_step[1]))) temp_step_list.pop(index) + for i in range(len(temp_step_list)): j = 0 while (temp_step_list[j][0].step_parent.id!=slice_steps_list[-1][0]): @@ -753,7 +776,8 @@ def form_step_obj(step,task,foreign=False): 'pattern': '.'.join(slice_pattern), 'totalSlice':total_slice, 'edit_mode':edit_mode, - 'show_reprocessing':show_reprocessing + 'show_reprocessing':show_reprocessing, + 'not_use_input_date_for_pattern':not use_input_date_for_pattern }) except Exception, e: _logger.error("Problem with request list page data forming: %s" % e) From 188c95303d4a61a103d86d13c336c36611e8f5c9 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 20 Oct 2014 20:44:58 +0400 Subject: [PATCH 84/93] fix provenance for hlt --- atlas/prodtask/request_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index cc9bcb17..70caeda8 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -204,7 +204,7 @@ def hlt_form_prefill(form_data, request): if not form_data.get('energy_gev'): form_data['energy_gev'] = 8000 if not form_data.get('provenance'): - form_data['provenance'] = 'test' + form_data['provenance'] = 'GP' task_config = {} if 'events_per_job' in output_dict: From 694452fa5b112a178bee019cff4799ed879a840e Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 20 Oct 2014 21:12:18 +0400 Subject: [PATCH 85/93] fix default group name --- atlas/prodtask/forms.py | 4 ++-- atlas/prodtask/request_views.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/atlas/prodtask/forms.py b/atlas/prodtask/forms.py index f8464dc4..06fa20d2 100644 --- a/atlas/prodtask/forms.py +++ b/atlas/prodtask/forms.py @@ -71,7 +71,7 @@ class TRequestHLTCreateCloneForm(TRequestCreateCloneConfirmation): cstatus = CharField(widget=forms.HiddenInput, required=False) request_type = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) - phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS), initial='THLT') campaign = CharField(required=False) hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden") description = CharField(label='Short description', widget=Textarea, required=False) @@ -88,7 +88,7 @@ class TRequestReprocessingCreateCloneForm(TRequestCreateCloneConfirmation): cstatus = CharField(widget=forms.HiddenInput, required=False) request_type = CharField(widget=forms.HiddenInput, required=False) project = ModelChoiceField(queryset=TProject.objects.all(),required=False) - phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS)) + phys_group = CharField(required=False, widget=forms.Select(choices=TRequest.PHYS_GROUPS), initial='REPR') campaign = CharField(required=False) hidden_json_slices = CharField(widget=forms.HiddenInput, required=False, label="Will be hidden") description = CharField(label='Short description', widget=Textarea, required=False) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index 70caeda8..37a5393f 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -205,6 +205,8 @@ def hlt_form_prefill(form_data, request): form_data['energy_gev'] = 8000 if not form_data.get('provenance'): form_data['provenance'] = 'GP' + if not form_data.get('phys_group'): + form_data['phys_group'] = 'THLT' task_config = {} if 'events_per_job' in output_dict: @@ -474,6 +476,8 @@ def reprocessing_form_prefill(form_data, request): form_data['energy_gev'] = 8000 if not form_data.get('provenance'): form_data['provenance'] = 'AP' + if not form_data.get('phys_group'): + form_data['phys_group'] = 'REPR' task_config = {} if 'events_per_job' in output_dict: nEventsPerJob = output_dict['events_per_job'][0] From 055111e4103b059652d2cb146061ec3e5c5bf370 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Mon, 20 Oct 2014 23:10:57 +0400 Subject: [PATCH 86/93] Make atomic transaction for request creation --- atlas/prodtask/request_views.py | 145 +++++++++--------- .../templates/prodtask/_reqdatatable.html | 2 +- 2 files changed, 74 insertions(+), 73 deletions(-) diff --git a/atlas/prodtask/request_views.py b/atlas/prodtask/request_views.py index 37a5393f..e972b4d2 100644 --- a/atlas/prodtask/request_views.py +++ b/atlas/prodtask/request_views.py @@ -10,6 +10,7 @@ from django.utils import timezone from django.db.models import Count, Q from django.views.decorators.csrf import csrf_protect +from django.db import transaction from ..prodtask.ddm_api import find_dataset_events import core.datatables as datatables import json @@ -676,7 +677,6 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone # Process the data from create form form elif 'file_dict' in request.session: #TODO: Waiting message - #TODO: One commission file_dict = request.session['file_dict'] form2 = TRequestCreateCloneConfirmation(request.POST, request.FILES) if not form2.is_valid(): @@ -705,82 +705,83 @@ def request_clone_or_create(request, rid, title, submit_url, TRequestCreateClone # del form.cleaned_data['tag_hierarchy'] form.cleaned_data['cstatus'] = 'waiting' try: - _logger.debug("Creating request : %s" % form.cleaned_data) - - req = TRequest(**form.cleaned_data) - req.save() - owner='' - try: - owner = request.user.username - except: - pass - if not owner: - owner = 'default' - request_status = RequestStatus(request=req,comment='Request created by WebUI',owner=owner, - status='waiting') - request_status.save_with_current_time() - current_uri = request.build_absolute_uri(reverse('prodtask:input_list_approve',args=(req.reqid,))) - _logger.debug("e-mail with link %s" % current_uri) - send_mail('Request %i: %s %s %s' % (req.reqid,req.phys_group,req.campaign,req.description), - request_email_body(longdesc, req.ref_link, req.energy_gev, req.campaign,current_uri), - APP_SETTINGS['prodtask.email.from'], - APP_SETTINGS['prodtask.default.email.list'] + cc.replace(';', ',').split(','), - fail_silently=True) - # Saving slices->steps - step_parent_dict = {} - for current_slice in file_dict: - input_data = current_slice["input_dict"] - input_data['request'] = req - priority_obj = get_priority_object(input_data['priority']) - if input_data.get('dataset'): - input_data['dataset'] = fill_dataset(input_data['dataset']) - _logger.debug("Filling input data: %s" % input_data) - irl = InputRequestList(**input_data) - irl.save() - for step in current_slice.get('step_exec_dict'): - st = fill_template(step['step_name'], step['tag'], input_data['priority'], - step.get('formats', None), step.get('memory', None)) - task_config= {} - upadte_after = False - if 'task_config' in step: - if 'nEventsPerJob' in step['task_config']: - task_config.update({'nEventsPerJob':int(step['task_config']['nEventsPerJob'].get(step['step_name'],-1))}) - if step['step_name']=='Evgen': - task_config.update({'nEventsPerInputFile':int(step['task_config']['nEventsPerJob'].get(step['step_name'],-1))}) - task_config_options = ['project_mode','input_format','token','nFilesPerMergeJob', - 'nGBPerMergeJob','nMaxFilesPerMergeJob','merging_tag'] - for task_config_option in task_config_options: - if task_config_option in step['task_config']: - task_config.update({task_config_option:step['task_config'][task_config_option]}) - step['step_exec']['request'] = req - step['step_exec']['slice'] = irl - step['step_exec']['step_template'] = st - step['step_exec']['priority'] = priority_obj.priority(st.step,st.ctag) - _logger.debug("Filling step execution data: %s" % step['step_exec']) - st_exec = StepExecution(**step['step_exec']) - if step_parent_dict: - if ('step_parent' in step) and ('step_order' in step): - if (step['step_parent']==step['step_order']): - upadte_after = True + with transaction.atomic(): + _logger.debug("Creating request : %s" % form.cleaned_data) + + req = TRequest(**form.cleaned_data) + req.save() + owner='' + try: + owner = request.user.username + except: + pass + if not owner: + owner = 'default' + request_status = RequestStatus(request=req,comment='Request created by WebUI',owner=owner, + status='waiting') + request_status.save_with_current_time() + current_uri = request.build_absolute_uri(reverse('prodtask:input_list_approve',args=(req.reqid,))) + _logger.debug("e-mail with link %s" % current_uri) + send_mail('Request %i: %s %s %s' % (req.reqid,req.phys_group,req.campaign,req.description), + request_email_body(longdesc, req.ref_link, req.energy_gev, req.campaign,current_uri), + APP_SETTINGS['prodtask.email.from'], + APP_SETTINGS['prodtask.default.email.list'] + cc.replace(';', ',').split(','), + fail_silently=True) + # Saving slices->steps + step_parent_dict = {} + for current_slice in file_dict: + input_data = current_slice["input_dict"] + input_data['request'] = req + priority_obj = get_priority_object(input_data['priority']) + if input_data.get('dataset'): + input_data['dataset'] = fill_dataset(input_data['dataset']) + _logger.debug("Filling input data: %s" % input_data) + irl = InputRequestList(**input_data) + irl.save() + for step in current_slice.get('step_exec_dict'): + st = fill_template(step['step_name'], step['tag'], input_data['priority'], + step.get('formats', None), step.get('memory', None)) + task_config= {} + upadte_after = False + if 'task_config' in step: + if 'nEventsPerJob' in step['task_config']: + task_config.update({'nEventsPerJob':int(step['task_config']['nEventsPerJob'].get(step['step_name'],-1))}) + if step['step_name']=='Evgen': + task_config.update({'nEventsPerInputFile':int(step['task_config']['nEventsPerJob'].get(step['step_name'],-1))}) + task_config_options = ['project_mode','input_format','token','nFilesPerMergeJob', + 'nGBPerMergeJob','nMaxFilesPerMergeJob','merging_tag'] + for task_config_option in task_config_options: + if task_config_option in step['task_config']: + task_config.update({task_config_option:step['task_config'][task_config_option]}) + step['step_exec']['request'] = req + step['step_exec']['slice'] = irl + step['step_exec']['step_template'] = st + step['step_exec']['priority'] = priority_obj.priority(st.step,st.ctag) + _logger.debug("Filling step execution data: %s" % step['step_exec']) + st_exec = StepExecution(**step['step_exec']) + if step_parent_dict: + if ('step_parent' in step) and ('step_order' in step): + if (step['step_parent']==step['step_order']): + upadte_after = True + else: + st_exec.step_parent = step_parent_dict[step['step_parent']] else: - st_exec.step_parent = step_parent_dict[step['step_parent']] + upadte_after = True else: upadte_after = True - else: - upadte_after = True - if task_config: - st_exec.set_task_config(task_config) - st_exec.save_with_current_time() - if ('step_parent' in step) and ('step_order' in step): - step_parent_dict.update({step['step_order']:st_exec}) - else: - step_parent_dict.update({0:st_exec}) - if upadte_after: + if task_config: + st_exec.set_task_config(task_config) + st_exec.save_with_current_time() if ('step_parent' in step) and ('step_order' in step): - st_exec.step_parent = step_parent_dict[step['step_parent']] + step_parent_dict.update({step['step_order']:st_exec}) else: - st_exec.step_parent = step_parent_dict[0] - st_exec.save() + step_parent_dict.update({0:st_exec}) + if upadte_after: + if ('step_parent' in step) and ('step_order' in step): + st_exec.step_parent = step_parent_dict[step['step_parent']] + else: + st_exec.step_parent = step_parent_dict[0] + st_exec.save() except Exception, e: _logger.error("Problem during request creat: %s" % str(e)) diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index d4153ff7..6f01b816 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -1085,7 +1085,7 @@
    With pattern: {{ pattern }}
    - + From e9a2eee7666a01d6b1557a33ec9741e2babc5630 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 23 Oct 2014 11:43:16 +0400 Subject: [PATCH 87/93] add link to jobsetid --- atlas/prodtask/templates/prodtask/_reqdatatable.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index 6f01b816..ffb940c8 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -867,8 +867,11 @@
    {{ inputList.slice }} - {{ inputList.dataset.name|truncatechars:"58" }} + {{ inputList.dataset.name }} {{ inputList.input_data }} - {{ inputList.input_data|truncatechars:"58" }} + {{ inputList.input_data}} {{ inputList.dataset.name }}{{ trequest.cstatus}}
    -Make as test +
    Total input: {{ totalSlice }}, from them approved: {{ approvedCount }}
    +{% if approvedCount > 0 %} + All jobs from request +{% endif %}
    With pattern: {{ pattern }}
    @@ -1291,5 +1294,5 @@
    With pattern: {{ pattern }}
    × - +Make as test {% endblock %} \ No newline at end of file From ab040f213acc17ecddb57b3b6759b6a738fbcf4f Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 23 Oct 2014 15:22:41 +0400 Subject: [PATCH 88/93] fix buttons to work in firefox --- .../templates/prodtask/_reqdatatable.html | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index ffb940c8..ce8f5919 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -509,7 +509,8 @@ return false; }; function tagFormatChange(){ - sendData = {'tag_format':$("#tagFormatSelect").val()}; + event.preventDefault(); + sendData = {'tag_format':$("#tagFormatSelect").val()}; $( "#waitingDialog" ).dialog(); $.ajax({ url: "{% url 'prodtask:project_mode_from_tag' pr_id %}", @@ -536,7 +537,7 @@ }; function refreshTagFormats(event){ - sendData = {}; + sendData = {}; $( "#waitingDialog" ).dialog(); $.ajax({ url: "{% url 'prodtask:get_tag_formats' pr_id%}", @@ -562,7 +563,8 @@ }; function updateStepParametrs(event){ - sendData = {'tag_format':$("#tagFormatSelect").val(),'project_mode':$("#project_mode_input").val(), + event.preventDefault(); + sendData = {'tag_format':$("#tagFormatSelect").val(),'project_mode':$("#project_mode_input").val(), 'input_events':$('#input_events_input').val(),'priority':$('#priority_input').val(), 'nEventsPerJob':$('#nEventsPerJob_input').val(), 'nEventsPerInputFile':$('#nEventsPerInputFile_input').val(), @@ -588,6 +590,7 @@ }; function findInputClick(event) { + event.preventDefault(); var sliceList=[]; var i=0; $(".checkboxSlice:checked").each(function(){ @@ -667,7 +670,7 @@ $(document).foundation(); function formSliceInfo(id){ - event.preventDefault(); +{# event.preventDefault();#} var sliceID = id.slice(10); $("#sliceInfo").attr({'class':'small content f-dropdown'}); $("#sliceInfo").html(''); @@ -712,7 +715,7 @@ compareTable.append($("",{text: 'save', class: 'button', onclick:'saveOneSlice(event);' })); $("#sliceInfo").append(compareTable); } - + return false; }; function saveOneSlice(event){ @@ -755,6 +758,7 @@ } } } + if (step_types.length != current_step_tag.length){ is_equal = false; } @@ -808,6 +812,7 @@ }; function rejectSlicesClick(event){ + event.preventDefault(); var sliceList=[]; var i=0; $(".checkboxSlice:checked").each(function(){ From 9c835890aac4127cd145c2dfda612f4f54880e16 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Thu, 23 Oct 2014 17:54:30 +0400 Subject: [PATCH 89/93] fix step fields --- atlas/prodtask/templates/prodtask/_reqdatatable.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/templates/prodtask/_reqdatatable.html b/atlas/prodtask/templates/prodtask/_reqdatatable.html index ce8f5919..e804fef7 100644 --- a/atlas/prodtask/templates/prodtask/_reqdatatable.html +++ b/atlas/prodtask/templates/prodtask/_reqdatatable.html @@ -509,7 +509,7 @@ return false; }; function tagFormatChange(){ - event.preventDefault(); +{# event.preventDefault();#} sendData = {'tag_format':$("#tagFormatSelect").val()}; $( "#waitingDialog" ).dialog(); $.ajax({ @@ -537,6 +537,10 @@ }; function refreshTagFormats(event){ + if(event != undefined){ + event.preventDefault(); + } + sendData = {}; $( "#waitingDialog" ).dialog(); $.ajax({ From 2c8c6f7aefb6475ff9c6a784eaf3ca8f5376e1f4 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Sun, 26 Oct 2014 00:26:37 +0400 Subject: [PATCH 90/93] change default edit mode to True --- atlas/prodtask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index 44ff85a6..3a77d3cf 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -636,7 +636,7 @@ def form_step_obj(step,task,foreign=False): approved_count = 0 total_slice = 0 slice_pattern = [] - edit_mode = False + edit_mode = True fully_approved = 0 if not input_lists_pre: edit_mode = True From 1ccaa256e98ec8f3a247b6faedcd6de520c0bfac Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Tue, 28 Oct 2014 15:26:42 +0300 Subject: [PATCH 91/93] Add table for output check. --- atlas/prodtask/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/atlas/prodtask/models.py b/atlas/prodtask/models.py index 2774823b..9a2ff790 100644 --- a/atlas/prodtask/models.py +++ b/atlas/prodtask/models.py @@ -173,6 +173,15 @@ class Meta: app_label = 'grisli' db_table = u'T_TRF_CONFIG' +class TDataFormatAmi(models.Model): + format = models.CharField(max_length=32, db_column='FORMAT', primary_key=True) + description = models.CharField(max_length=256, db_column='DESCRIPTION') + status = models.CharField(max_length=8, db_column='STATUS') + last_modified = models.DateTimeField(db_column='LASTMODIFIED') + + class Meta: + app_label = 'grisli' + db_table = u'T_DATA_FORMAT_AMI' class ProductionDataset(models.Model): name = models.CharField(max_length=150, db_column='NAME', primary_key=True) From 14c91e11a88e31340c6dba2fe660798cf19e8eb3 Mon Sep 17 00:00:00 2001 From: mishaborodin Date: Tue, 28 Oct 2014 18:55:25 +0300 Subject: [PATCH 92/93] fix empty slice handling --- atlas/prodtask/views.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/atlas/prodtask/views.py b/atlas/prodtask/views.py index 3a77d3cf..3a84f613 100644 --- a/atlas/prodtask/views.py +++ b/atlas/prodtask/views.py @@ -656,7 +656,10 @@ def form_step_obj(step,task,foreign=False): #step_execs = StepExecution.objects.filter(slice=slice) #step_execs = [x for x in steps if x.slice == slice] - step_execs = steps[slice.id] + try: + step_execs = steps[slice.id] + except: + step_execs = [] slice_steps = {} total_slice += 1 From 9d8d9dc4daed60461b1521c266768f76c820b8ef Mon Sep 17 00:00:00 2001 From: sbel Date: Mon, 3 Nov 2014 19:48:29 +0100 Subject: [PATCH 93/93] DEFT API client updated to the recent version --- atlas/deftcore/api/client/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/atlas/deftcore/api/client/__init__.py b/atlas/deftcore/api/client/__init__.py index 1618f2d1..5234c5c9 100644 --- a/atlas/deftcore/api/client/__init__.py +++ b/atlas/deftcore/api/client/__init__.py @@ -88,6 +88,14 @@ def retry_task(self, owner, task_id): body = {'task_id': task_id} return self._create_request('retry_task', owner, body) + def change_task_ram_count(self, owner, task_id, ram_count): + body = {'task_id': task_id, 'ram_count': ram_count} + return self._create_request('change_task_ram_count', owner, body) + + def change_task_wall_time(self, owner, task_id, wall_time): + body = {'task_id': task_id, 'wall_time': wall_time} + return self._create_request('change_task_wall_time', owner, body) + def add_task_comment(self, owner, task_id, comment): body = {'task_id': task_id, 'comment_body': comment} return self._create_request('add_task_comment', owner, body)