From 95954b39d25514a3cd11b66052a81b56002909b7 Mon Sep 17 00:00:00 2001 From: JAMES LETTS Date: Tue, 29 Apr 2014 16:48:32 -0700 Subject: [PATCH] Cleanup. --- bashrc | 8 ++--- condor_check | 34 ++++++++---------- dashboard_functions.sh | 80 ++++++++++++++---------------------------- 3 files changed, 45 insertions(+), 77 deletions(-) diff --git a/bashrc b/bashrc index dc2c190..c135bd6 100644 --- a/bashrc +++ b/bashrc @@ -9,9 +9,9 @@ source /etc/profile.d/condor.sh #################### MACE CHANGES ABOVE THIS LINE #################### # Discover the directory where the software sits: -RELEASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +export glideinWMSMonitor_RELEASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" # source the functions to discover SiteDB, HTCondor and CMS Dashboard information -source $RELEASE_DIR/sitedb_functions.sh -source $RELEASE_DIR/condor_functions.sh -source $RELEASE_DIR/dashboard_functions.sh +source $glideinWMSMonitor_RELEASE_DIR/sitedb_functions.sh +source $glideinWMSMonitor_RELEASE_DIR/condor_functions.sh +source $glideinWMSMonitor_RELEASE_DIR/dashboard_functions.sh diff --git a/condor_check b/condor_check index 0665c42..0ffdeec 100644 --- a/condor_check +++ b/condor_check @@ -1,16 +1,16 @@ #!/bin/bash + +if [ -z $glideinWMSMonitor_RELEASE_DIR ] ; then + echo "ERROR: source code missing." + exit 1 +else + source $glideinWMSMonitor_RELEASE_DIR/bashrc +fi + POOLNAME=$1 SHORT=$2 if [ "X"$SHORT == "X" ] ; then SHORT="long" ; fi -source /etc/profile.d/condor.sh - -source /home/letts/scripts/sitedb_functions.sh -source /home/letts/scripts/condor_functions.sh -source /home/letts/scripts/dashboard_functions.sh - -export X509_USER_PROXY=/tmp/x509up_u500 - # Header echo Summary Table for glidinWMS pool $POOLNAME at `/bin/date -u` # Summary Table from the Collector @@ -20,21 +20,23 @@ echo negotime=`condor_status -pool $POOLNAME -nego -l | grep LastNegotiationCycleDuration0 | awk '{print $3}'` || exit 3 echo "Negotiation time = ${negotime}s" echo + # dont print the long table if asked if [ $SHORT == "short" ] ; then exit 0 fi - # get information from sitedb about pledges and se names by CMSSite name PLEDGES=`get_pledges_from_sitedb` SEDFILE=`translate_se_names_in_sitedb_to_cmssite` # get information from dashboard about avg and max usage by CMSSite name -USAGE=`dashboard_usage_by_site analysis` -USAGEUW=`dashboard_usage_at_wisconsin analysis` -MAXUSAGE=`dashboard_usage_by_site all` -MAXUSAGEUW=`dashboard_usage_at_wisconsin all` +USAGE=`dashboard_usage_by_site analysis "3 months ago"` +MAXUSAGE=`dashboard_usage_by_site all "3 months ago"` + +# Information from T2_US_Wisconsin before April 3, 2014 is inflated. +USAGEUW=`dashboard_usage_at_site analysis "3 weeks ago"` +MAXUSAGEUW=`dashboard_usage_at_site all "3 weeks ago"` # get information from SSB about site downtimes DOWNTIMES=`site_downtimes_from_ssb` @@ -50,7 +52,6 @@ printf "%-20s%10s%10s%10s%10s%10s%10s%10s%10s%10s %-18s\n" "Site" "Pledge" "Ple printf "%30s%10s%10s%10s%40s%10s\n" "Updated" "Analysis" "Usage" "Usage" "Pressure" "Prod" echo - # Sum up some interesting quantities: totalpledge=0 totalclaimed=0 @@ -60,18 +61,15 @@ totalpledge=0 totalavgusage=0 totalmaxusage=0 - # Loop over sites for the table: sites=`cat $SEDFILE | awk -F\/ '{print $3}' | sort | uniq` for site in $sites ; do - # Claimed pilots at site claimed=`grep \{$site\} $CLAIMED | awk '{print $1}'` if [ "X"$claimed == "X" ] ; then claimed=0 ; fi totalclaimed=$[$totalclaimed+$claimed] - # Idle pilots at site = running - claimed: unclaimed=`grep \{$site\} $RUNNING | awk -v x=$claimed '{print $1-x}'` if [ "X"$unclaimed == "X" ] ; then unclaimed=0 ; fi @@ -85,12 +83,10 @@ for site in $sites ; do # Get queeud jobs for site (not exclusively): pressure=`grep $site $DESIRED | wc -l` - # Get jobs queued only for this site: exclusivepressure=`grep $site $DESIRED | awk -v site=$site '$1==site{print $0}' | wc -l` totalexclusivepressure=$[$totalexclusivepressure+$exclusivepressure] - # Extract pledge for this site (total for all activities): # Federation pledges may have duplicated entries (one SiteDB, one REBUS), so take the last one. pledge=`grep ^$site\, $PLEDGES | tail -1 | awk -F\, '{print int($2/2.)}'` diff --git a/dashboard_functions.sh b/dashboard_functions.sh index d2087cd..f38e9d1 100644 --- a/dashboard_functions.sh +++ b/dashboard_functions.sh @@ -1,7 +1,14 @@ #!/bin/sh -# get the downtimes from SSB, remove any quotes so the output is just csv list of site,downtime string site_downtimes_from_ssb() { + # get the downtimes from SSB, remove any quotes so the output + # is just csv list of [site,downtime string] + # + # Usage: + # site_downtimes_from_ssb + # Output: + # File name of temporary file containing the csv list. + OUTPUTFILE=`mktemp -t DOWNTIMES.csv.XXXXXXXXXX` || return 1 url="http://dashb-ssb.cern.ch/dashboard/request.py/getallshort?view=maint" curl -ks -H 'Accept:text/csv' $url | awk -F\, '{print $1 "," $3}' \ @@ -10,68 +17,33 @@ site_downtimes_from_ssb() { return 0 } - -# function to print out a csv list of sites and avg and max job slots used daily during the past -# 3 months a certain activity, which can be "all", "analysis", "analysistest", "production", etc. - dashboard_usage_by_site() { - OUTPUTFILE=`mktemp -t USAGE.csv.XXXXXXXXXX` || return 1 - - # argument is the activity to list e.g. analysis, production, all - activity=$1 + # function to print out a csv list of sites and avg and + # max job slots used daily during the past TIMEFRAME of + # a certain activity, which can be "all", "analysis", + # "analysistest", "production", etc. + # + # Usage: + # dashboard_usage_by_site analysis "3 months ago" + # dashboard_usage_by_site all "2 weeks ago" + # + # Output: + # File name of a temporary file containing the csv list. + # + # N.B. Data before April 3, 2014 for T2_US_Wisconsin is over-inflated! - # look at the last 3 months: - date1=`date -d "3 month ago" +%F` - date2=`date +%F` - - # Data before April 3, 2014 for Wisconsin is garbage. - - # url for dashboard historical usage by site: - url="http://dashb-cms-jobsmry.cern.ch/dashboard/request.py/jobnumberscsv?sites=All%20T3210&datatiers=All%20DataTiers&applications=All%20Application%20Versions&submissions=All%20Submission%20Types&accesses=All%20Access%20Types&activities=${activity}&sitesSort=7&start=${date1}&end=${date2}&timeRange=daily&granularity=daily&generic=0&sortBy=0&series=All&type=r" - curl -ks $url | dos2unix | sort -t \, -k 3 | awk -F\, ' - BEGIN{ - lastsite="None" - totaljobslots=0 - totaldays=0 - maxjobslots=0 - } - { - site=$3 - if ( site != lastsite && lastsite != "None" ){ - slotsperday=int(totaljobslots/totaldays) - printf("%s,%i,%i\n",lastsite,slotsperday,maxjobslots) - totaljobslots=0 - totaldays=0 - maxjobslots=0 - } - lastsite=site - totaljobslots+=$1 - totaldays+=1 - if ( $1 > maxjobslots ) { maxjobslots=$1 } - } - END{ - slotsperday=int(totaljobslots/totaldays) - printf("%s,%i,%i\n",site,slotsperday,maxjobslots) - }' > $OUTPUTFILE - echo "$OUTPUTFILE" - return 0 -} - - -dashboard_usage_at_wisconsin() { OUTPUTFILE=`mktemp -t USAGE.csv.XXXXXXXXXX` || return 1 # argument is the activity to list e.g. analysis, production, all - activity=$1 + ACTIVITY=$1 + TIMEFRAME=$2 # look at the last 3 months: - date1=`date -d "2 weeks ago" +%F` + date1=`date -d "$TIMEFRAME" +%F` date2=`date +%F` - # Data before April 3, 2014 for Wisconsin is garbage. - # url for dashboard historical usage by site: - url="http://dashb-cms-jobsmry.cern.ch/dashboard/request.py/jobnumberscsv?sites=All%20T3210&datatiers=All%20DataTiers&applications=All%20Application%20Versions&submissions=All%20Submission%20Types&accesses=All%20Access%20Types&activities=${activity}&sitesSort=7&start=${date1}&end=${date2}&timeRange=daily&granularity=daily&generic=0&sortBy=0&series=All&type=r" + url="http://dashb-cms-jobsmry.cern.ch/dashboard/request.py/jobnumberscsv?sites=All%20T3210&datatiers=All%20DataTiers&applications=All%20Application%20Versions&submissions=All%20Submission%20Types&accesses=All%20Access%20Types&activities=${ACTIVITY}&sitesSort=7&start=${date1}&end=${date2}&timeRange=daily&granularity=daily&generic=0&sortBy=0&series=All&type=r" curl -ks $url | dos2unix | sort -t \, -k 3 | awk -F\, ' BEGIN{ lastsite="None" @@ -81,7 +53,7 @@ dashboard_usage_at_wisconsin() { } { site=$3 - if ( site != lastsite && lastsite == "T2_US_Wisconsin" ){ + if ( site != lastsite && lastsite != "None" ){ slotsperday=int(totaljobslots/totaldays) printf("%s,%i,%i\n",lastsite,slotsperday,maxjobslots) totaljobslots=0